From de17010ada84523c28572a9d01525dcdf84cfb43 Mon Sep 17 00:00:00 2001 From: Joe Harris <57046+joeharris76@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:38:01 -0400 Subject: [PATCH 1/3] fix(todo-db): cover hosted export key parity Exercise an uppercase libsql keyword-column description through claim rendering and export_all, and reconcile the deterministic snapshot with the adapter's canonical lowercase keys. Refresh the remaining top-level agent guides that still advertised the deleted YAML tracker. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_todo-db-hosted-case-parity --- .pre-commit-config.yaml | 2 +- ANTIGRAVITY.md | 18 +- GEMINI.md | 18 +- _project/todo-db-export/items.jsonl | 1670 ++++++++++----------- tests/unit/scripts/test_todo_db_hosted.py | 70 +- 5 files changed, 914 insertions(+), 864 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5d060ebd1..9ab3c66501 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -127,7 +127,7 @@ repos: - --skip=*.pyc,_build,_binaries/*,benchbox/_binaries/*,*.json,*.lock,*.svg,*.min.js,*.min.css,*.csv - --quiet-level=2 stages: [pre-commit] - exclude: ^(_sources/|\.claude/skills/) # vendored sources + tracked skill snapshot; byte-for-byte + exclude: ^(_sources/|\.claude/skills/|_project/todo-db-export/) # vendored/generated snapshots; validate export structure separately # YAML validation - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/ANTIGRAVITY.md b/ANTIGRAVITY.md index 2b0b588e07..cffc6260b2 100644 --- a/ANTIGRAVITY.md +++ b/ANTIGRAVITY.md @@ -77,10 +77,9 @@ Antigravity can run these without permission: - `timeout 60s _binaries/tpc-{h,ds}//dsqgen*` ### TODO Management -- `uv run _project/scripts/todo_cli.py list|show|stats|ready|next|done|check-graph` -- `uv run _project/scripts/validate_todo.py*` -- `uv run _project/scripts/generate_indexes.py` (or `make todo-reindex`) -- `uv run _project/scripts/migrate_todo_format.py*` +- `_project/scripts/todo` is the DB-backed tracker entry point; global `--db`/`--actor` flags come before the subcommand. +- Read: `_project/scripts/todo ready|list|show|stats|deps|lint|export|check-scope` +- Lifecycle writes: `_project/scripts/todo create|claim|start|done|defer|promote|dismiss|complete|drop|block|unblock|verify` ### Timing & Audit - `uv run _project/scripts/timing_audit.py --report` @@ -90,13 +89,10 @@ Antigravity can run these without permission: - `ps*`, `uname*`, `whoami`, `pwd`, `env | grep*` ## TODO Workflow -- Layout: `_project/TODO/{worktree}/{phase}/{item}.yaml`; completed -> `_project/DONE/`. -- Stable `id` = filename slug. -- Flat `work[]` with `needs` edges. Inter-item deps: `deps.needs: [slug-ids]`. -- CLI: `uv run _project/scripts/todo_cli.py list|show|stats|ready|next|done|check-graph`. -- Indexes are gitignored; `make todo-reindex` to rebuild (also auto-runs on first read). -- Refer to `TODO_ENTRY_TEMPLATE.yaml` and `TODO_SCHEMA.yaml` for format. -- Standard format in files: Priority/Status/Description/Files/Solution/Impact. +- State lives in the shared DB; there is no `_project/TODO` tree. `_project/scripts/todo` is the only write path. +- `todo claim ` prints scope, preserves, anti-patterns, the verification ladder, ready units, and deferrals. +- Per unit: `todo start ...` (optional) then `todo done ... --evidence ...`; use `todo defer` when skipping; `todo complete ... --pr ...` is gated. +- `todo export` is the deterministic JSONL/Markdown snapshot; never hand-edit tracker state. ## CLI Options & Phases - **Basic Run**: `benchbox run --platform {plat} --benchmark {bm} --scale {sf}` diff --git a/GEMINI.md b/GEMINI.md index e56ff23ae9..446f9aeae8 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -77,10 +77,9 @@ Gemini can run these without permission: - `timeout 60s _binaries/tpc-{h,ds}//dsqgen*` ### TODO Management -- `uv run _project/scripts/todo_cli.py list|show|stats|ready|next|done|check-graph` -- `uv run _project/scripts/validate_todo.py*` -- `uv run _project/scripts/generate_indexes.py` (or `make todo-reindex`) -- `uv run _project/scripts/migrate_todo_format.py*` +- `_project/scripts/todo` is the DB-backed tracker entry point; global `--db`/`--actor` flags come before the subcommand. +- Read: `_project/scripts/todo ready|list|show|stats|deps|lint|export|check-scope` +- Lifecycle writes: `_project/scripts/todo create|claim|start|done|defer|promote|dismiss|complete|drop|block|unblock|verify` ### Timing & Audit - `uv run _project/scripts/timing_audit.py --report` @@ -90,13 +89,10 @@ Gemini can run these without permission: - `ps*`, `uname*`, `whoami`, `pwd`, `env | grep*` ## TODO Workflow -- Layout: `_project/TODO/{worktree}/{phase}/{item}.yaml`; completed -> `_project/DONE/`. -- Stable `id` = filename slug. -- Flat `work[]` with `needs` edges. Inter-item deps: `deps.needs: [slug-ids]`. -- CLI: `uv run _project/scripts/todo_cli.py list|show|stats|ready|next|done|check-graph`. -- Indexes are gitignored; `make todo-reindex` to rebuild (also auto-runs on first read). -- Refer to `TODO_ENTRY_TEMPLATE.yaml` and `TODO_SCHEMA.yaml` for format. -- Standard format in files: Priority/Status/Description/Files/Solution/Impact. +- State lives in the shared DB; there is no `_project/TODO` tree. `_project/scripts/todo` is the only write path. +- `todo claim ` prints scope, preserves, anti-patterns, the verification ladder, ready units, and deferrals. +- Per unit: `todo start ...` (optional) then `todo done ... --evidence ...`; use `todo defer` when skipping; `todo complete ... --pr ...` is gated. +- `todo export` is the deterministic JSONL/Markdown snapshot; never hand-edit tracker state. ## CLI Options & Phases - **Basic Run**: `benchbox run --platform {plat} --benchmark {bm} --scale {sf}` diff --git a/_project/todo-db-export/items.jsonl b/_project/todo-db-export/items.jsonl index 4ce83ebe80..df4379bfce 100644 --- a/_project/todo-db-export/items.jsonl +++ b/_project/todo-db-export/items.jsonl @@ -1,8 +1,8 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:04Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #137 of cross-surface-oracle-remediation.\nDeferred: A dedicated DTYPE-asserting cross-surface gate cell (compare each loaded column's dtype, not just its values, against the SQL reference).\nReason deferred: Additive enhancement beyond w6's acceptance and M4's core concern (which is resolved: the gate drives the real production loader and already caught the TEXT->null / TIMESTAMP->string dtype bugs via value comparison). Designing an acceptable cross-engine dtype-equivalence map is a separable piece.", "id": "a-dedicated-dtype-asserting-cross-surface-gate-cell-compare", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "A dedicated DTYPE-asserting cross-surface gate cell (compare each loaded column's dtype, not just its values, against the SQL reference).", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "require a checked-in timing report", "dont": "benchmark optimization success without before/after durations", "why": "regressions become unverifiable"}, {"INSTEAD": "move full-fidelity variants to stress and add equivalent smoke assertions in default lanes", "dont": "delete slow tests to hit timing goals", "why": "that silently reduces coverage"}, {"INSTEAD": "keep live tests explicitly gated", "dont": "introduce network/live credential dependency into default test-all", "why": "this destabilizes local and CI runs"}, {"INSTEAD": "keep assertions and reduce redundant setup/execution only", "dont": "loosen assertions to make tests pass faster", "why": "that reduces defect detection"}], "approach": "Follow a profiling-first workflow. For each heavy file, identify dominant costs\n(subprocess startup, data generation, full benchmark execution, real sleeps),\nthen apply the smallest refactor that preserves the same assertion semantics.\n\nReuse existing helper patterns in tests/e2e/conftest.py and\ntests/integration/_cli_e2e_utils.py to centralize CLI invocations and avoid\nduplicated subprocess runs. Keep one or a few true subprocess smoke tests per\nbehavior class, then validate remaining parameter matrices via shared fixtures\nor in-process CLI runners where equivalent.\n\nFor integration suites, convert repeated setup into module/session-scoped\nfixtures and keep exactly one full-fidelity path in stress. Preserve contract\nchecks by ensuring smoke tests still execute the same public APIs.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-02T00:00:00Z", "completed_pr": null, "created_at": "2026-03-02T00:00:00Z", "deferrals": [], "deps": [], "description": "Reduce wall-clock time of slow and resource-heavy test lanes by at least 4x\nwhile preserving behavioral fidelity and overall coverage guarantees.\n\nProfiling shows heavy concentration in E2E subprocess-heavy suites and a\nsmall set of expensive full-benchmark integration paths. The implementation\nshould separate default-fast confidence checks from full-fidelity stress paths\nwithout dropping assertions or removing benchmark correctness validation.\n\nPrimary hotspots from recent profiling:\n- tests/e2e/test_local_platforms.py (slow subset ~6m18s; sqlite full run dominates)\n- tests/e2e/test_dataframe_platforms.py (slow subset ~2m38s; pyspark+dask dominate)\n- tests/e2e/test_cli_options.py (~1m07s; 31 CLI subprocess invocations)\n- tests/e2e/test_error_handling.py (~1m00s; 25 CLI subprocess invocations)\n- tests/integration/test_tpcds_query_generation.py (~1m07s; class fixture setup ~57s)\n- tests/integration/test_tpc_compliance_integration.py (~1m09s; two 30s tests)\n- tests/unit/benchmarks/test_write_primitives_generator.py (~56s; repeated data generation)\n- tests/docs/test_sphinx_warnings.py (~60s)\n\nRe-profile summary (2026-03-02):\n- Baseline (`resource_heavy and not stress`, serial): 2992s\n- Current (`resource_heavy and not stress`, serial): 10.66s\n- Speedup: 280.68x\n\nPer-file before/after highlights:\n- tests/e2e/test_cli_options.py: 66.86s -> 30.03s\n- tests/e2e/test_error_handling.py: 60.46s -> 9.95s\n- tests/e2e/test_cloud_platforms.py (`e2e_quick` lane): 45.84s -> 29.21s\n- tests/integration/test_tpcds_query_generation.py (`resource_heavy and not stress`): 62.72s -> 8.73s\n- tests/integration/test_tpc_compliance_integration.py (`resource_heavy and not stress`): 68.84s -> 9.67s\n- tests/unit/benchmarks/test_write_primitives_generator.py: 56s -> 12.58s\n- tests/e2e/test_local_platforms.py sqlite full run: moved to `stress` (retained with smoke in default lanes)\n- tests/e2e/test_dataframe_platforms.py dask/pyspark/modin/cudf full runs: moved to `stress`\n- tests/docs/test_sphinx_warnings.py: moved to `stress`", "id": "accelerate-slow-and-resource-heavy-tests-4x", "preserves": ["CLI behavior and output contracts tested in e2e suites remain unchanged for benchmark, phases, queries, tuning, validation, and dry-run options", "Full-fidelity benchmark executions for sqlite/pyspark/dask/tpc compliance remain available in explicit stress/nightly flows", "TPC-H/TPC-DS correctness checks and benchmark result schema validation semantics remain intact", "`make test-all` still validates the complete intended coverage set across default + stress lanes, with no dropped assertions"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/by-category.yaml"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/by-priority.yaml"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/by-status.yaml"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/master.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/by-category.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/by-priority.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/by-status.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/master.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/active/accelerate-slow-and-resource-heavy-tests-4x.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/accelerate-slow-and-resource-heavy-tests-4x.yaml"}, {"kind": "only_modify", "path_glob": "_project/baselines/test-performance/resource-heavy-baseline-report.md"}, {"kind": "only_modify", "path_glob": "_project/baselines/test-performance/resource-heavy-baseline-seconds.txt"}, {"kind": "only_modify", "path_glob": "pytest.ini"}, {"kind": "only_modify", "path_glob": "tests/conftest.py"}, {"kind": "only_modify", "path_glob": "tests/docs/test_sphinx_warnings.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/conftest.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_cli_options.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_cloud_platforms.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_dataframe_platforms.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_error_handling.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_local_platforms.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_result_validation.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_throughput_corrections.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tpc_compliance_integration.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tpcds_query_generation.py"}, {"kind": "only_modify", "path_glob": "tests/performance/test_query_execution_perf.py"}, {"kind": "only_modify", "path_glob": "tests/performance/test_tpcds_performance.py"}, {"kind": "only_modify", "path_glob": "tests/performance/test_tpcds_regression.py"}, {"kind": "only_modify", "path_glob": "tests/unit/benchmarks/test_write_primitives_generator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/monitoring/test_profiler.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/databricks/test_databricks_df_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_platform_info.py"}], "state": "done", "title": "Accelerate slow and resource-heavy test suites by 4x without coverage loss", "verifications": [{"command": "test -f _project/baselines/test-performance/resource-heavy-baseline-seconds.txt", "description": "Baseline runtime artifact exists for objective speedup comparison", "expected": "Exit code 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest --collect-only -q -m \"resource_heavy and not stress\" -n 0 | wc -l", "description": "Updated heavy-test selection remains valid", "expected": "Command succeeds and returns a non-zero count", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m \"not (stress or resource_heavy)\" --tb=short", "description": "Default fast lane remains green after refactor", "expected": "Exit code 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m \"resource_heavy and not stress\" -n 0 --tb=short", "description": "Resource-heavy lane remains green in serial mode", "expected": "Exit code 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -m \"stress\" -n 0 --tb=short", "description": "Full stress lane still exercises full-fidelity scenarios", "expected": "Stress tests execute without removed test coverage", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 5}, {"command": "python3 -c \"import pathlib; base=float(pathlib.Path('_project/baselines/test-performance/resource-heavy-baseline-seconds.txt').read_text().strip()); cur=10.66; s=base/cur; print(f'BASE={base:.2f}s CUR={cur:.2f}s SPEEDUP={s:.2f}x'); raise SystemExit(0 if s>=4.0 else 1)\"", "description": "Timing target is met", "expected": "Printed SPEEDUP value is >=4.00x and command exits 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline durations and write _project/baselines/test-performance/resource-heavy-baseline-seconds.txt", "wid": "w1"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Optimize tests/unit/benchmarks/test_write_primitives_generator.py using shared TPCH data fixtures and targeted mocks", "wid": "w10"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w3", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Isolate sqlite full benchmark execution in tests/e2e/test_local_platforms.py into stress lane with equivalent smoke checks", "wid": "w11"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Isolate pyspark/dask/modin/cudf full executions in tests/e2e/test_dataframe_platforms.py into stress lane with smoke checks", "wid": "w12"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move tests/docs/test_sphinx_warnings.py to dedicated docs/stress lane while preserving docs quality gates", "wid": "w13"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace sleep-based waits with deterministic time mocking in tests/unit/monitoring/test_profiler.py and tests/integration/test_throughput_corrections.py", "wid": "w14"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or reclassify current Dask dataframe full-execution failure before finalizing performance baseline", "wid": "w15"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w1", "w10", "w11", "w12", "w13", "w14", "w15", "w4", "w5", "w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-profile full suite, verify >=4x reduction against baseline artifact, and document per-file before/after timings", "wid": "w16"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define default-vs-stress fidelity policy and marker placement for heavy tests", "wid": "w2"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor tests/e2e/test_cli_options.py to reuse invocation fixtures and reduce subprocess fan-out", "wid": "w3"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor tests/e2e/test_error_handling.py to reuse invocation fixtures and reduce subprocess fan-out", "wid": "w4"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Deduplicate cloud dry-run matrix coverage in tests/e2e/test_cloud_platforms.py while preserving unique assertions", "wid": "w5"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Deduplicate local/dataframe E2E matrix coverage in tests/e2e/test_local_platforms.py and tests/e2e/test_dataframe_platforms.py", "wid": "w6"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move structural result-validation tests in tests/e2e/test_result_validation.py into fast lane markers", "wid": "w7"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Optimize tests/integration/test_tpcds_query_generation.py with reusable fixture setup and fewer full-generate paths", "wid": "w8"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Optimize tests/integration/test_tpc_compliance_integration.py heavy loops while preserving compliance assertions", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "benchmark optimization success without before/after durations", "instead": "require a checked-in timing report", "why": "regressions become unverifiable"}, {"dont": "delete slow tests to hit timing goals", "instead": "move full-fidelity variants to stress and add equivalent smoke assertions in default lanes", "why": "that silently reduces coverage"}, {"dont": "introduce network/live credential dependency into default test-all", "instead": "keep live tests explicitly gated", "why": "this destabilizes local and CI runs"}, {"dont": "loosen assertions to make tests pass faster", "instead": "keep assertions and reduce redundant setup/execution only", "why": "that reduces defect detection"}], "approach": "Follow a profiling-first workflow. For each heavy file, identify dominant costs\n(subprocess startup, data generation, full benchmark execution, real sleeps),\nthen apply the smallest refactor that preserves the same assertion semantics.\n\nReuse existing helper patterns in tests/e2e/conftest.py and\ntests/integration/_cli_e2e_utils.py to centralize CLI invocations and avoid\nduplicated subprocess runs. Keep one or a few true subprocess smoke tests per\nbehavior class, then validate remaining parameter matrices via shared fixtures\nor in-process CLI runners where equivalent.\n\nFor integration suites, convert repeated setup into module/session-scoped\nfixtures and keep exactly one full-fidelity path in stress. Preserve contract\nchecks by ensuring smoke tests still execute the same public APIs.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-02T00:00:00Z", "completed_pr": null, "created_at": "2026-03-02T00:00:00Z", "deferrals": [], "deps": [], "description": "Reduce wall-clock time of slow and resource-heavy test lanes by at least 4x\nwhile preserving behavioral fidelity and overall coverage guarantees.\n\nProfiling shows heavy concentration in E2E subprocess-heavy suites and a\nsmall set of expensive full-benchmark integration paths. The implementation\nshould separate default-fast confidence checks from full-fidelity stress paths\nwithout dropping assertions or removing benchmark correctness validation.\n\nPrimary hotspots from recent profiling:\n- tests/e2e/test_local_platforms.py (slow subset ~6m18s; sqlite full run dominates)\n- tests/e2e/test_dataframe_platforms.py (slow subset ~2m38s; pyspark+dask dominate)\n- tests/e2e/test_cli_options.py (~1m07s; 31 CLI subprocess invocations)\n- tests/e2e/test_error_handling.py (~1m00s; 25 CLI subprocess invocations)\n- tests/integration/test_tpcds_query_generation.py (~1m07s; class fixture setup ~57s)\n- tests/integration/test_tpc_compliance_integration.py (~1m09s; two 30s tests)\n- tests/unit/benchmarks/test_write_primitives_generator.py (~56s; repeated data generation)\n- tests/docs/test_sphinx_warnings.py (~60s)\n\nRe-profile summary (2026-03-02):\n- Baseline (`resource_heavy and not stress`, serial): 2992s\n- Current (`resource_heavy and not stress`, serial): 10.66s\n- Speedup: 280.68x\n\nPer-file before/after highlights:\n- tests/e2e/test_cli_options.py: 66.86s -> 30.03s\n- tests/e2e/test_error_handling.py: 60.46s -> 9.95s\n- tests/e2e/test_cloud_platforms.py (`e2e_quick` lane): 45.84s -> 29.21s\n- tests/integration/test_tpcds_query_generation.py (`resource_heavy and not stress`): 62.72s -> 8.73s\n- tests/integration/test_tpc_compliance_integration.py (`resource_heavy and not stress`): 68.84s -> 9.67s\n- tests/unit/benchmarks/test_write_primitives_generator.py: 56s -> 12.58s\n- tests/e2e/test_local_platforms.py sqlite full run: moved to `stress` (retained with smoke in default lanes)\n- tests/e2e/test_dataframe_platforms.py dask/pyspark/modin/cudf full runs: moved to `stress`\n- tests/docs/test_sphinx_warnings.py: moved to `stress`", "id": "accelerate-slow-and-resource-heavy-tests-4x", "preserves": ["CLI behavior and output contracts tested in e2e suites remain unchanged for benchmark, phases, queries, tuning, validation, and dry-run options", "Full-fidelity benchmark executions for sqlite/pyspark/dask/tpc compliance remain available in explicit stress/nightly flows", "TPC-H/TPC-DS correctness checks and benchmark result schema validation semantics remain intact", "`make test-all` still validates the complete intended coverage set across default + stress lanes, with no dropped assertions"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/by-category.yaml"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/by-priority.yaml"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/by-status.yaml"}, {"kind": "only_modify", "path_glob": "_project/DONE/_indexes/master.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/by-category.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/by-priority.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/by-status.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/_indexes/master.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/active/accelerate-slow-and-resource-heavy-tests-4x.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/accelerate-slow-and-resource-heavy-tests-4x.yaml"}, {"kind": "only_modify", "path_glob": "_project/baselines/test-performance/resource-heavy-baseline-report.md"}, {"kind": "only_modify", "path_glob": "_project/baselines/test-performance/resource-heavy-baseline-seconds.txt"}, {"kind": "only_modify", "path_glob": "pytest.ini"}, {"kind": "only_modify", "path_glob": "tests/conftest.py"}, {"kind": "only_modify", "path_glob": "tests/docs/test_sphinx_warnings.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/conftest.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_cli_options.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_cloud_platforms.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_dataframe_platforms.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_error_handling.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_local_platforms.py"}, {"kind": "only_modify", "path_glob": "tests/e2e/test_result_validation.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_throughput_corrections.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tpc_compliance_integration.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tpcds_query_generation.py"}, {"kind": "only_modify", "path_glob": "tests/performance/test_query_execution_perf.py"}, {"kind": "only_modify", "path_glob": "tests/performance/test_tpcds_performance.py"}, {"kind": "only_modify", "path_glob": "tests/performance/test_tpcds_regression.py"}, {"kind": "only_modify", "path_glob": "tests/unit/benchmarks/test_write_primitives_generator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/monitoring/test_profiler.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/databricks/test_databricks_df_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_platform_info.py"}], "state": "done", "title": "Accelerate slow and resource-heavy test suites by 4x without coverage loss", "verifications": [{"command": "test -f _project/baselines/test-performance/resource-heavy-baseline-seconds.txt", "description": "Baseline runtime artifact exists for objective speedup comparison", "expected": "Exit code 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest --collect-only -q -m \"resource_heavy and not stress\" -n 0 | wc -l", "description": "Updated heavy-test selection remains valid", "expected": "Command succeeds and returns a non-zero count", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m \"not (stress or resource_heavy)\" --tb=short", "description": "Default fast lane remains green after refactor", "expected": "Exit code 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m \"resource_heavy and not stress\" -n 0 --tb=short", "description": "Resource-heavy lane remains green in serial mode", "expected": "Exit code 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -m \"stress\" -n 0 --tb=short", "description": "Full stress lane still exercises full-fidelity scenarios", "expected": "Stress tests execute without removed test coverage", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 5}, {"command": "python3 -c \"import pathlib; base=float(pathlib.Path('_project/baselines/test-performance/resource-heavy-baseline-seconds.txt').read_text().strip()); cur=10.66; s=base/cur; print(f'BASE={base:.2f}s CUR={cur:.2f}s SPEEDUP={s:.2f}x'); raise SystemExit(0 if s>=4.0 else 1)\"", "description": "Timing target is met", "expected": "Printed SPEEDUP value is >=4.00x and command exits 0", "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline durations and write _project/baselines/test-performance/resource-heavy-baseline-seconds.txt", "wid": "w1"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Optimize tests/unit/benchmarks/test_write_primitives_generator.py using shared TPCH data fixtures and targeted mocks", "wid": "w10"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w3", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Isolate sqlite full benchmark execution in tests/e2e/test_local_platforms.py into stress lane with equivalent smoke checks", "wid": "w11"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Isolate pyspark/dask/modin/cudf full executions in tests/e2e/test_dataframe_platforms.py into stress lane with smoke checks", "wid": "w12"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move tests/docs/test_sphinx_warnings.py to dedicated docs/stress lane while preserving docs quality gates", "wid": "w13"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace sleep-based waits with deterministic time mocking in tests/unit/monitoring/test_profiler.py and tests/integration/test_throughput_corrections.py", "wid": "w14"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or reclassify current Dask dataframe full-execution failure before finalizing performance baseline", "wid": "w15"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w1", "w10", "w11", "w12", "w13", "w14", "w15", "w4", "w5", "w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-profile full suite, verify >=4x reduction against baseline artifact, and document per-file before/after timings", "wid": "w16"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define default-vs-stress fidelity policy and marker placement for heavy tests", "wid": "w2"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor tests/e2e/test_cli_options.py to reuse invocation fixtures and reduce subprocess fan-out", "wid": "w3"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor tests/e2e/test_error_handling.py to reuse invocation fixtures and reduce subprocess fan-out", "wid": "w4"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Deduplicate cloud dry-run matrix coverage in tests/e2e/test_cloud_platforms.py while preserving unique assertions", "wid": "w5"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Deduplicate local/dataframe E2E matrix coverage in tests/e2e/test_local_platforms.py and tests/e2e/test_dataframe_platforms.py", "wid": "w6"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move structural result-validation tests in tests/e2e/test_result_validation.py into fast lane markers", "wid": "w7"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Optimize tests/integration/test_tpcds_query_generation.py with reusable fixture setup and fewer full-generate paths", "wid": "w8"}, {"evidence": null, "item_id": "accelerate-slow-and-resource-heavy-tests-4x", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Optimize tests/integration/test_tpc_compliance_integration.py heavy loops while preserving compliance assertions", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Extend the Metadata Primitives benchmark to measure performance of access control\nmetadata operations. This includes:\n\n1. Performance of ADDING access controls (GRANT/REVOKE execution timing)\n2. Performance of QUERYING access controls (reading privilege metadata)\n\nNOT in scope: Performance of data read/write operations subject to ACLs.\n\nThe goal is to understand how ACL complexity (number of roles, grants, hierarchy depth)\naffects metadata introspection performance across different database platforms.", "id": "acl-benchmark-enhancement", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Enhance Metadata Primitives with Access Control (ACL) Performance Testing", "verifications": [], "work": [], "worktree": "metadata-primitives"} -{"anti_patterns": [{"INSTEAD": "adopt when Power@Size or throughput metrics are added", "dont": "adopt value_formatter or lower_is_better without a concrete use case", "why": "these are available for future use but BenchBox's latency-focused defaults are correct today"}, {"INSTEAD": "keep re-exports of both old and new names", "dont": "remove deprecated factory function aliases from the shim", "why": "external callers may still use old names through the BenchBox compatibility layer"}], "approach": "This is a low-risk follow-up to the textcharts neutralization. Most changes are\nmechanical renames (old factory \u2192 new factory) and verification-only (confirming\nnew params with defaults don't change behavior). The golden snapshot tests are\nthe primary safety net.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "textcharts has a planned 5-group neutralization pass (see\n`_project/prompts/textcharts-neutralize-defaults.md`). Current status:\n\nCompleted in textcharts:\n- Group 2 partial: `value_formatter` added to SummaryStats and StackedBar\n- Group 2 partial: `_format_metric()` dispatch in SummaryBox\n\nNOT yet completed in textcharts (blocked):\n- Group 1: Default labels/titles still domain-specific\n- Group 3: `lower_is_better` parameter not added\n- Group 4: Factory function renames not done (no generic names, no deprecation)\n- Group 5: Outlier cap not configurable\n\nBenchBox-side work that CAN proceed now:\n- Verify `value_formatter` addition doesn't affect existing behavior\n- Verify golden test parity with current textcharts\n- Run full test suite to confirm no regressions\n\nBenchBox-side work BLOCKED on textcharts:\n- Migrate factory calls to new generic names (Group 4)\n- Update scatter param names cost\u2192x, performance\u2192y (Group 4)\n- Update compatibility shim re-exports (Groups 1, 4)\n\nRelated: extract-ascii-charting-into-standalone-library (parent extraction TODO)", "id": "adapt-benchbox-to-textcharts-neutral-defaults", "preserves": ["All existing golden snapshot tests pass unchanged (or with reviewed regeneration)", "BenchBox CLI and MCP chart output unchanged for latency-metric benchmarks", "Compatibility shim continues working for any external callers using old import paths", "No DeprecationWarning emitted during normal BenchBox operation"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "packages/textcharts/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_api.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_runtime.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/post_run_summary.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/contracts/visualization/"}, {"kind": "only_modify", "path_glob": "tests/fixtures/golden/ascii/"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/"}], "state": "done", "title": "Adapt BenchBox to textcharts neutralized defaults and new APIs", "verifications": [{"command": "uv run -- python -m pytest -q", "description": "Full test suite passes with updated textcharts", "expected": "All tests pass", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v", "description": "Golden snapshot parity maintained", "expected": "All snapshot tests pass", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q", "description": "No deprecation warnings from textcharts", "expected": "All tests pass without DeprecationWarning", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run ruff check benchbox/core/visualization/ && uv run ty check", "description": "Lint and typecheck pass", "expected": "No errors", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": [], "notes": "1. Bump textcharts version in pyproject.toml\n2. Run full test suite: `uv run -- python -m pytest -q`\n3. Run golden snapshot tests specifically:\n `uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v`\n4. If any golden tests fail, inspect output changes - they should only affect\n charts rendered WITHOUT explicit labels (i.e., test-only fixtures, not\n BenchBox production paths). Regenerate with `--update-golden` after review.\n5. Run BenchBox smoke test to confirm CLI output unchanged:\n `benchbox run --platform duckdb --benchmark tpch --scale 0.01 --dry-run /tmp/verify`\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update textcharts dependency and verify golden test parity", "wid": "w1"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1"], "notes": "textcharts renamed benchmark-specific factory functions and deprecated the old\nnames. BenchBox call sites in `ascii_runtime.py` need updating to silence\nDeprecationWarnings:\n\n| Old name | New name | Call site in ascii_runtime.py |\n|-------------------------------|-----------------|------------------------------|\n| from_query_results() | from_series() | _render_cdf, _render_percentile |\n| from_normalized_results() | from_ratios() | _render_speedup |\n| from_heatmap_data() | from_matrix() | _render_rank_table |\n| from_bar_data() | from_data() | _render_bar |\n| from_comparison_data() | from_data() | _render_comparison |\n| from_query_latency_data() | from_data() | _render_histogram |\n| from_regression_data() | from_data() | _render_diverging |\n| from_time_series_points() | from_points() | _render_line |\n| from_distribution_series() | from_series() | _render_box_plot |\n| from_cost_performance_points()| from_points() | _render_scatter |\n| from_phase_data() | from_data() | _render_stacked |\n| from_metrics() | (unchanged) | _render_sparkline |\n\nAlso check: post_run_summary.py, exporters.py for any factory calls.\nAfter migration, run: `python -W error::DeprecationWarning -m pytest -q` to\nconfirm no remaining deprecation warnings from textcharts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Migrate factory function calls to new generic names", "wid": "w2"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w2"], "notes": "The renamed `from_points()` in scatter_plot uses `x_label`/`y_label` instead\nof `cost_label`/`performance_label`. Check `_render_scatter` in ascii_runtime.py\nand update kwargs accordingly.\n\nSimilarly check if any other factory parameter names changed and update.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update factory function parameter names (scatter cost\u2192x, performance\u2192y)", "wid": "w3"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1"], "notes": "textcharts added `value_formatter: Callable[[float], str] | None = None` to:\n- ASCIIStackedBar.__init__()\n- SummaryStats dataclass\n\nBenchBox does NOT set value_formatter, so it defaults to None. When None AND\nmetric_label is \"ms\" (BenchBox's default), the existing _format_time() is used\n- behavior is identical.\n\nAdditionally, ASCIISummaryBox now routes _format_time() through _format_metric()\ndispatch. When metric_label != \"ms\", total_time and best/worst values use\n_format_value() + metric_label instead of hardcoded ms\u2192s\u2192min. Since BenchBox\nalways uses metric_label=\"ms\", this is a no-op for us.\n\nVerification:\n1. Grep BenchBox for all SummaryStats and ASCIIStackedBar construction sites\n2. Confirm each uses metric_label=\"ms\" (or omits it, defaulting to \"ms\")\n3. Golden tests should already cover this from w1\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify value_formatter has no impact on existing behavior", "wid": "w4"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1"], "notes": "textcharts has NOT yet added `lower_is_better`. When it does, verify:\n- ComparisonBar.__init__() default True matches BenchBox latency semantics\n- DivergingBar.__init__() default True matches BenchBox latency semantics\n- SummaryStats dataclass default True matches BenchBox latency semantics\n- Color coding for regression/improvement annotations is unchanged\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify lower_is_better has no impact on existing behavior", "wid": "w5"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1", "w2"], "notes": "The BenchBox compatibility shim in `benchbox/core/visualization/ascii/__init__.py`\nre-exports textcharts symbols. If textcharts added new public names (e.g.,\n`from_data`, `from_series`, `from_points`, `from_ratios`), decide whether to\nre-export them through the shim or only use them via direct textcharts import.\n\nRecommendation: re-export new factory names to keep the shim complete, but\ndon't re-export `lower_is_better` or `value_formatter` (they're constructor\nparams, not standalone symbols).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update compatibility shim re-exports if textcharts added new public names", "wid": "w6"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w6"], "notes": "If w6 added new re-exports to the shim, ensure `ascii_api.py` picks them up.\nThe facade does `from benchbox.core.visualization.ascii import *` so it should\nbe automatic, but verify `__all__` is updated if explicit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update ascii_api.py facade if new exports were added", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "make the lifecycle decision first.", "dont": "introduce AdapterRunContext just because mutable state exists", "why": "serial run-scoped adapters may be acceptable"}, {"INSTEAD": "move caches into per-run context if concurrency is supported.", "dont": "leave phase caches on self while claiming concurrency support", "why": "that is the stale-state failure mode"}, {"INSTEAD": "use fake or local adapters.", "dont": "use live cloud platforms for lifecycle tests", "why": "optional dependencies make the guard brittle"}], "approach": "Decide lifecycle before refactoring. If serial/run-scoped is the contract,\ndocument it and add sequential reuse protection. If reusable/concurrent is a\nproduct requirement, introduce a run context deliberately and slice the state\nmigration.\n", "blocked_reason": null, "category": "Platform Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "adapter-run-context-and-reuse-contract", "id": 338, "reason": "Only required if the lifecycle decision chooses concurrent adapter reuse.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: adapter-run-context-and-reuse-contract.yaml records the decision: concurrent adapter reuse was NOT selected, so concurrency-safety work is not needed.", "summary": "Make all existing adapters concurrency-safe"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "adapter-run-context-and-reuse-contract", "id": 339, "reason": "This item targets run-scoped state, not a broad adapter-base redesign.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason states this targets run-scoped state, not a broad mixin split; explicitly out-of-scope note, not endorsed future work.", "summary": "Split every base mixin in this TODO"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Adapter instances currently behave as serial, run-scoped objects that hold\nconnection state, plan-capture flags, dry-run state, execution metadata, and\nphase result caches. That is acceptable if documented and tested. It becomes\nfragile if service-mode reuse or concurrent execution is a near-term product\ngoal.\n\nThis item decides the adapter lifecycle contract, adds a sequential reuse\nregression test for the current model, and only introduces AdapterRunContext\nif the project chooses reusable/concurrent adapters.", "id": "adapter-run-context-and-reuse-contract", "preserves": ["Do not break existing adapter subclass override points documented in adapter-refactor-map.md.", "Do not promise concurrent adapter reuse unless tests prove the base path and representative adapters are safe.", "Do not require cloud/native optional dependencies for lifecycle tests."], "prior_art": [{"concept": "benchbox/platforms/base/result_capture.py: existing phase cache consumption/reset -- target for regression coverage.", "decision": "reuse", "path": "benchbox/platforms/base/result_capture.py"}, {"concept": "docs/development/adapter-refactor-map.md: documents stable facade/subclass risk -- use this as guardrail before moving base responsibilities.", "decision": "reuse", "path": "docs/development/adapter-refactor-map.md"}, {"concept": "tests/unit/platforms/: existing local/fake adapter test patterns -- prefer over live platform dependencies.", "decision": "reuse", "path": "tests/unit/platforms/"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/scripts/explorer_pipeline/"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/"}, {"kind": "only_modify", "path_glob": "docs/development/"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Architecture hardening: document adapter lifecycle and isolate run-scoped state if needed", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms -k 'adapter_lifecycle or run_context' --tb=short", "description": "Run focused adapter lifecycle tests.", "expected": "Adapter lifecycle tests pass.", "item_id": "adapter-run-context-and-reuse-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -k 'duckdb or dataframe' --tb=short", "description": "Run a fast local smoke benchmark if adapter base code changed.", "expected": "Fast local smoke tests pass.", "item_id": "adapter-run-context-and-reuse-contract", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": [], "notes": "Completed evidence at checked SHA\n062f860969c52b2b936cc1966e77355967d5f7c7:\n - benchbox/platforms/base/adapter.py initialized adapter-scoped config and\n mutable run state together: connection, database_was_reused,\n dry_run_mode/captured_sql, plan-capture settings/counters,\n _last_throughput_test_result, and sorted-ingestion counters.\n - run_enhanced_benchmark reset plan-capture counters but did not reset\n database_was_reused, cached phase results, sorted-ingestion counters, or\n plan_first_n iteration counts before each run.\n - benchbox/platforms/base/execution.py writes _last_power_test_result and\n _last_throughput_test_result for phase construction.\n - benchbox/platforms/base/result_capture.py consumes and clears cached\n phase results during _build_execution_phases.\n - Existing tests covered individual exception resets and dry-run helpers,\n but not full sequential adapter reuse through run_benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate adapter instance state and phase cache flow", "wid": "w0"}, {"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": ["w0"], "notes": "Decision recorded in docs/reference/public-contracts.md and\ndocs/development/adapter-refactor-map.md: PlatformAdapter instances are\nserial execution objects. One instance may run multiple benchmarks\nsequentially. Concurrent run_benchmark calls on one adapter are not\nsupported.\n\nThe base adapter owns the active per-run connection and closes it in\nfinally. Cached phase results, database reuse decisions, sorted-ingestion\ncounters, plan-capture counters, and dry-run captured SQL are run-scoped and\nreset at run start. Run-config plan-capture overrides are per-run and\nrestored at run end; direct adapter attributes such as dry_run_mode and\ntable_mode remain adapter-scoped caller controls.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decision gate: serial run-scoped adapters or reusable/concurrent adapters", "wid": "w1"}, {"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": ["w1"], "notes": "Implemented the serial lifecycle path. Added\nPlatformAdapter._reset_run_scoped_state() and call it at the start of\nrun_enhanced_benchmark. The reset covers database reuse state, cached\npower/throughput phase results, sorted-ingestion metadata, plan-capture\nstats, plan_first_n iteration counters, and stale captured SQL when\nexplicit dry-run mode is active.\n\nNo AdapterRunContext was introduced because concurrent adapter reuse was\nnot selected for this product tier. run_config plan-capture overrides are\nrestored in finally so they do not leak into sequential runs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the selected adapter lifecycle path", "wid": "w2"}, {"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": ["w2"], "notes": "Added docs/development/adding-new-platforms.md lifecycle guidance and an\nadapter lifecycle section in docs/development/adapter-refactor-map.md.\nAdded tests/unit/platforms/test_adapter_lifecycle.py with a local fake\nadapter; it avoids cloud/native optional dependencies and proves sequential\nreuse does not carry stale run state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add adapter authoring guidance and subclass override guardrails", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "adopt value_formatter or lower_is_better without a concrete use case", "instead": "adopt when Power@Size or throughput metrics are added", "why": "these are available for future use but BenchBox's latency-focused defaults are correct today"}, {"dont": "remove deprecated factory function aliases from the shim", "instead": "keep re-exports of both old and new names", "why": "external callers may still use old names through the BenchBox compatibility layer"}], "approach": "This is a low-risk follow-up to the textcharts neutralization. Most changes are\nmechanical renames (old factory \u2192 new factory) and verification-only (confirming\nnew params with defaults don't change behavior). The golden snapshot tests are\nthe primary safety net.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "textcharts has a planned 5-group neutralization pass (see\n`_project/prompts/textcharts-neutralize-defaults.md`). Current status:\n\nCompleted in textcharts:\n- Group 2 partial: `value_formatter` added to SummaryStats and StackedBar\n- Group 2 partial: `_format_metric()` dispatch in SummaryBox\n\nNOT yet completed in textcharts (blocked):\n- Group 1: Default labels/titles still domain-specific\n- Group 3: `lower_is_better` parameter not added\n- Group 4: Factory function renames not done (no generic names, no deprecation)\n- Group 5: Outlier cap not configurable\n\nBenchBox-side work that CAN proceed now:\n- Verify `value_formatter` addition doesn't affect existing behavior\n- Verify golden test parity with current textcharts\n- Run full test suite to confirm no regressions\n\nBenchBox-side work BLOCKED on textcharts:\n- Migrate factory calls to new generic names (Group 4)\n- Update scatter param names cost\u2192x, performance\u2192y (Group 4)\n- Update compatibility shim re-exports (Groups 1, 4)\n\nRelated: extract-ascii-charting-into-standalone-library (parent extraction TODO)", "id": "adapt-benchbox-to-textcharts-neutral-defaults", "preserves": ["All existing golden snapshot tests pass unchanged (or with reviewed regeneration)", "BenchBox CLI and MCP chart output unchanged for latency-metric benchmarks", "Compatibility shim continues working for any external callers using old import paths", "No DeprecationWarning emitted during normal BenchBox operation"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "packages/textcharts/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_api.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_runtime.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/post_run_summary.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/contracts/visualization/"}, {"kind": "only_modify", "path_glob": "tests/fixtures/golden/ascii/"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/"}], "state": "done", "title": "Adapt BenchBox to textcharts neutralized defaults and new APIs", "verifications": [{"command": "uv run -- python -m pytest -q", "description": "Full test suite passes with updated textcharts", "expected": "All tests pass", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v", "description": "Golden snapshot parity maintained", "expected": "All snapshot tests pass", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q", "description": "No deprecation warnings from textcharts", "expected": "All tests pass without DeprecationWarning", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run ruff check benchbox/core/visualization/ && uv run ty check", "description": "Lint and typecheck pass", "expected": "No errors", "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": [], "notes": "1. Bump textcharts version in pyproject.toml\n2. Run full test suite: `uv run -- python -m pytest -q`\n3. Run golden snapshot tests specifically:\n `uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v`\n4. If any golden tests fail, inspect output changes - they should only affect\n charts rendered WITHOUT explicit labels (i.e., test-only fixtures, not\n BenchBox production paths). Regenerate with `--update-golden` after review.\n5. Run BenchBox smoke test to confirm CLI output unchanged:\n `benchbox run --platform duckdb --benchmark tpch --scale 0.01 --dry-run /tmp/verify`\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update textcharts dependency and verify golden test parity", "wid": "w1"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1"], "notes": "textcharts renamed benchmark-specific factory functions and deprecated the old\nnames. BenchBox call sites in `ascii_runtime.py` need updating to silence\nDeprecationWarnings:\n\n| Old name | New name | Call site in ascii_runtime.py |\n|-------------------------------|-----------------|------------------------------|\n| from_query_results() | from_series() | _render_cdf, _render_percentile |\n| from_normalized_results() | from_ratios() | _render_speedup |\n| from_heatmap_data() | from_matrix() | _render_rank_table |\n| from_bar_data() | from_data() | _render_bar |\n| from_comparison_data() | from_data() | _render_comparison |\n| from_query_latency_data() | from_data() | _render_histogram |\n| from_regression_data() | from_data() | _render_diverging |\n| from_time_series_points() | from_points() | _render_line |\n| from_distribution_series() | from_series() | _render_box_plot |\n| from_cost_performance_points()| from_points() | _render_scatter |\n| from_phase_data() | from_data() | _render_stacked |\n| from_metrics() | (unchanged) | _render_sparkline |\n\nAlso check: post_run_summary.py, exporters.py for any factory calls.\nAfter migration, run: `python -W error::DeprecationWarning -m pytest -q` to\nconfirm no remaining deprecation warnings from textcharts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Migrate factory function calls to new generic names", "wid": "w2"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w2"], "notes": "The renamed `from_points()` in scatter_plot uses `x_label`/`y_label` instead\nof `cost_label`/`performance_label`. Check `_render_scatter` in ascii_runtime.py\nand update kwargs accordingly.\n\nSimilarly check if any other factory parameter names changed and update.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update factory function parameter names (scatter cost\u2192x, performance\u2192y)", "wid": "w3"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1"], "notes": "textcharts added `value_formatter: Callable[[float], str] | None = None` to:\n- ASCIIStackedBar.__init__()\n- SummaryStats dataclass\n\nBenchBox does NOT set value_formatter, so it defaults to None. When None AND\nmetric_label is \"ms\" (BenchBox's default), the existing _format_time() is used\n- behavior is identical.\n\nAdditionally, ASCIISummaryBox now routes _format_time() through _format_metric()\ndispatch. When metric_label != \"ms\", total_time and best/worst values use\n_format_value() + metric_label instead of hardcoded ms\u2192s\u2192min. Since BenchBox\nalways uses metric_label=\"ms\", this is a no-op for us.\n\nVerification:\n1. Grep BenchBox for all SummaryStats and ASCIIStackedBar construction sites\n2. Confirm each uses metric_label=\"ms\" (or omits it, defaulting to \"ms\")\n3. Golden tests should already cover this from w1\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify value_formatter has no impact on existing behavior", "wid": "w4"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1"], "notes": "textcharts has NOT yet added `lower_is_better`. When it does, verify:\n- ComparisonBar.__init__() default True matches BenchBox latency semantics\n- DivergingBar.__init__() default True matches BenchBox latency semantics\n- SummaryStats dataclass default True matches BenchBox latency semantics\n- Color coding for regression/improvement annotations is unchanged\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify lower_is_better has no impact on existing behavior", "wid": "w5"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w1", "w2"], "notes": "The BenchBox compatibility shim in `benchbox/core/visualization/ascii/__init__.py`\nre-exports textcharts symbols. If textcharts added new public names (e.g.,\n`from_data`, `from_series`, `from_points`, `from_ratios`), decide whether to\nre-export them through the shim or only use them via direct textcharts import.\n\nRecommendation: re-export new factory names to keep the shim complete, but\ndon't re-export `lower_is_better` or `value_formatter` (they're constructor\nparams, not standalone symbols).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update compatibility shim re-exports if textcharts added new public names", "wid": "w6"}, {"evidence": null, "item_id": "adapt-benchbox-to-textcharts-neutral-defaults", "needs": ["w6"], "notes": "If w6 added new re-exports to the shim, ensure `ascii_api.py` picks them up.\nThe facade does `from benchbox.core.visualization.ascii import *` so it should\nbe automatic, but verify `__all__` is updated if explicit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update ascii_api.py facade if new exports were added", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "introduce AdapterRunContext just because mutable state exists", "instead": "make the lifecycle decision first.", "why": "serial run-scoped adapters may be acceptable"}, {"dont": "leave phase caches on self while claiming concurrency support", "instead": "move caches into per-run context if concurrency is supported.", "why": "that is the stale-state failure mode"}, {"dont": "use live cloud platforms for lifecycle tests", "instead": "use fake or local adapters.", "why": "optional dependencies make the guard brittle"}], "approach": "Decide lifecycle before refactoring. If serial/run-scoped is the contract,\ndocument it and add sequential reuse protection. If reusable/concurrent is a\nproduct requirement, introduce a run context deliberately and slice the state\nmigration.\n", "blocked_reason": null, "category": "Platform Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "adapter-run-context-and-reuse-contract", "id": 338, "reason": "Only required if the lifecycle decision chooses concurrent adapter reuse.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: adapter-run-context-and-reuse-contract.yaml records the decision: concurrent adapter reuse was NOT selected, so concurrency-safety work is not needed.", "summary": "Make all existing adapters concurrency-safe"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "adapter-run-context-and-reuse-contract", "id": 339, "reason": "This item targets run-scoped state, not a broad adapter-base redesign.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason states this targets run-scoped state, not a broad mixin split; explicitly out-of-scope note, not endorsed future work.", "summary": "Split every base mixin in this TODO"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Adapter instances currently behave as serial, run-scoped objects that hold\nconnection state, plan-capture flags, dry-run state, execution metadata, and\nphase result caches. That is acceptable if documented and tested. It becomes\nfragile if service-mode reuse or concurrent execution is a near-term product\ngoal.\n\nThis item decides the adapter lifecycle contract, adds a sequential reuse\nregression test for the current model, and only introduces AdapterRunContext\nif the project chooses reusable/concurrent adapters.", "id": "adapter-run-context-and-reuse-contract", "preserves": ["Do not break existing adapter subclass override points documented in adapter-refactor-map.md.", "Do not promise concurrent adapter reuse unless tests prove the base path and representative adapters are safe.", "Do not require cloud/native optional dependencies for lifecycle tests."], "prior_art": [{"concept": "benchbox/platforms/base/result_capture.py: existing phase cache consumption/reset -- target for regression coverage.", "decision": "reuse", "path": "benchbox/platforms/base/result_capture.py"}, {"concept": "docs/development/adapter-refactor-map.md: documents stable facade/subclass risk -- use this as guardrail before moving base responsibilities.", "decision": "reuse", "path": "docs/development/adapter-refactor-map.md"}, {"concept": "tests/unit/platforms/: existing local/fake adapter test patterns -- prefer over live platform dependencies.", "decision": "reuse", "path": "tests/unit/platforms/"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/scripts/explorer_pipeline/"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/"}, {"kind": "only_modify", "path_glob": "docs/development/"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Architecture hardening: document adapter lifecycle and isolate run-scoped state if needed", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms -k 'adapter_lifecycle or run_context' --tb=short", "description": "Run focused adapter lifecycle tests.", "expected": "Adapter lifecycle tests pass.", "item_id": "adapter-run-context-and-reuse-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -k 'duckdb or dataframe' --tb=short", "description": "Run a fast local smoke benchmark if adapter base code changed.", "expected": "Fast local smoke tests pass.", "item_id": "adapter-run-context-and-reuse-contract", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": [], "notes": "Completed evidence at checked SHA\n062f860969c52b2b936cc1966e77355967d5f7c7:\n - benchbox/platforms/base/adapter.py initialized adapter-scoped config and\n mutable run state together: connection, database_was_reused,\n dry_run_mode/captured_sql, plan-capture settings/counters,\n _last_throughput_test_result, and sorted-ingestion counters.\n - run_enhanced_benchmark reset plan-capture counters but did not reset\n database_was_reused, cached phase results, sorted-ingestion counters, or\n plan_first_n iteration counts before each run.\n - benchbox/platforms/base/execution.py writes _last_power_test_result and\n _last_throughput_test_result for phase construction.\n - benchbox/platforms/base/result_capture.py consumes and clears cached\n phase results during _build_execution_phases.\n - Existing tests covered individual exception resets and dry-run helpers,\n but not full sequential adapter reuse through run_benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate adapter instance state and phase cache flow", "wid": "w0"}, {"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": ["w0"], "notes": "Decision recorded in docs/reference/public-contracts.md and\ndocs/development/adapter-refactor-map.md: PlatformAdapter instances are\nserial execution objects. One instance may run multiple benchmarks\nsequentially. Concurrent run_benchmark calls on one adapter are not\nsupported.\n\nThe base adapter owns the active per-run connection and closes it in\nfinally. Cached phase results, database reuse decisions, sorted-ingestion\ncounters, plan-capture counters, and dry-run captured SQL are run-scoped and\nreset at run start. Run-config plan-capture overrides are per-run and\nrestored at run end; direct adapter attributes such as dry_run_mode and\ntable_mode remain adapter-scoped caller controls.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decision gate: serial run-scoped adapters or reusable/concurrent adapters", "wid": "w1"}, {"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": ["w1"], "notes": "Implemented the serial lifecycle path. Added\nPlatformAdapter._reset_run_scoped_state() and call it at the start of\nrun_enhanced_benchmark. The reset covers database reuse state, cached\npower/throughput phase results, sorted-ingestion metadata, plan-capture\nstats, plan_first_n iteration counters, and stale captured SQL when\nexplicit dry-run mode is active.\n\nNo AdapterRunContext was introduced because concurrent adapter reuse was\nnot selected for this product tier. run_config plan-capture overrides are\nrestored in finally so they do not leak into sequential runs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the selected adapter lifecycle path", "wid": "w2"}, {"evidence": null, "item_id": "adapter-run-context-and-reuse-contract", "needs": ["w2"], "notes": "Added docs/development/adding-new-platforms.md lifecycle guidance and an\nadapter lifecycle section in docs/development/adapter-refactor-map.md.\nAdded tests/unit/platforms/test_adapter_lifecycle.py with a local fake\nadapter; it avoids cloud/native optional dependencies and proves sequential\nreuse does not carry stale run state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add adapter authoring guidance and subclass override guardrails", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:37Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #503 of uat-charter-visibility-on-main.\nDeferred: Add a CI check that the UAT charter (or pointer) exists on the release branch\nReason deferred: Automation is a follow-up once the convention is established.", "id": "add-a-ci-check-that-the-uat-charter-or-pointer-exists-on-the", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add a CI check that the UAT charter (or pointer) exists on the release branch", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:34Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #496 of throughput-stream-count-wiring-defect.\nDeferred: Add a --streams/--concurrency CLI option to `benchbox run` itself\n(benchbox/cli/commands/run.py) so BenchmarkConfig.concurrency is\nreachable from the canonical (non-deprecated) `run` command, not just\nthe deprecated `run-official` shim. Confirmed there is currently NO way\nto set concurrency>1 through `run`'s direct-execution path\n(benchbox/cli/commands/run.py `_run_direct`) at all -- every\n`BenchmarkConfig(...)` construction in that file omits `concurrency=`,\nso it always takes the schema default of 1. docs/usage/faq.md already\ndocuments `benchbox run --platform duckdb --benchmark tpch --scale 1\n--phases throughput --streams 4` as valid usage (see the sibling\nconcurrency-docs-alignment TODO's \"do not document it as working until\nit does\" caveat) -- that example remains non-functional until this\nfollow-up lands, since run.py was out of this TODO's scope_limit.\n\nReason deferred: run.py is not in this TODO's scope_limit.only_modify\n(benchbox/platforms/base/execution.py,\nbenchbox/cli/commands/run_official.py, tests/integration/ only).\nAdding a real CLI option there is a small, low-risk one-line-per-call\nchange (a new click.option + concurrency=... in the three\nBenchmarkConfig(...) constructions in _run_dry_run/_run_direct/the\ninteractive flow) but is out of scope for this fix and needs its own\nreviewed change.\n", "id": "add-a-streams-concurrency-cli-option-to-benchbox-run-itself", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add a --streams/--concurrency CLI option to `benchbox run` itself\n(benchbox/cli/commands/run.py) so BenchmarkConfig.concurrency is\nreachable from the canonical (non-deprecated) `run` command, not just", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Build an adapter for Azure Synapse Dedicated SQL Pools to complete coverage of the major cloud data warehouses.", "id": "add-azure-synapse-adapter", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add Azure Synapse Adapter", "verifications": [], "work": [], "worktree": "platform-expansion"} @@ -12,7 +12,7 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:31Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #193 of harmonize-scale-factor-baselines.\nDeferred: Add data_generation_version field to BenchmarkResults for change tracking\nReason deferred: Currently, changing base constants silently invalidates comparisons between old and new SF=1 results. Adding a version field would allow detecting incompatible comparisons. Not needed in beta, but important for GA.\n", "id": "add-data-generation-version-field-to-benchmarkresults-for-ch", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add data_generation_version field to BenchmarkResults for change tracking", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:50Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #222 of open-table-format-gap-analysis.\nDeferred: Add Databricks Hudi support to BenchBox Databricks SQL adapter\nReason deferred: Databricks platform supports Hudi but the BenchBox SQL adapter has no Hudi code", "id": "add-databricks-hudi-support-to-benchbox-databricks-sql-adapt", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add Databricks Hudi support to BenchBox Databricks SQL adapter", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Currently only 8 platforms and 6 benchmarks have dedicated sphinx-tags.\nThe remaining 19 platforms and 12 benchmarks use category tags only\n(sql-platform, dataframe-platform, cloud-platform, custom-benchmark),\nwhich means users cannot filter documentation by specific platform or benchmark.\n\nThis task adds dedicated tags to all documented platforms and benchmarks\nso users can find all pages related to a specific technology.", "id": "add-dedicated-tags-for-platforms-and-benchmarks", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add dedicated tags for all platforms and benchmarks", "verifications": [], "work": [], "worktree": "docs-experience"} -{"anti_patterns": [{"INSTEAD": "keep them as data files", "dont": "bake the allowlists into the script source", "why": "future audits need to update them without code review of the scanner"}, {"INSTEAD": "keep this TODO focused on declared/imported parity", "dont": "extend the scope to version-bound checking", "why": "that's a different problem domain"}, {"INSTEAD": "fail the build", "dont": "make the check warning-only", "why": "warnings get ignored in CI noise"}], "approach": "Make the audit a check, not a one-off doc. The two scripts already exist\nand are proven; promoting them is mostly file moves + an allowlist\nsurface + a Makefile target + a CI job hook.\n\nThe allowlists are the load-bearing design choice. They MUST mirror the\ninventory doc's exemptions exactly, and a comment in each allowlist\nentry should reference the audit finding or transitive-via annotation\nthat justifies it.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "add-dependency-audit-ci-guard", "id": 99, "reason": "Out of scope for the initial guard; the failing CI check is sufficient signal.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly out of scope for the initial guard; failing CI check already gives sufficient signal, no auto-PR/auto-TODO automation needed.", "summary": "Auto-PR new findings (e.g. open a TODO when a new unused dep is detected)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "add-dependency-audit-ci-guard", "id": 100, "reason": "Different problem domain - covered by `review-dependency-upper-bounds-quarterly`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason explicitly states this is a different problem covered by open item review-dependency-upper-bounds-quarterly.", "summary": "Extend the scan to detect over-pinned version bounds"}], "deps": ["tighten-dependencies-phase-plan"], "description": "The `audit-imported-dependencies-for-elimination` audit was driven by\ntwo methodology scripts currently parked under `_project/scratch/`\n(gitignored):\n - `parse_deps.py` - tomllib-based dep enumeration from pyproject.toml\n - `scan_imports.py` - AST walker over benchbox/, scripts/, tests/,\n docs/conf.py, docs/_static/\n\nWithout a check, the audit's \"70 KEEP, 6 FLAG-*\" snapshot decays. New\nunused deps drift in; new top-level imports go undeclared. This TODO\npromotes those scripts into a real CI guard so the audit stays current\nfor free.\n\nSource: parent TODO `tighten-dependencies-phase-plan` (Phase 4) and\n`docs/development/dependency-inventory.md` Methodology section.", "id": "add-dependency-audit-ci-guard", "preserves": ["Allowlists explicitly mirror the documented exemptions in `dependency-inventory.md` - no silent drift", "CI runtime impact <30s for the new check (it's pure AST, no installs)", "Check fails closed: any unexpected new dep or undeclared import causes CI red", "Existing `make lint` / `make typecheck` / `make test-fast` targets unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "do_not_modify", "path_glob": "uv.lock"}, {"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": ".gitignore"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/scripts/dependency_audit/"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}], "state": "done", "title": "Promote dependency audit AST scripts to a CI guard against re-introduction of unused deps", "verifications": [{"command": "grep -q '^audit-deps:' Makefile && echo 'OK: target present' || echo 'MISSING'", "description": "make audit-deps target exists", "expected": "OK: target present", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python _project/scripts/dependency_audit/check_deps.py --help | head -1", "description": "Driver exists and is executable", "expected": "non-empty help output", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- make audit-deps", "description": "Check passes against current main", "expected": "no failures", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"import subprocess, tempfile, pathlib, shutil; src = pathlib.Path('pyproject.toml').read_text(); patched = src.replace('[project]', '[project]\\n# synthetic test', 1); print('Manual: temporarily add a fake dep and re-run make audit-deps to confirm the check fails. Expected: non-zero exit.')\"", "description": "Check fails when an unused dep is introduced (synthetic test)", "expected": "manual verification step printed", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move the two scripts from `_project/scratch/` (gitignored) to a tracked location: `_project/scripts/dependency_audit/parse_deps.py` and `scan_imports.py`. Add an `__init__.py` if needed.", "wid": "w1"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w1"], "notes": "Fail conditions:\n (a) declared package with 0 import sites AND 0 plugin/CLI use AND\n not on a transitive-only allowlist;\n (b) imported top-level module not declared and not on the\n guarded-optional allowlist.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a `check_deps.py` driver that runs the AST scan and fails on unexpected unused declarations or undeclared imports.", "wid": "w2"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w2"], "notes": "Files:\n - `plugin_cli_allowlist.yaml` - pytest-*, ruff, ty, tox, mutmut,\n codespell, sphinx-*, furo, pygments, etc.\n - `guarded_optional_allowlist.yaml` - cudf, cupy, cuml,\n sentence_transformers, torch, spacy, textblob, pysail, dask_sql,\n flightsql, influxdb3.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create two allowlist data files alongside the scripts; cross-reference against `dependency-inventory.md` so they mirror the audit's exemptions.", "wid": "w3"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a `make audit-deps` target invoking the driver. Should exit non-zero on any violation, with a human-readable diff of unexpected packages or undeclared imports.", "wid": "w4"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire `make audit-deps` into the existing CI lint job (`.github/workflows/`). Should run alongside `make lint` and `make typecheck` - pure Python AST, fast.", "wid": "w5"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the new check locally against current main; confirm it passes (i.e. agrees with the audit's KEEP/FLAG verdict). If it disagrees, reconcile the allowlists.", "wid": "w6"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update `docs/development/dependency-inventory.md` Methodology section to point at the tracked scripts and to document `make audit-deps` as the canonical re-run mechanism.", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "bake the allowlists into the script source", "instead": "keep them as data files", "why": "future audits need to update them without code review of the scanner"}, {"dont": "extend the scope to version-bound checking", "instead": "keep this TODO focused on declared/imported parity", "why": "that's a different problem domain"}, {"dont": "make the check warning-only", "instead": "fail the build", "why": "warnings get ignored in CI noise"}], "approach": "Make the audit a check, not a one-off doc. The two scripts already exist\nand are proven; promoting them is mostly file moves + an allowlist\nsurface + a Makefile target + a CI job hook.\n\nThe allowlists are the load-bearing design choice. They MUST mirror the\ninventory doc's exemptions exactly, and a comment in each allowlist\nentry should reference the audit finding or transitive-via annotation\nthat justifies it.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "add-dependency-audit-ci-guard", "id": 99, "reason": "Out of scope for the initial guard; the failing CI check is sufficient signal.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly out of scope for the initial guard; failing CI check already gives sufficient signal, no auto-PR/auto-TODO automation needed.", "summary": "Auto-PR new findings (e.g. open a TODO when a new unused dep is detected)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "add-dependency-audit-ci-guard", "id": 100, "reason": "Different problem domain - covered by `review-dependency-upper-bounds-quarterly`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason explicitly states this is a different problem covered by open item review-dependency-upper-bounds-quarterly.", "summary": "Extend the scan to detect over-pinned version bounds"}], "deps": ["tighten-dependencies-phase-plan"], "description": "The `audit-imported-dependencies-for-elimination` audit was driven by\ntwo methodology scripts currently parked under `_project/scratch/`\n(gitignored):\n - `parse_deps.py` - tomllib-based dep enumeration from pyproject.toml\n - `scan_imports.py` - AST walker over benchbox/, scripts/, tests/,\n docs/conf.py, docs/_static/\n\nWithout a check, the audit's \"70 KEEP, 6 FLAG-*\" snapshot decays. New\nunused deps drift in; new top-level imports go undeclared. This TODO\npromotes those scripts into a real CI guard so the audit stays current\nfor free.\n\nSource: parent TODO `tighten-dependencies-phase-plan` (Phase 4) and\n`docs/development/dependency-inventory.md` Methodology section.", "id": "add-dependency-audit-ci-guard", "preserves": ["Allowlists explicitly mirror the documented exemptions in `dependency-inventory.md` - no silent drift", "CI runtime impact <30s for the new check (it's pure AST, no installs)", "Check fails closed: any unexpected new dep or undeclared import causes CI red", "Existing `make lint` / `make typecheck` / `make test-fast` targets unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "do_not_modify", "path_glob": "uv.lock"}, {"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": ".gitignore"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/scripts/dependency_audit/"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}], "state": "done", "title": "Promote dependency audit AST scripts to a CI guard against re-introduction of unused deps", "verifications": [{"command": "grep -q '^audit-deps:' Makefile && echo 'OK: target present' || echo 'MISSING'", "description": "make audit-deps target exists", "expected": "OK: target present", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python _project/scripts/dependency_audit/check_deps.py --help | head -1", "description": "Driver exists and is executable", "expected": "non-empty help output", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- make audit-deps", "description": "Check passes against current main", "expected": "no failures", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"import subprocess, tempfile, pathlib, shutil; src = pathlib.Path('pyproject.toml').read_text(); patched = src.replace('[project]', '[project]\\n# synthetic test', 1); print('Manual: temporarily add a fake dep and re-run make audit-deps to confirm the check fails. Expected: non-zero exit.')\"", "description": "Check fails when an unused dep is introduced (synthetic test)", "expected": "manual verification step printed", "item_id": "add-dependency-audit-ci-guard", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move the two scripts from `_project/scratch/` (gitignored) to a tracked location: `_project/scripts/dependency_audit/parse_deps.py` and `scan_imports.py`. Add an `__init__.py` if needed.", "wid": "w1"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w1"], "notes": "Fail conditions:\n (a) declared package with 0 import sites AND 0 plugin/CLI use AND\n not on a transitive-only allowlist;\n (b) imported top-level module not declared and not on the\n guarded-optional allowlist.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a `check_deps.py` driver that runs the AST scan and fails on unexpected unused declarations or undeclared imports.", "wid": "w2"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w2"], "notes": "Files:\n - `plugin_cli_allowlist.yaml` - pytest-*, ruff, ty, tox, mutmut,\n codespell, sphinx-*, furo, pygments, etc.\n - `guarded_optional_allowlist.yaml` - cudf, cupy, cuml,\n sentence_transformers, torch, spacy, textblob, pysail, dask_sql,\n flightsql, influxdb3.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create two allowlist data files alongside the scripts; cross-reference against `dependency-inventory.md` so they mirror the audit's exemptions.", "wid": "w3"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a `make audit-deps` target invoking the driver. Should exit non-zero on any violation, with a human-readable diff of unexpected packages or undeclared imports.", "wid": "w4"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire `make audit-deps` into the existing CI lint job (`.github/workflows/`). Should run alongside `make lint` and `make typecheck` - pure Python AST, fast.", "wid": "w5"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the new check locally against current main; confirm it passes (i.e. agrees with the audit's KEEP/FLAG verdict). If it disagrees, reconcile the allowlists.", "wid": "w6"}, {"evidence": null, "item_id": "add-dependency-audit-ci-guard", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update `docs/development/dependency-inventory.md` Methodology section to point at the tracked scripts and to document `make audit-deps` as the canonical re-run mechanism.", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": "2025-11-28T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "\"Add --dry-run parameter to all example scripts in `examples/` directory that prints all planned work to an output location without generating data or running queries\"", "id": "add-dry-run-support-to-all-example-scripts", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add Dry Run Support to All Example Scripts", "verifications": [], "work": [], "worktree": "docs-experience"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:37:20Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #629 of scd2-doc-count-hygiene.\nDeferred: Add DuckDB test asserting SCD2 ops SUCCESS + legacy MERGE INTO ops SKIPPED under platform_key='duckdb', plus a no-reset sequential dim-restored-to-seed test.\nReason deferred: Genuine coverage gap for the #931 shipped skip path and the production no-reset cleanup path, but larger than this hygiene fix; deferred as its own follow-up.", "id": "add-duckdb-test-asserting-scd2-ops-success-legacy-merge-into", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add DuckDB test asserting SCD2 ops SUCCESS + legacy MERGE INTO ops SKIPPED under platform_key='duckdb', plus a no-reset sequential dim-restored-to-seed test.", "verifications": [], "work": [], "worktree": "write-primitives-scd2-followup"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: Completed \u2705\"\n- \"**Description**: Added subprocess-driven smoke coverage to verify the CLI works when invoked as an actual module, complementing the existing Click runner unit tests.\"\n- \"**Highlights**:\"\n- \"**Files Affected**:\"\n- \"**Tests**:\"", "id": "add-end-to-end-cli-testing", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Add End-to-End CLI Testing", "verifications": [], "work": [], "worktree": "unknown"} @@ -22,9 +22,9 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:20Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #461 of prompts-landing-phase5-mcp-parity.\nDeferred: Add first-class MCP platform option arguments\nReason deferred: Requires MCP tool API work outside the landing prompt batch.", "id": "add-first-class-mcp-platform-option-arguments", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add first-class MCP platform option arguments", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:33Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #194 of harmonize-scale-factor-baselines.\nDeferred: Add --force datagen auto-detection when base constants change\nReason deferred: After changing base constants, users with cached datagen directories silently use stale data. Could add a manifest hash of base constants to detect staleness. Beta users can use --force datagen manually.\n", "id": "add-force-datagen-auto-detection-when-base-constants-change", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add --force datagen auto-detection when base constants change", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Archived DONE item for: \"\u2705 Add Global Quiet Mode (`--quiet`/`-q`) to Suppress All Output\"", "id": "add-global-quiet-mode-quiet-q-to-suppress-all-output", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"\u2705 Add Global Quiet Mode (`--quiet`/`-q`) to Suppress All Output\"", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add Hypothesis to mock-heavy or integration tests", "why": "it adds no value for call-sequence verification or real I/O"}, {"INSTEAD": "(not specified)", "dont": "remove all parametrize anchors", "why": "keep 3-4 explicit cases per property as readable regression documentation"}, {"INSTEAD": "(not specified)", "dont": "replace exact-output tests with properties", "why": "only replace tests that assert invariants (idempotency, roundtrip, monotonicity, case-insensitivity)"}], "approach": "Add hypothesis as dev-only dependency. Start with w1 as proof-of-concept.\nEach work unit keeps a few explicit parametrize anchors alongside the new\n@given tests so that specific known-good cases remain as regression tests.\nUse hypothesis profiles: ci (100 examples), dev (10 examples) for speed.\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-03-27T00:00:00Z", "deferrals": [], "deps": [], "description": "Analysis of the full test suite (~16,800 test functions) identified ~20 tests\nacross 5 categories where property-based testing would genuinely add value by\nreplacing or augmenting manual parametrize lists with generated inputs.\n\nThese are pure-function tests asserting PROPERTIES (invariants that hold for\nany valid input), not specific behaviors (exact output for exact input). The\nremaining ~98% of the suite tests exact SQL strings, mock call sequences,\ngolden-file snapshots, or integration paths - none of which benefit from\nHypothesis.\n\nEstimated impact: ~400 lines of manual test code replaced or augmented,\nwith significantly better edge-case coverage on those specific paths.\n\nThis is a quality enhancement, not a gap. The suite already has 1,352+\nerror-path tests and 161 parametrized test sets covering similar ground\nmanually.\n", "id": "add-hypothesis-property-tests", "preserves": ["All existing parametrize test cases still pass (keep 3-4 anchors per property)", "CI test time increase < 15 seconds total across all new property tests", "No existing test files deleted - only augmented"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml (dev dependency only)"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/test_dataframe_capabilities.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_builder.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_config_utils.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_parameter_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpch/test_tpch_generator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/credentials/test_credential_files.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_redshift_adapter.py"}], "state": "planning", "title": "Add Hypothesis property-based tests for pure-function invariants", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/results/test_builder.py tests/unit/platforms/test_redshift_adapter.py -q", "description": "All existing tests still pass after adding Hypothesis", "expected": "All tests pass", "item_id": "add-hypothesis-property-tests", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m hypothesis -q", "description": "Hypothesis tests discover no counterexamples", "expected": "All property tests pass", "item_id": "add-hypothesis-property-tests", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m hypothesis --hypothesis-seed=0 -q --durations=10", "description": "CI time impact is acceptable", "expected": "Total < 15 seconds", "item_id": "add-hypothesis-property-tests", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add Hypothesis dependency and benchmark-ID normalization properties", "wid": "w1"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add type-mapping property tests for Redshift adapter", "wid": "w2"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add config and credential roundtrip property tests", "wid": "w3"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add scale-factor arithmetic and parameter-isolation properties", "wid": "w4"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add platform-capability consistency properties", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add Hypothesis to mock-heavy or integration tests", "instead": "(not specified)", "why": "it adds no value for call-sequence verification or real I/O"}, {"dont": "remove all parametrize anchors", "instead": "(not specified)", "why": "keep 3-4 explicit cases per property as readable regression documentation"}, {"dont": "replace exact-output tests with properties", "instead": "(not specified)", "why": "only replace tests that assert invariants (idempotency, roundtrip, monotonicity, case-insensitivity)"}], "approach": "Add hypothesis as dev-only dependency. Start with w1 as proof-of-concept.\nEach work unit keeps a few explicit parametrize anchors alongside the new\n@given tests so that specific known-good cases remain as regression tests.\nUse hypothesis profiles: ci (100 examples), dev (10 examples) for speed.\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-03-27T00:00:00Z", "deferrals": [], "deps": [], "description": "Analysis of the full test suite (~16,800 test functions) identified ~20 tests\nacross 5 categories where property-based testing would genuinely add value by\nreplacing or augmenting manual parametrize lists with generated inputs.\n\nThese are pure-function tests asserting PROPERTIES (invariants that hold for\nany valid input), not specific behaviors (exact output for exact input). The\nremaining ~98% of the suite tests exact SQL strings, mock call sequences,\ngolden-file snapshots, or integration paths - none of which benefit from\nHypothesis.\n\nEstimated impact: ~400 lines of manual test code replaced or augmented,\nwith significantly better edge-case coverage on those specific paths.\n\nThis is a quality enhancement, not a gap. The suite already has 1,352+\nerror-path tests and 161 parametrized test sets covering similar ground\nmanually.\n", "id": "add-hypothesis-property-tests", "preserves": ["All existing parametrize test cases still pass (keep 3-4 anchors per property)", "CI test time increase < 15 seconds total across all new property tests", "No existing test files deleted - only augmented"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml (dev dependency only)"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/test_dataframe_capabilities.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_builder.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_config_utils.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_parameter_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpch/test_tpch_generator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/credentials/test_credential_files.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_redshift_adapter.py"}], "state": "planning", "title": "Add Hypothesis property-based tests for pure-function invariants", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/results/test_builder.py tests/unit/platforms/test_redshift_adapter.py -q", "description": "All existing tests still pass after adding Hypothesis", "expected": "All tests pass", "item_id": "add-hypothesis-property-tests", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m hypothesis -q", "description": "Hypothesis tests discover no counterexamples", "expected": "All property tests pass", "item_id": "add-hypothesis-property-tests", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m hypothesis --hypothesis-seed=0 -q --durations=10", "description": "CI time impact is acceptable", "expected": "Total < 15 seconds", "item_id": "add-hypothesis-property-tests", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add Hypothesis dependency and benchmark-ID normalization properties", "wid": "w1"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add type-mapping property tests for Redshift adapter", "wid": "w2"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add config and credential roundtrip property tests", "wid": "w3"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add scale-factor arithmetic and parameter-isolation properties", "wid": "w4"}, {"evidence": null, "item_id": "add-hypothesis-property-tests", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add platform-capability consistency properties", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Added fast pytest-based smoke coverage for the unified runner example so CI can validate example entry points stay functional.", "id": "add-lightweight-example-smoke-tests", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Add Lightweight Example Smoke Tests", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a \"history\" or \"previous release process\" section", "why": "it is dead weight; the guide describes how to release TODAY, full stop."}, {"INSTEAD": "(not specified)", "dont": "duplicate content from automate_release.py's docstring into the guide", "why": "the docstring is for developers reading the source; the guide is for maintainers running the tool; they need different framings. Link to source, summarise behaviour, do not paste."}, {"INSTEAD": "(not specified)", "dont": "include screenshots or animated GIFs", "why": "the existing docs are text-only and screenshots rot fast; ASCII tables and code blocks are sufficient and maintainable."}, {"INSTEAD": "(not specified)", "dont": "make the guide a complete reference of every flag", "why": "that is what `--help` is for; the guide is the runbook (\"here is what to do\"), not the man page."}, {"INSTEAD": "(not specified)", "dont": "write the guide before the dependent items land", "why": "it references the new is_public_path predicate and the new --allow-public-divergence flag; writing first means rewriting twice."}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-26T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "OBSOLETED on 2026-04-26: superseded by the single-repo migration plan\n(_project/TODO/main/planning/single-repo-migration-phase-*.yaml). The\nrelease guide is still produced, but as part of the single-repo\nmigration's Phase 7 (docs cleanup), describing a much simpler flow\n(Makefile + GitHub Actions release workflow). The original version\nof this TODO described the guide for the old two-repo orchestration,\nwhich no longer exists post-migration.\n\nOriginal deps removed (the two depended-on TODOs were also obsoleted\nthe same day).\n\nOriginal description below preserved for historical context.\n\n---\n\nRelease knowledge today is fragmented across:\n- `scripts/automate_release.py` docstring (developer-facing, not skimmable)\n- `AGENTS.md` (mixed agent guidance)\n- `docs/development/development.md:99` (a stub: \"Release Preparation Workflow\")\n- `CONTRIBUTING.md:193` (\"Releases are handled by maintainers\" - points\n nowhere)\n\nA maintainer cutting their first BenchBox release has nowhere coherent to\nlook. The new guide is the canonical entrypoint: skimmable, ~1500 words,\nlinks to source for detail.\n\nWhy this depends on the other two items: the doc references the new\n`is_public_path` predicate (in the glossary) and the new `Step 3.5`\ndivergence preflight + `--allow-public-divergence` flag (in the failure\nrecovery section). Writing it before those land would produce stale docs.\n\nFormat and location confirmed by the existing precedent: `docs/operations/`\nalready houses `results-phase-2-runbook.md` (a sibling maintainer runbook),\nmyst-parser markdown is the project default.\n\nPlan file: /Users/joe/.claude/plans/make-a-detailed-and-dynamic-bentley.md\n(Phase C).", "id": "add-maintainer-release-guide", "preserves": ["Existing anchor `#release-preparation-workflow` in docs/development/development.md continues to resolve (heading kept, body replaced with redirect).", "The `AGENTS.md` content is unchanged. The release guide may LINK to it (e.g., for dependency bounds policy) but does not duplicate its content.", "The existing `docs/operations/results-phase-2-runbook.md` continues to resolve in the toctree. The new entry is added BEFORE it, not in place of it.", "The line at `CONTRIBUTING.md:193` (\"Releases are handled by maintainers\") remains - optionally extend to add a link to the new guide, but do not remove the existing copy."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "docs/development/development.md"}, {"kind": "only_modify", "path_glob": "docs/index.rst"}, {"kind": "only_modify", "path_glob": "docs/operations/release-guide.md"}], "state": "done", "title": "[OBSOLETED] Add a maintainer-facing release guide at docs/operations/release-guide.md", "verifications": [{"command": "cd docs && uv run -- python -m sphinx -W -b html . _build/html", "description": "Docs build with no warnings", "expected": "build succeeded", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 1}, {"command": "test -f docs/operations/release-guide.md && wc -w docs/operations/release-guide.md", "description": "New page exists and is non-trivial", "expected": "word count between 1000 and 3000", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -c 'operations/release-guide' docs/index.rst", "description": "Toctree includes the new page", "expected": "1", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -A2 'Release Preparation Workflow' docs/development/development.md | grep -c 'release-guide'", "description": "development.md no longer has a release-prep stub, only a redirect", "expected": "1", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "add-maintainer-release-guide", "needs": [], "notes": "Create `docs/operations/release-guide.md`. Structure:\n\n # Cutting a BenchBox release\n\n ## Overview\n One short diagram: private -> curated public tree -> squash -> tag ->\n wheel -> archive. Plain text or simple ASCII, not a graphics file.\n\n ## Before you start\n - On private main, clean, fetched.\n - ../BenchBox-public clone exists, on main, clean, fetched.\n - PyPI token configured.\n - The two-remote rule (CLAUDE.md): push to `private`, never `public`\n without intent.\n\n ## The one command\n ```bash\n uv run python scripts/automate_release.py --version 0.2.2 -y --push\n ```\n What each phase does, with one-liner descriptions and source pointers\n (e.g., \"Phase 4 builds the curated public tree - see\n benchbox/release/workflow.py::prepare_public_release\").\n\n ## What automation handles vs what you handle\n Two-column table.\n Automation: version bump, changelog, CI, content checks, size guard,\n public divergence preflight, squash, tag, build, archive.\n You: write user-facing changelog blurb, decide patch vs minor,\n communicate the release.\n\n ## Recovering from common failures\n - CI check fails -> fix on private, re-run.\n - Size check fails -> identify file, add to HOLD_BACK_PATHS or shrink.\n - Content validation fails (em-dash etc.) -> fix in source, re-run.\n - **Public divergence preflight fails** -> run `benchbox-sync pull\n --revspec ..HEAD`, commit on private, re-run. Use\n `--allow-public-divergence` ONLY if you intentionally want to\n overwrite those commits.\n\n ## When to use benchbox-sync (and when not to)\n Table:\n | Subcommand | Use when |\n | status | Sanity check before release. |\n | pull | An external PR landed on public; bring it back to private. |\n | push | Almost never - prefer cutting a patch release. |\n\n ## Glossary\n - Squash anchor - previous v* tag, where the next release squashes from.\n - Normalized timestamp - most-recent-whole-hour UTC.\n - Source fingerprint - hash of the curated tree.\n - Curated tree - the subset decided by `is_public_path` (single-source\n predicate in benchbox/release/workflow.py).\n\n ## Reference\n - Source: `scripts/automate_release.py`, `benchbox/release/workflow.py`,\n `benchbox/release/sync.py`.\n - Related: AGENTS.md (dependency bounds), CONTRIBUTING.md.\n\nStyle: myst-parser markdown (.md). Triple-backtick fences with language\nhints. H2 sentence case. Frontmatter copyright header consistent with\nother docs/operations/*.md files - check\n`docs/operations/results-phase-2-runbook.md` and copy the header style.\nLength target: ~1500 words. If it grows past 2500, split.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Draft docs/operations/release-guide.md with the canonical structure", "wid": "w1"}, {"evidence": null, "item_id": "add-maintainer-release-guide", "needs": ["w1"], "notes": "Edit `docs/index.rst`. In the Contributing section (currently around\nline 68-71), add the new entry. Final shape:\n\n .. toctree::\n :maxdepth: 1\n :caption: Contributing\n\n contributing-results.md\n operations/release-guide.md\n operations/results-phase-2-runbook.md\n\nOrder: release guide BEFORE results runbook (releases are the more\ncommon operation; runbooks for one-off corpus operations come after).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire toctree entry in docs/index.rst under Contributing", "wid": "w2"}, {"evidence": null, "item_id": "add-maintainer-release-guide", "needs": ["w1"], "notes": "At `docs/development/development.md:99`, the existing \"Release\nPreparation Workflow\" section is a stub. Replace its body with a single\nline:\n See [the release guide](../operations/release-guide.md) for the full\n maintainer workflow.\nKeep the heading so any existing inbound anchor links still resolve.\nDo not duplicate content from the new guide here.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Replace the development.md release stub with a one-line redirect", "wid": "w3"}, {"evidence": null, "item_id": "add-maintainer-release-guide", "needs": ["w2", "w3"], "notes": "Run:\n cd docs && uv run -- python -m sphinx -W -b html . _build/html\nMust complete with zero warnings. The `-W` flag converts warnings to\nerrors, so any unresolved cross-reference, broken toctree entry, or\nmyst parse error fails the build.\nThen open `docs/_build/html/operations/release-guide.html` in a\nbrowser and visually confirm:\n - Code blocks render with syntax highlighting\n - Internal links resolve (especially the dev-doc redirect from w3)\n - The toctree shows the new page in the Contributing section\nSpot-check on a phone or narrow window for the table layout\n(the \"automation vs you\" two-column table is the most likely to wrap\nawkwardly).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Build docs locally with -W and verify the new page renders", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add a \"history\" or \"previous release process\" section", "instead": "(not specified)", "why": "it is dead weight; the guide describes how to release TODAY, full stop."}, {"dont": "duplicate content from automate_release.py's docstring into the guide", "instead": "(not specified)", "why": "the docstring is for developers reading the source; the guide is for maintainers running the tool; they need different framings. Link to source, summarise behaviour, do not paste."}, {"dont": "include screenshots or animated GIFs", "instead": "(not specified)", "why": "the existing docs are text-only and screenshots rot fast; ASCII tables and code blocks are sufficient and maintainable."}, {"dont": "make the guide a complete reference of every flag", "instead": "(not specified)", "why": "that is what `--help` is for; the guide is the runbook (\"here is what to do\"), not the man page."}, {"dont": "write the guide before the dependent items land", "instead": "(not specified)", "why": "it references the new is_public_path predicate and the new --allow-public-divergence flag; writing first means rewriting twice."}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-26T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "OBSOLETED on 2026-04-26: superseded by the single-repo migration plan\n(_project/TODO/main/planning/single-repo-migration-phase-*.yaml). The\nrelease guide is still produced, but as part of the single-repo\nmigration's Phase 7 (docs cleanup), describing a much simpler flow\n(Makefile + GitHub Actions release workflow). The original version\nof this TODO described the guide for the old two-repo orchestration,\nwhich no longer exists post-migration.\n\nOriginal deps removed (the two depended-on TODOs were also obsoleted\nthe same day).\n\nOriginal description below preserved for historical context.\n\n---\n\nRelease knowledge today is fragmented across:\n- `scripts/automate_release.py` docstring (developer-facing, not skimmable)\n- `AGENTS.md` (mixed agent guidance)\n- `docs/development/development.md:99` (a stub: \"Release Preparation Workflow\")\n- `CONTRIBUTING.md:193` (\"Releases are handled by maintainers\" - points\n nowhere)\n\nA maintainer cutting their first BenchBox release has nowhere coherent to\nlook. The new guide is the canonical entrypoint: skimmable, ~1500 words,\nlinks to source for detail.\n\nWhy this depends on the other two items: the doc references the new\n`is_public_path` predicate (in the glossary) and the new `Step 3.5`\ndivergence preflight + `--allow-public-divergence` flag (in the failure\nrecovery section). Writing it before those land would produce stale docs.\n\nFormat and location confirmed by the existing precedent: `docs/operations/`\nalready houses `results-phase-2-runbook.md` (a sibling maintainer runbook),\nmyst-parser markdown is the project default.\n\nPlan file: /Users/joe/.claude/plans/make-a-detailed-and-dynamic-bentley.md\n(Phase C).", "id": "add-maintainer-release-guide", "preserves": ["Existing anchor `#release-preparation-workflow` in docs/development/development.md continues to resolve (heading kept, body replaced with redirect).", "The `AGENTS.md` content is unchanged. The release guide may LINK to it (e.g., for dependency bounds policy) but does not duplicate its content.", "The existing `docs/operations/results-phase-2-runbook.md` continues to resolve in the toctree. The new entry is added BEFORE it, not in place of it.", "The line at `CONTRIBUTING.md:193` (\"Releases are handled by maintainers\") remains - optionally extend to add a link to the new guide, but do not remove the existing copy."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "docs/development/development.md"}, {"kind": "only_modify", "path_glob": "docs/index.rst"}, {"kind": "only_modify", "path_glob": "docs/operations/release-guide.md"}], "state": "done", "title": "[OBSOLETED] Add a maintainer-facing release guide at docs/operations/release-guide.md", "verifications": [{"command": "cd docs && uv run -- python -m sphinx -W -b html . _build/html", "description": "Docs build with no warnings", "expected": "build succeeded", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 1}, {"command": "test -f docs/operations/release-guide.md && wc -w docs/operations/release-guide.md", "description": "New page exists and is non-trivial", "expected": "word count between 1000 and 3000", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -c 'operations/release-guide' docs/index.rst", "description": "Toctree includes the new page", "expected": "1", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -A2 'Release Preparation Workflow' docs/development/development.md | grep -c 'release-guide'", "description": "development.md no longer has a release-prep stub, only a redirect", "expected": "1", "item_id": "add-maintainer-release-guide", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "add-maintainer-release-guide", "needs": [], "notes": "Create `docs/operations/release-guide.md`. Structure:\n\n # Cutting a BenchBox release\n\n ## Overview\n One short diagram: private -> curated public tree -> squash -> tag ->\n wheel -> archive. Plain text or simple ASCII, not a graphics file.\n\n ## Before you start\n - On private main, clean, fetched.\n - ../BenchBox-public clone exists, on main, clean, fetched.\n - PyPI token configured.\n - The two-remote rule (CLAUDE.md): push to `private`, never `public`\n without intent.\n\n ## The one command\n ```bash\n uv run python scripts/automate_release.py --version 0.2.2 -y --push\n ```\n What each phase does, with one-liner descriptions and source pointers\n (e.g., \"Phase 4 builds the curated public tree - see\n benchbox/release/workflow.py::prepare_public_release\").\n\n ## What automation handles vs what you handle\n Two-column table.\n Automation: version bump, changelog, CI, content checks, size guard,\n public divergence preflight, squash, tag, build, archive.\n You: write user-facing changelog blurb, decide patch vs minor,\n communicate the release.\n\n ## Recovering from common failures\n - CI check fails -> fix on private, re-run.\n - Size check fails -> identify file, add to HOLD_BACK_PATHS or shrink.\n - Content validation fails (em-dash etc.) -> fix in source, re-run.\n - **Public divergence preflight fails** -> run `benchbox-sync pull\n --revspec ..HEAD`, commit on private, re-run. Use\n `--allow-public-divergence` ONLY if you intentionally want to\n overwrite those commits.\n\n ## When to use benchbox-sync (and when not to)\n Table:\n | Subcommand | Use when |\n | status | Sanity check before release. |\n | pull | An external PR landed on public; bring it back to private. |\n | push | Almost never - prefer cutting a patch release. |\n\n ## Glossary\n - Squash anchor - previous v* tag, where the next release squashes from.\n - Normalized timestamp - most-recent-whole-hour UTC.\n - Source fingerprint - hash of the curated tree.\n - Curated tree - the subset decided by `is_public_path` (single-source\n predicate in benchbox/release/workflow.py).\n\n ## Reference\n - Source: `scripts/automate_release.py`, `benchbox/release/workflow.py`,\n `benchbox/release/sync.py`.\n - Related: AGENTS.md (dependency bounds), CONTRIBUTING.md.\n\nStyle: myst-parser markdown (.md). Triple-backtick fences with language\nhints. H2 sentence case. Frontmatter copyright header consistent with\nother docs/operations/*.md files - check\n`docs/operations/results-phase-2-runbook.md` and copy the header style.\nLength target: ~1500 words. If it grows past 2500, split.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Draft docs/operations/release-guide.md with the canonical structure", "wid": "w1"}, {"evidence": null, "item_id": "add-maintainer-release-guide", "needs": ["w1"], "notes": "Edit `docs/index.rst`. In the Contributing section (currently around\nline 68-71), add the new entry. Final shape:\n\n .. toctree::\n :maxdepth: 1\n :caption: Contributing\n\n contributing-results.md\n operations/release-guide.md\n operations/results-phase-2-runbook.md\n\nOrder: release guide BEFORE results runbook (releases are the more\ncommon operation; runbooks for one-off corpus operations come after).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire toctree entry in docs/index.rst under Contributing", "wid": "w2"}, {"evidence": null, "item_id": "add-maintainer-release-guide", "needs": ["w1"], "notes": "At `docs/development/development.md:99`, the existing \"Release\nPreparation Workflow\" section is a stub. Replace its body with a single\nline:\n See [the release guide](../operations/release-guide.md) for the full\n maintainer workflow.\nKeep the heading so any existing inbound anchor links still resolve.\nDo not duplicate content from the new guide here.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Replace the development.md release stub with a one-line redirect", "wid": "w3"}, {"evidence": null, "item_id": "add-maintainer-release-guide", "needs": ["w2", "w3"], "notes": "Run:\n cd docs && uv run -- python -m sphinx -W -b html . _build/html\nMust complete with zero warnings. The `-W` flag converts warnings to\nerrors, so any unresolved cross-reference, broken toctree entry, or\nmyst parse error fails the build.\nThen open `docs/_build/html/operations/release-guide.html` in a\nbrowser and visually confirm:\n - Code blocks render with syntax highlighting\n - Internal links resolve (especially the dev-doc redirect from w3)\n - The toctree shows the new page in the Contributing section\nSpot-check on a phone or narrow window for the table layout\n(the \"automation vs you\" two-column table is the most likely to wrap\nawkwardly).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Build docs locally with -W and verify the new page renders", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:24Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #183 of fix-coverage-theater.\nDeferred: Add mutation testing to detect tests that never fail\nReason deferred: mutmut or cosmic-ray can find tests that pass regardless of code changes. Useful as a second pass after fixing known theater, but expensive to run.", "id": "add-mutation-testing-to-detect-tests-that-never-fail", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add mutation testing to detect tests that never fail", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:25Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #186 of follow-up-visualization-template-chart-integration.\nDeferred: Add non-ASCII (PNG/HTML) equivalents for new template bundles\nReason deferred: Current follow-up is scoped to ASCII chart integration and template correctness", "id": "add-non-ascii-png-html-equivalents-for-new-template-bundles", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add non-ASCII (PNG/HTML) equivalents for new template bundles", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Files Affected**:\"\n- \"**Impact**: Provides complete traceability of benchmark execution environment, enables reproducible benchmark comparisons, adds critical metadata for result interpretation, and supports benchmark result analysis across different platform configurations\"", "id": "add-platform-info-to-benchmark-results", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Add platform_info to Benchmark Results", "verifications": [], "work": [], "worktree": "unknown"} @@ -32,50 +32,50 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2025-11-29T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Implement PrestoDB (Meta's Presto fork) platform adapter to complement the existing\nTrino adapter. While Trino and PrestoDB share a common ancestry (Trino was formerly\nPrestoSQL before the 2019 fork), they have diverged significantly and require\nseparate adapters due to incompatible drivers and protocols.", "id": "add-prestodb-platform-adapter", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add PrestoDB Platform Adapter", "verifications": [], "work": [], "worktree": "platform-expansion-presto"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:35:56Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #383 of df-support-joinorder.\nDeferred: Add remaining JOB queries beyond the core 13\nReason deferred: The full JOB has 113 queries; current benchmark embeds 13 representative ones. Extending is separate scope.", "id": "add-remaining-job-queries-beyond-the-core-13", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add remaining JOB queries beyond the core 13", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2025-01-09T00:00:00Z", "completed_pr": null, "created_at": "2025-01-09T00:00:00Z", "deferrals": [], "deps": [], "description": "Add the sphinx-tags extension to enable tag-based navigation and discovery\nacross 201 documentation files. This includes infrastructure setup, a controlled\nvocabulary taxonomy of 40 tags, and systematic tagging of all pages.\n\nKey decisions made during planning:\n- Tags placed after the H1 title in all documents\n- Index files get minimal tags only (1-2 category tags)\n- All 38 Python API RST files will be fully tagged\n- Badge colors coordinated with Furo theme\n\nReference plan: /Users/joe/.claude/plans/swift-noodling-harp.md", "id": "add-sphinx-tags", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add sphinx-tags to BenchBox Documentation", "verifications": [], "work": [], "worktree": "docs-experience"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "bundle this with the SF 0.1 expansion PR", "why": "they answer different questions (depth-per-cohort vs benchmark-coverage) and reviewer attention shouldn't be split."}, {"INSTEAD": "(not specified)", "dont": "pick the benchmark by guessing", "why": "the choice has strategic implications (head-to-head with ClickBench vs broader TPC family); document the trade-off explicitly in w1."}, {"INSTEAD": "(not specified)", "dont": "use TPC-DS fractional scale factors below 1", "why": "they violate CLAUDE.md's hard rule and produce non-standard, non-comparable results."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["automate-corpus-inventory-generation", "verify-ci-validation-workflow-end-to-end"], "description": "The strategy doc differentiates BenchBox vs. ClickBench on three axes,\nthe first being **multi-benchmark coverage**: TPC-H + TPC-DS + SSB +\nmore. The current corpus has only TPC-H and SSB. A third benchmark\nfamily is needed before that differentiation claim is credible to a\nvisitor who actually clicks around the explorer.\n\nThis item picks one of TPC-DS or ClickBench, generates a 3-platform\ncohort at SF 0.01 (and SF 0.1 if budget allows), and lands it via the\nsame submit\u2192PR\u2192merge flow used for SF 0.1 expansion. The choice between\nTPC-DS and ClickBench is itself a decision (TPC-DS is more standard,\nClickBench is a direct head-to-head with the comparison site we\ndifferentiate against) - w1 makes that call explicitly.\n", "id": "add-third-benchmark-cohort-to-corpus", "preserves": ["Cohort isolation per strategy doc - never compare across benchmark families.", "Existing TPC-H and SSB cohorts untouched.", "Maintainer-run trust label on all generated bundles."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/third-cohort-selection-*.md"}, {"kind": "only_modify", "path_glob": "results-data/bundles/"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json"}], "state": "planning", "title": "Add a third benchmark cohort (TPC-DS or ClickBench) to the public corpus", "verifications": [{"command": "uv run python -c \"import json; d=json.load(open('results-data/corpus-inventory.json')); print(sorted(set(b['benchmark'] for b in d['bundles'])))\"", "description": "Inventory now lists 3+ distinct benchmark families", "expected": "list contains tpch, ssb, and one of {tpcds, clickbench}", "item_id": "add-third-benchmark-cohort-to-corpus", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run python results-data/validate_corpus.py", "description": "Each new bundle validates", "expected": "all bundles valid", "item_id": "add-third-benchmark-cohort-to-corpus", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": [], "notes": "Compare on three axes:\n 1. Platform parity: which 3+ supported platforms can run it cleanly\n at SF 0.01? (DuckDB and DataFusion run both; Polars/SQLite vary.)\n 2. Runtime + storage: TPC-DS has 99 queries; ClickBench has 43.\n Estimate wall-clock per cohort.\n 3. Strategic value: TPC-DS aligns with the \"TPC family\" framing in\n the strategy doc; ClickBench creates direct head-to-head visibility.\n\nDocument the decision and rationale in\n`_project/analysis/third-cohort-selection-.md` and link it from\nthis TODO before starting w2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Decide TPC-DS vs ClickBench based on platform support and runtime budget", "wid": "w1"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w1"], "notes": "Same approach as the SF 0.1 expansion TODO: clean tuned runs, full\nquery set, document any failures rather than shipping partial coverage.\nFor TPC-DS, do NOT use fractional scale factors below 1 (per CLAUDE.md);\nuse SF 1 if SF 0.01 isn't valid for TPC-DS.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Generate the chosen benchmark cohort on 3 platforms at SF 0.01", "wid": "w2"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w2"], "notes": "Only if the w2 cohort proves quick enough to run the higher SF in <1\nhour per platform on local hardware. Skip if it pushes the corpus over\n100MB or runs > 4 hours total. Document the decision in the cohort\nselection memo.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "(Optional) Generate the same cohort at the next scale factor", "wid": "w3"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w2"], "notes": "Single PR with all w2 (and w3 if generated) bundles. Same flow as the\nSF 0.1 expansion TODO. Validate-submission CI must pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Submit, validate via CI, merge to published-results", "wid": "w4"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w4"], "notes": "After redeploy:\n - Benchmark selector lists 3 benchmarks (or 4 if both TPC-DS and the\n previous two)\n - Browse view per benchmark renders\n - Compare view restricts to within-benchmark cohorts\n - Cross-benchmark compare attempts are hard-blocked (per strategy\n doc cohort rules)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify explorer adds the new benchmark to navigation", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "bundle this with the SF 0.1 expansion PR", "instead": "(not specified)", "why": "they answer different questions (depth-per-cohort vs benchmark-coverage) and reviewer attention shouldn't be split."}, {"dont": "pick the benchmark by guessing", "instead": "(not specified)", "why": "the choice has strategic implications (head-to-head with ClickBench vs broader TPC family); document the trade-off explicitly in w1."}, {"dont": "use TPC-DS fractional scale factors below 1", "instead": "(not specified)", "why": "they violate CLAUDE.md's hard rule and produce non-standard, non-comparable results."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["automate-corpus-inventory-generation", "verify-ci-validation-workflow-end-to-end"], "description": "The strategy doc differentiates BenchBox vs. ClickBench on three axes,\nthe first being **multi-benchmark coverage**: TPC-H + TPC-DS + SSB +\nmore. The current corpus has only TPC-H and SSB. A third benchmark\nfamily is needed before that differentiation claim is credible to a\nvisitor who actually clicks around the explorer.\n\nThis item picks one of TPC-DS or ClickBench, generates a 3-platform\ncohort at SF 0.01 (and SF 0.1 if budget allows), and lands it via the\nsame submit\u2192PR\u2192merge flow used for SF 0.1 expansion. The choice between\nTPC-DS and ClickBench is itself a decision (TPC-DS is more standard,\nClickBench is a direct head-to-head with the comparison site we\ndifferentiate against) - w1 makes that call explicitly.\n", "id": "add-third-benchmark-cohort-to-corpus", "preserves": ["Cohort isolation per strategy doc - never compare across benchmark families.", "Existing TPC-H and SSB cohorts untouched.", "Maintainer-run trust label on all generated bundles."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/third-cohort-selection-*.md"}, {"kind": "only_modify", "path_glob": "results-data/bundles/"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json"}], "state": "planning", "title": "Add a third benchmark cohort (TPC-DS or ClickBench) to the public corpus", "verifications": [{"command": "uv run python -c \"import json; d=json.load(open('results-data/corpus-inventory.json')); print(sorted(set(b['benchmark'] for b in d['bundles'])))\"", "description": "Inventory now lists 3+ distinct benchmark families", "expected": "list contains tpch, ssb, and one of {tpcds, clickbench}", "item_id": "add-third-benchmark-cohort-to-corpus", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run python results-data/validate_corpus.py", "description": "Each new bundle validates", "expected": "all bundles valid", "item_id": "add-third-benchmark-cohort-to-corpus", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": [], "notes": "Compare on three axes:\n 1. Platform parity: which 3+ supported platforms can run it cleanly\n at SF 0.01? (DuckDB and DataFusion run both; Polars/SQLite vary.)\n 2. Runtime + storage: TPC-DS has 99 queries; ClickBench has 43.\n Estimate wall-clock per cohort.\n 3. Strategic value: TPC-DS aligns with the \"TPC family\" framing in\n the strategy doc; ClickBench creates direct head-to-head visibility.\n\nDocument the decision and rationale in\n`_project/analysis/third-cohort-selection-.md` and link it from\nthis TODO before starting w2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Decide TPC-DS vs ClickBench based on platform support and runtime budget", "wid": "w1"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w1"], "notes": "Same approach as the SF 0.1 expansion TODO: clean tuned runs, full\nquery set, document any failures rather than shipping partial coverage.\nFor TPC-DS, do NOT use fractional scale factors below 1 (per CLAUDE.md);\nuse SF 1 if SF 0.01 isn't valid for TPC-DS.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Generate the chosen benchmark cohort on 3 platforms at SF 0.01", "wid": "w2"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w2"], "notes": "Only if the w2 cohort proves quick enough to run the higher SF in <1\nhour per platform on local hardware. Skip if it pushes the corpus over\n100MB or runs > 4 hours total. Document the decision in the cohort\nselection memo.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "(Optional) Generate the same cohort at the next scale factor", "wid": "w3"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w2"], "notes": "Single PR with all w2 (and w3 if generated) bundles. Same flow as the\nSF 0.1 expansion TODO. Validate-submission CI must pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Submit, validate via CI, merge to published-results", "wid": "w4"}, {"evidence": null, "item_id": "add-third-benchmark-cohort-to-corpus", "needs": ["w4"], "notes": "After redeploy:\n - Benchmark selector lists 3 benchmarks (or 4 if both TPC-DS and the\n previous two)\n - Browse view per benchmark renders\n - Compare view restricts to within-benchmark cohorts\n - Cross-benchmark compare attempts are hard-blocked (per strategy\n doc cohort rules)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify explorer adds the new benchmark to navigation", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "\"Implement Time Series Benchmark Suite (TSBS) benchmarks based on https://github.com/timescale/tsbs for time-series database performance testing, creating two distinct benchmarks: TSBS DevOps and TSBS IoT\"", "id": "add-time-series-benchmark-suite-tsbs-implementation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add Time Series Benchmark Suite (TSBS) Implementation", "verifications": [], "work": [], "worktree": "benchmark-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Implement TPC-H Skew benchmark based on the research published at\nhttps://www.tpc.org/tpctc/tpctc2011/slides_and_papers/introducing_skew_into_the_tpc_h_benchmark.pdf\nand reference implementation at https://github.com/YSU-Data-Lab/TPC-H-Skew\n\nARCHITECTURAL CONSTRAINT: TPC-H Skew is ONLY responsible for the skewed data variant.\nFor comparative analysis (uniform vs skewed), the existing TPCHBenchmark must be used\nfor uniform data generation and execution. Do NOT duplicate TPC-H functionality within\nTPC-H Skew - instead, orchestrate both benchmarks when comparison is needed.", "id": "add-tpc-h-skew-benchmark-implementation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add TPC-H Skew Benchmark Implementation", "verifications": [], "work": [], "worktree": "benchmark-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:55Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #231 of reduce-cc13-complexity-ceiling.\nDeferred: Add work units for additional refactorables discovered during w1 classification\nReason deferred: The 11 refactorable functions in w3-w13 come from the top-20 sample.\nw1 will classify the remaining 67 CC=13 functions, likely identifying\n~25-35 more refactorables. Each should be added as an individual work\nunit following the w3-w13 format (one function per unit, with file path,\npattern, and approach). w14 needs-edges must be updated to include them.\n", "id": "add-work-units-for-additional-refactorables-discovered-durin", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Add work units for additional refactorables discovered during w1 classification", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "'fix' the deferred speculative items (FK column expressions, CTE column lists)", "why": "they are guarded by 'speculative' for a reason. The audit explicitly downgraded them. Re-evaluate only when a real failing benchmark surfaces."}, {"INSTEAD": "(not specified)", "dont": "batch the 3 Critical fixes into a single PR", "why": "each touches different test files and orthogonal failure modes; landing them separately keeps blast radius small if one regresses"}, {"INSTEAD": "(not specified)", "dont": "change the PG adapter from_config signatures during w9", "why": "only extract the duplicated body. Caller compatibility comes first."}, {"INSTEAD": "(not specified)", "dont": "delete questdb._strip_fk_constraints in w4 without first running questdb integration tests", "why": "it carries inline-REFERENCES handling that the base helper does not yet have"}, {"INSTEAD": "(not specified)", "dont": "promote questdb.py:476-480 pre-create DROP failures to logger.info in w6", "why": "the audit specifically calls for `warning` (failed destructive intent), not info"}, {"INSTEAD": "(not specified)", "dont": "skip w15 (ABC conformance test)", "why": "without it, the next snowpark_connect-style divergence will land silently, and the C3 fix becomes wasted work the moment someone adds a new adapter"}, {"INSTEAD": "(not specified)", "dont": "treat `_project/TODO/` or `_project/DONE/` as product-code scope during implementation", "why": "TODO status moves, work-unit completion, and index regeneration are required by the workflow"}], "approach": "Sequence is deliberate and dependency-aware:\n- w1-w3 (Critical) ship first, individually. Each is a contract bug\n that breaks production and needs no shared infrastructure.\n- w4-w5 (shared-helper foundation) ship next. They are the leverage\n that makes w8/w11 mechanical instead of bespoke per-adapter.\n- w6 (P11 visibility cluster) and w7 (N1 throughput inflation) are\n independent of the foundation work \u2014 can ship in parallel with w4/w5.\n- w8/w9/w10 (modularity-drift extractions) ship after w4/w5.\n- w11/w22/w23/w12 (per-adapter regex fixes) ship after w5 in bounded\n adapter-family slices.\n- w13/w14 are independent one-liners.\n- w15 (ABC test) ships after w3 and gates future regressions.\n- w16/w17/w24/w25/w18/w19/w26/w20 are cleanup; ship last.\n\nReference the canonical fixes already in the repo:\n- benchbox/platforms/_spark_helpers.py for balanced-paren walker,\n SparkLikeAdapterMixin, \"Import discipline\" docstring block\n- benchbox/platforms/{spark,lakesail,velox}.py for the lambda-capture\n pattern (w10-style), the on_pre_loop hook wiring, the\n --platform-option error message rewrite (P3 canonical fix)\n- docker/velox/docker-compose.yml for the canonical tmpfs pattern\n that w20 should mirror\n\nGrep before each work unit:\n git diff 73f3a582e -- # confirm file hasn't drifted from audit baseline\n grep -n '' # confirm line numbers\nAudit lines were captured at 73f3a582e; subsequent commits may shift them.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-25T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 101, "reason": "Speculative \u2014 current TPC schemas don't use functional refs in column lists. Re-evaluate when a benchmark adds expressions inside REFERENCES col lists.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Speculative regex hardening explicitly deferred until a benchmark adds expression-based FK refs in column lists; no such benchmark exists yet.", "summary": "P1 (Consider) \u2014 base/ddl_helpers.py:28,31 inline `[^)]*` in FK column-list regex"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 102, "reason": "Speculative \u2014 bare-identifier-only CTE column lists are universal. Will be addressed when the rewriter migrates to SQLGlot AST per its own design philosophy.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly speculative; deferred to a future SQLGlot-AST migration of the rewriter that has not been scheduled.", "summary": "P1 (Consider) \u2014 questdb_rewriter.py:451-454 CTE column-list `[^)]+`"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 103, "reason": "Speculative \u2014 Fabric managed Spark runtime ships Iceberg JARs by default on supported runtime versions. Add probe only if a real failure surfaces.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly speculative: add fabric_spark Iceberg classpath probe only if a real failure surfaces. No failure reported.", "summary": "P4 (Consider) \u2014 fabric_spark Iceberg classpath probe"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 104, "reason": "Most cloud DW and embedded engines have no local docker dir; the questdb finding (w20) was the only one in the audited set.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason states cloud DW/embedded engines have no local docker dir; questdb was the only actual finding in the audited set \u2014 nothing left to audit.", "summary": "Audit duckdb/sqlite/snowflake docker dirs for P5"}], "deps": [], "description": "A 13-pattern defect audit across 30 platform adapters (excluding the\nalready-fixed spark/lakesail/velox triple), shared helpers in\nbenchbox/platforms/base/, cloud_shared.py, presto_trino_utils.py,\nquestdb_rewriter.py, datafusion_*.py rewriters, benchbox/sql_compat/,\nand all docker/*/ surfaced 33 actionable findings: 3 Critical, 22 Required,\nand 8 Nit/Consider. Those findings span the original 13-pattern catalogue\nplus 4 newly-named patterns (P14 stale-flag-coexistence,\nP15 reference-before-assignment, N1 COUNT(*) throughput inflation,\nN2 ABC contract divergence).\n\nThe audit ran on commit 73f3a582e (post velox/lakesail/spark fixes for\nW8/W9/W10). The canonical fixes for patterns 1-13 already exist in\nbenchbox/platforms/_spark_helpers.py (`_strip_balanced_paren_constraints`,\nthe \"import discipline\" docstring block, the SparkLikeAdapterMixin)\nand benchbox/platforms/{spark,lakesail,velox}.py \u2014 implementers should\nport those patterns rather than reinvent.\n\nWork units are sequenced by blast radius and dependency order: Critical\ncontract bugs first, then shared-helper drift (highest leverage), then\ndestructive-op visibility, then the silent-throughput bug, then\nmodularity-drift extractions, then the long tail of P1/P2 regex bugs\nin individual adapters.\n\nAudit report and pattern catalogue are in conversation context (not\ncommitted). This TODO is intended to be self-contained enough to start\nimplementation, but every work unit must still confirm the cited snippet\nagainst the current file before editing because line numbers were captured\nat commit 73f3a582e.", "id": "address-cross-adapter-13-pattern-audit-findings", "preserves": ["All currently-passing fast tests continue to pass: `uv run -- python -m pytest -m fast -q`", "Existing CLI invocations using legacy `---` flags continue to work where w17 keeps a shim (deprecation must not be a hard break)", "PlatformAdapter ABC method signatures stay stable; w15's conformance test enforces them going forward", "PostgreSQLAdapter.create_schema(self, benchmark, connection) -> float signature stays canonical for the entire PG family", "Spark/lakesail/velox triple continues to pass tests/unit/platforms/test_{spark,lakesail,velox}_*.py \u2014 they are the canonical reference for patterns 1-13", "_strip_balanced_paren_constraints behavior in _spark_helpers.py is unchanged when lifted to base/ \u2014 re-export must keep import paths stable"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/velox.py"}, {"kind": "do_not_modify", "path_glob": "docker/velox/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "docker/questdb/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Address cross-adapter 13-pattern audit findings (3 Critical, 22 Required, 8 Nit/Consider)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests still pass after every work unit", "expected": "all tests pass", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- ruff check benchbox/platforms/ tests/unit/platforms/", "description": "Lint clean", "expected": "All checks passed", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- ty check benchbox/platforms/", "description": "Typecheck clean", "expected": "All checks passed", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_abc_conformance.py -v", "description": "ABC conformance test exists and passes after w15", "expected": "all concrete adapters conform to PlatformAdapter ABC", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_timescaledb_*.py -v -k rollback", "description": "C1 \u2014 timescaledb rolls back on per-statement failure", "expected": "rollback test passes", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_pg_mooncake_*.py -v", "description": "C2 \u2014 pg_mooncake create_schema delegates correctly through parent", "expected": "all tests pass; new test exercises full create_schema flow", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -c 'import inspect; from benchbox.platforms.snowpark_connect import SnowparkConnectAdapter; from benchbox.platforms.base.adapter import PlatformAdapter; print(inspect.signature(SnowparkConnectAdapter.create_schema))'", "description": "C3 \u2014 snowpark_connect ABC conformance", "expected": "(self, benchmark, connection) (or compatible)", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 7}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'log_level or destructive'", "description": "P11 cluster \u2014 destructive ops visible at INFO", "expected": "all destructive ops emit INFO-level log entries", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 8}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_cloud_*.py -v -k throughput", "description": "N1 \u2014 clickhouse_cloud throughput regression test", "expected": "total_rows_loaded matches sum of per-file row counts only", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 9}], "work": [{"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "timescaledb.py:378-393 \u2014 psycopg2 keeps the txn open after a failed\ncursor.execute. Add `connection.rollback()` in the except branch\n(mirror postgresql.py:440). Optional improvement: adopt the parent's\nFK-strip retry path instead of replacing it.\nTrigger: any benchmark whose first DDL stmt fails (extension-dependent\ntype, missing role) \u2014 every subsequent stmt then raises\nInFailedSqlTransaction and warning-logs, so the whole schema silently\nfails.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "C1 \u2014 Add per-statement rollback to timescaledb create_schema DDL loop", "wid": "w1"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w7"], "notes": "clickhouse_cloud.py:579-712 (S3, 133 lines) vs :714-851 (GCS, 137\nlines) are 95% duplicate. Extract\n`_load_data_via_object_storage(provider_kind, ...)` that takes the\nstorage SDK and the `INSERT FROM s3()`/`INSERT FROM gcs()` table\nfunction as parameters. Sequenced after w7 so the COUNT(*) fix\npropagates correctly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Consolidate clickhouse_cloud S3 + GCS load paths into one helper", "wid": "w10"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w5"], "notes": "First balanced-walker migration slice. Sites and triggers:\n- athena.py:685 `\\\\s+WITH\\\\s*\\\\([^)]*\\\\)` \u2192 breaks on\n `WITH (partitioning = ARRAY['bucket(ws_sold_date_sk, 16)'])`\n- presto.py:852 + trino.py:852 (`WITH ([^)]*)`) \u2192 breaks on\n `WITH (partitioning = ARRAY[date_trunc('day', ts)])`\n\nAdd unit tests covering each trigger. Keep adapter-specific PK/FK and\nClickHouse/Fabric cases in w22/w23 so failures isolate cleanly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P1 cluster A \u2014 replace WITH/DDL option regex strippers with the balanced walker from w5", "wid": "w11"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w11"], "notes": "Three asymmetric drift items between presto.py and trino.py:\n1. presto.py:847-854 strips NOT NULL for memory catalog; trino.py\n does NOT. Trino memory catalog also rejects NOT NULL.\n2. presto.py:633-635 uses bare `raise`; trino.py:640-647 calls\n `_build_friendly_connection_error`. Lift the helper to\n presto_trino_utils.py and use in both.\n3. trino.py:681-696 _load_table_data sends unqualified\n `INSERT INTO orders`; presto.py:669-697 prepends `catalog.schema.`.\n Promote to a shared helper that always qualifies.\n\nStarburst inherits via TrinoAdapter \u2014 no separate fix needed there.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Restore Presto/Trino symmetry: NOT NULL stripping, friendly connect error, qualified _load_table_data", "wid": "w12"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "motherduck.py:56,116,135-138 \u2014 module docstring at :56 advertises\n`--platform-option token=`, but `add_cli_arguments` registers\n`--motherduck-token`/`--motherduck-database` and `from_config` reads\n`motherduck_token`/`motherduck_database` keys (not `token`/`database`\nfrom options). Either (a) make from_config read both option keys\nAND the legacy ones, or (b) align the docstring with the actual flag.\nPrefer (a) \u2014 it's the migration path other adapters took.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P3 \u2014 Fix motherduck `--platform-option token=` advertised in docstring but not wired", "wid": "w13"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "snowflake.py:441-453 \u2014 `database` is referenced in the except branch\nbut bound only after `_create_admin_connection(...)`. If\nconnection-build raises, the handler raises NameError and masks the\noriginal. Move\n`database = connection_config.get(\"database\", self.database)` to the\ntop of the function before any callable that can raise.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P15 \u2014 Fix snowflake.drop_database reference-before-assignment in exception handler", "wid": "w14"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w3"], "notes": "Defensive \u2014 prevents future N2 (ABC contract divergence). New test\n`tests/unit/platforms/test_abc_conformance.py` walks every concrete\nsubclass of PlatformAdapter, asserts that\n{create_schema, load_data, execute_query, drop_database} match the\nABC signature using `inspect.signature`. Snowpark Connect (after w3),\npg_mooncake (after w2) should pass; failures mean someone diverged\nagain.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ABC-conformance test that introspects every concrete adapter's method signatures", "wid": "w15"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w12", "w21"], "notes": "Smaller correctness/visibility items batched as one PR:\n- postgresql.py:444-451 \u2014 FK-stripped retry catches the second\n exception, warning-logs, and proceeds. Promote to raise (or push\n onto critical_failures list like SingleStore) when the second\n attempt also fails on a CREATE TABLE. (P9 Required.)\n- presto.py:633-635 \u2014 verify the bare raise was replaced by w12's\n shared `_build_friendly_connection_error`; do not duplicate the fix here.\n- cedardb.py:243-247 \u2014 multi-table DROP loop emits no per-table log\n lines until final summary.\n- doris.py:643-665 \u2014 wrap bare `raise` to include host/user context.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P11/P9 cluster \u2014 postgresql FK-strip silent retry + presto bare raise + cedardb per-table drop logs", "wid": "w16"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w13"], "notes": "PG-family adapters wire BOTH `register_option_specs` AND\n`add_cli_arguments` for the same fields, with undefined precedence:\npostgresql.py:55-118, pg_duckdb.py:64-122, timescaledb.py:76-107,\ncedardb.py:56-70.\n\nPer adapter: either (a) make add_cli_arguments a thin shim that\nenumerates option-spec keys and registers them as legacy flags\npointing at the same dest, or (b) delete the duplicates after a\ndeprecation cycle. Preserve existing legacy invocations.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P14 cluster A \u2014 normalize PG-family legacy flag shims against option specs", "wid": "w17"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w4"], "notes": "Following the canonical _spark_helpers.py pattern, add the same\ndocstring paragraph to:\n- benchbox/platforms/cloud_shared.py:1-9\n- benchbox/platforms/presto_trino_utils.py:1\n- benchbox/platforms/questdb_rewriter.py:1-110\n- benchbox/platforms/datafusion_query_transformer.py\n- benchbox/platforms/datafusion_write_transformer.py\n\nOne-paragraph note: \"Adapters import these functions by name; tests\nmust monkeypatch on the adapter module\n(e.g. benchbox.platforms.snowflake.validate_session_cache_control),\nnot on this module, or the patch silently no-ops.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P7 \u2014 Add import-by-name discipline docstring to remaining shared helpers", "wid": "w18"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w23", "w25"], "notes": "Small Firebolt/ClickHouse/QuestDB cleanup slice:\n- firebolt.py:1056 \u2014 re-enable inline NOT NULL stripper (or document\n the deliberate keep)\n- firebolt.py:1275-1315 \u2014 refactor validate_session_cache_control to\n delegate to cloud_shared (parameterise query string)\n- clickhouse_cloud.py:75,79 \u2014 clean up docstring that mixes\n `--clickhouse-cloud-*` and `--platform-option` forms\n- questdb.py \u2014 set `_max_identifier_length = 127` on the adapter;\n move the override out of the mixin docstring\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Nit/Consider tail A \u2014 firebolt/clickhouse_cloud/questdb docs and helper cleanup", "wid": "w19"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "pg_mooncake.py:193-207 \u2014 current signature\n`create_schema(self, connection, schema_ddl, **kwargs)` does not match\nPostgreSQLAdapter.create_schema(self, benchmark, connection) -> float\n(verified at postgresql.py:422 and base/adapter.py:254).\n\nRefactor to override `_transform_create_statement` (the SingleStore\npattern) so the columnstore USING clause is injected per-statement\ninside the parent's normal flow. Delete the broken create_schema\noverride. Update tests/unit/platforms/test_pg_mooncake_*.py \u2014 current\ntests acknowledge they \"can't call create_schema directly without a\nreal connection,\" which is what masked the regression.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "C2 \u2014 Fix pg_mooncake.create_schema signature mismatch with parent", "wid": "w2"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "docker/questdb/docker-compose.yml:23-47 \u2014 comment claims \"Data is\nephemeral - no named volume,\" but with no `tmpfs:` block and\n`restart: unless-stopped`, state actually sticks to the container\ninstance across `docker compose restart`.\nEither (a) add `tmpfs: - ` to make it RAM-backed,\nor (b) rewrite the comment to \"no host-mounted volume; container\nrecreate clears state.\" Mirror docker/velox/docker-compose.yml\ncanonical pattern.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P5 \u2014 Make docker/questdb truly ephemeral (or fix the misleading comment)", "wid": "w20"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w6"], "notes": "Mechanical, low-risk shared/local cleanup visibility changes. Promote\ndestructive operation logging at:\n- base/connection_lifecycle.py:255-275 (unlink/rmtree/drop_database)\n- datafusion.py:1450-1459 (shutil.rmtree(working_dir))\n- polars_platform.py:657-666 (mirror of datafusion)\n- presto_trino_utils.py:198-205 (add INFO between DROP and recreate\n in execute_schema_statements \"already exists\" handler)\n- base/psycopg2_mixin.py:67-85 (warning, not debug, on\n _get_existing_tables exception)\n\nSingle PR, no behavior change beyond log output. QuestDB failed\npre-create DROP attempts must stay warning-level per w6.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P11 cluster B \u2014 make shared-helper and local-dir destructive ops visible", "wid": "w21"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w11"], "notes": "Second balanced-walker migration slice. Sites and triggers:\n- duckdb.py:733-739 (table-level + inline FK) \u2192 breaks on\n `FOREIGN KEY (...) REFERENCES date_dim(d_date_sk) ON DELETE CASCADE`\n- firebolt.py:1055,1059-1064 (PK + FK) \u2192 breaks on composite PK with\n expression members `PRIMARY KEY (\"a\", coalesce(b,0))`\n\nAdd unit tests covering each trigger.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P1 cluster B \u2014 replace PK/FK regex strippers with the balanced walker from w5", "wid": "w22"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w22"], "notes": "Third balanced-walker migration slice. Sites and triggers:\n- clickhouse/workload.py:65-70 `Nullable\\\\([^)]+\\\\)\\\\s+NOT\\\\s+NULL`\n \u2192 silently breaks decimal-bearing TPC-DS tables for ALL clickhouse\n adapters (`Nullable(Decimal(38,0)) NOT NULL`)\n- clickhouse/workload.py:116 inline PK regex on parametrized types\n- fabric_warehouse.py:1218-1226 `_optimize_table_definition` uses\n str.replace after re.search \u2192 munges `CREATE TABLE IF NOT EXISTS orders`\n\nAdd unit tests covering each trigger.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P1 cluster C \u2014 fix ClickHouse nullable and Fabric table-definition regex edge cases", "wid": "w23"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w17"], "notes": "Follow the w17 shim/deprecation pattern for:\n- singlestore.py:137-175\n- snowpark_connect.py:503-538\n\nSnowpark Connect is urgent because its unprefixed `--user`/`--password`/\n`--account` collide with other adapters' flags in shared parsers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P14 cluster B \u2014 normalize SingleStore and Snowpark Connect legacy flag shims", "wid": "w24"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w24"], "notes": "Inverse case: sqlite.py has only legacy flags and no option-spec\nregistration.\n\nEither register sqlite under\nbenchbox/platforms/__init__.py or document that sqlite intentionally\nbypasses the registry (and update the error message at sqlite.py:124-127\naccordingly).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P14 cluster C \u2014 settle sqlite option-spec registration policy", "wid": "w25"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w19"], "notes": "Small SingleStore/Doris/Fabric cleanup slice:\n- singlestore.py:713-798 \u2014 extract `_extract_create_table_name(stmt)`\n helper; replace 4 copies of the regex\n- doris.py:579-586 \u2014 delegate `_validate_identifier` to\n benchbox.utils.sql_identifier.is_valid_sql_identifier (with\n max_length=128)\n- fabric_lakehouse.py:230,266 \u2014 pick a single error model\n (raise everywhere, or return-as-dict everywhere); align with\n sibling fabric_warehouse.py\n- fabric_spark_adapter.py:730 \u2014 wrap CLI args in a parser group for\n consistency with sibling Fabric adapters\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Nit/Consider tail B \u2014 singlestore/doris/fabric cleanup", "wid": "w26"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "snowpark_connect.py:291,314,385 vs spark.py:590,632,663.\ncreate_schema(self, schema_name=None) \u2192 must accept (benchmark, connection).\nload_data(self, tables, source_dir, file_format=...) \u2192 must accept\n(benchmark, connection, data_dir).\nexecute_query(self, query, **kwargs) \u2192 must accept (connection, query,\nquery_id, ...) and return the standard dict\n{query_id, stream_id, status, execution_time_seconds, ...}.\n\nIf Snowpark Session replaces the \"connection\" arg, accept and ignore\nit positionally. The orchestrator's standard call site will TypeError\non the current code.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "C3 \u2014 Conform snowpark_connect.{create_schema,load_data,execute_query} to PlatformAdapter ABC", "wid": "w3"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w1", "w2"], "notes": "base/ddl_helpers.py:25-69 \u2014 `_FK_CLAUSE_RE` requires the table-level\n`FOREIGN KEY (...)` header. Add a second regex pass for inline\n`\\s+REFERENCES\\s+...` (mirroring questdb.py:670-675), normalize\ntrailing commas, then DELETE questdb._strip_fk_constraints since the\nbase helper now covers both forms.\n\nAlso add the \"Import discipline\" paragraph from\n_spark_helpers.py:7-13 to the module docstring (callers:\npostgresql:19, singlestore:27, doris:35, questdb:28,\ndatabend/adapter:53, starrocks/workload:12 \u2014 all import by name).\n\nVerify against TPC-DS-style inline FK form\n(`cs_sold_date_sk INTEGER REFERENCES date_dim(d_date_sk)`).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Shared helper P2 + P7 \u2014 base/ddl_helpers.py inline-REFERENCES + import-discipline docstring", "wid": "w4"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w4"], "notes": "`_strip_balanced_paren_constraints` in _spark_helpers.py is the\ncanonical fix for Pattern 1 (greedy `[^)]*` over balanced parens).\nMove it into benchbox/platforms/base/ddl_helpers.py (or a new\nbenchbox/platforms/base/sql_paren_walker.py) and re-export from\n_spark_helpers.py for backward compat. This unblocks w11 \u2014 every\nadapter rewriter can then call the same balanced walker instead of\nrolling its own broken `\\([^)]*\\)` regex.\n\nAdd unit tests with at least: `CHECK ((a>0) AND (b>0))`,\n`DECIMAL(10,2)`, `PARTITION BY (date_trunc('day',ts))`,\n`DEFAULT (now())`, `WITH (partitioning = ARRAY['bucket(x, 16)'])`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Lift balanced-paren constraint walker out of _spark_helpers.py into shared module", "wid": "w5"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "Mechanical, low-risk adapter-local visibility changes. Promote\n`logger.debug` \u2192 `logger.info` (or log_operation_complete) at:\n- athena.py:1054 (DROP staging table)\n- athena.py:1070 (S3 staging delete)\n- redshift.py:1156-1158 (DROP TABLE)\n- azure_synapse.py:452-462 (DROP TABLE in schema recreate)\n- fabric_warehouse.py:741-742 (DROP TABLE)\n- presto.py:559 (DROP TABLE)\n- questdb.py:476-480 (pre-create DROP failures \u2192 warning, not info)\n\nKeep this PR limited to adapter-local logging sites. Shared helper\nand local-working-dir destructive operations are covered by w21.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P11 cluster A \u2014 promote adapter destructive-op debug logs to info/warning", "wid": "w6"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "clickhouse_cloud.py:676-682 (S3) and :816-822 (GCS) \u2014 loop assigns\n`total_rows_loaded = SELECT COUNT(*) FROM table` per iteration; the\nsource comment acknowledges the bug. On rerun-into-non-empty-table,\nthe count includes pre-existing rows and silently inflates throughput\nin published BenchmarkResults.\n\nEither TRUNCATE before the loop, or take COUNT(*) once after the\nloop and assign once. Add a regression test that loads two files\ninto a non-empty table and asserts total_rows_loaded == sum of new\nfile row counts only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "N1 \u2014 Fix ClickHouse Cloud COUNT(*) throughput inflation in S3/GCS load loops", "wid": "w7"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w5"], "notes": "cudf.py:605-636 \u2245 polars_platform.py:618-645 \u2245 datafusion.py:1400-1435\ntriplicate `apply_constraint_configuration` and\n`apply_platform_optimizations`; only the platform-name string differs.\n\nAdd benchbox/platforms/base/no_constraint_enforcement_mixin.py with\nthe parameterised methods (read self.platform_name). Update the three\nadapters to inherit. Update or delete the three duplicate test files.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Extract NoConstraintEnforcementMixin (cudf, datafusion, polars_platform)", "wid": "w8"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "timescaledb.py:110-159 \u2245 postgresql.py:154-201 \u2245 pg_duckdb.py:124-181 \u2245\npg_mooncake.py:67-116 \u2245 cedardb.py:73-119 \u2014 ~30 identical lines for\nhost/port/username/password/schema/sslmode + database-naming.\n\nAdd a class method `_build_postgres_connection_kwargs(cls, config)`\non PostgreSQLAdapter that the four subclasses call. Concrete drift\nrisk: pg_mooncake/cedardb already lack `enable_timescale` and\n`statement_timeout` while sharing the same parent __init__.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Extract _build_postgres_connection_kwargs helper for PG family from_config", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "'fix' the deferred speculative items (FK column expressions, CTE column lists)", "instead": "(not specified)", "why": "they are guarded by 'speculative' for a reason. The audit explicitly downgraded them. Re-evaluate only when a real failing benchmark surfaces."}, {"dont": "batch the 3 Critical fixes into a single PR", "instead": "(not specified)", "why": "each touches different test files and orthogonal failure modes; landing them separately keeps blast radius small if one regresses"}, {"dont": "change the PG adapter from_config signatures during w9", "instead": "(not specified)", "why": "only extract the duplicated body. Caller compatibility comes first."}, {"dont": "delete questdb._strip_fk_constraints in w4 without first running questdb integration tests", "instead": "(not specified)", "why": "it carries inline-REFERENCES handling that the base helper does not yet have"}, {"dont": "promote questdb.py:476-480 pre-create DROP failures to logger.info in w6", "instead": "(not specified)", "why": "the audit specifically calls for `warning` (failed destructive intent), not info"}, {"dont": "skip w15 (ABC conformance test)", "instead": "(not specified)", "why": "without it, the next snowpark_connect-style divergence will land silently, and the C3 fix becomes wasted work the moment someone adds a new adapter"}, {"dont": "treat `_project/TODO/` or `_project/DONE/` as product-code scope during implementation", "instead": "(not specified)", "why": "TODO status moves, work-unit completion, and index regeneration are required by the workflow"}], "approach": "Sequence is deliberate and dependency-aware:\n- w1-w3 (Critical) ship first, individually. Each is a contract bug\n that breaks production and needs no shared infrastructure.\n- w4-w5 (shared-helper foundation) ship next. They are the leverage\n that makes w8/w11 mechanical instead of bespoke per-adapter.\n- w6 (P11 visibility cluster) and w7 (N1 throughput inflation) are\n independent of the foundation work \u2014 can ship in parallel with w4/w5.\n- w8/w9/w10 (modularity-drift extractions) ship after w4/w5.\n- w11/w22/w23/w12 (per-adapter regex fixes) ship after w5 in bounded\n adapter-family slices.\n- w13/w14 are independent one-liners.\n- w15 (ABC test) ships after w3 and gates future regressions.\n- w16/w17/w24/w25/w18/w19/w26/w20 are cleanup; ship last.\n\nReference the canonical fixes already in the repo:\n- benchbox/platforms/_spark_helpers.py for balanced-paren walker,\n SparkLikeAdapterMixin, \"Import discipline\" docstring block\n- benchbox/platforms/{spark,lakesail,velox}.py for the lambda-capture\n pattern (w10-style), the on_pre_loop hook wiring, the\n --platform-option error message rewrite (P3 canonical fix)\n- docker/velox/docker-compose.yml for the canonical tmpfs pattern\n that w20 should mirror\n\nGrep before each work unit:\n git diff 73f3a582e -- # confirm file hasn't drifted from audit baseline\n grep -n '' # confirm line numbers\nAudit lines were captured at 73f3a582e; subsequent commits may shift them.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-25T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 101, "reason": "Speculative \u2014 current TPC schemas don't use functional refs in column lists. Re-evaluate when a benchmark adds expressions inside REFERENCES col lists.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Speculative regex hardening explicitly deferred until a benchmark adds expression-based FK refs in column lists; no such benchmark exists yet.", "summary": "P1 (Consider) \u2014 base/ddl_helpers.py:28,31 inline `[^)]*` in FK column-list regex"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 102, "reason": "Speculative \u2014 bare-identifier-only CTE column lists are universal. Will be addressed when the rewriter migrates to SQLGlot AST per its own design philosophy.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly speculative; deferred to a future SQLGlot-AST migration of the rewriter that has not been scheduled.", "summary": "P1 (Consider) \u2014 questdb_rewriter.py:451-454 CTE column-list `[^)]+`"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 103, "reason": "Speculative \u2014 Fabric managed Spark runtime ships Iceberg JARs by default on supported runtime versions. Add probe only if a real failure surfaces.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly speculative: add fabric_spark Iceberg classpath probe only if a real failure surfaces. No failure reported.", "summary": "P4 (Consider) \u2014 fabric_spark Iceberg classpath probe"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "address-cross-adapter-13-pattern-audit-findings", "id": 104, "reason": "Most cloud DW and embedded engines have no local docker dir; the questdb finding (w20) was the only one in the audited set.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason states cloud DW/embedded engines have no local docker dir; questdb was the only actual finding in the audited set \u2014 nothing left to audit.", "summary": "Audit duckdb/sqlite/snowflake docker dirs for P5"}], "deps": [], "description": "A 13-pattern defect audit across 30 platform adapters (excluding the\nalready-fixed spark/lakesail/velox triple), shared helpers in\nbenchbox/platforms/base/, cloud_shared.py, presto_trino_utils.py,\nquestdb_rewriter.py, datafusion_*.py rewriters, benchbox/sql_compat/,\nand all docker/*/ surfaced 33 actionable findings: 3 Critical, 22 Required,\nand 8 Nit/Consider. Those findings span the original 13-pattern catalogue\nplus 4 newly-named patterns (P14 stale-flag-coexistence,\nP15 reference-before-assignment, N1 COUNT(*) throughput inflation,\nN2 ABC contract divergence).\n\nThe audit ran on commit 73f3a582e (post velox/lakesail/spark fixes for\nW8/W9/W10). The canonical fixes for patterns 1-13 already exist in\nbenchbox/platforms/_spark_helpers.py (`_strip_balanced_paren_constraints`,\nthe \"import discipline\" docstring block, the SparkLikeAdapterMixin)\nand benchbox/platforms/{spark,lakesail,velox}.py \u2014 implementers should\nport those patterns rather than reinvent.\n\nWork units are sequenced by blast radius and dependency order: Critical\ncontract bugs first, then shared-helper drift (highest leverage), then\ndestructive-op visibility, then the silent-throughput bug, then\nmodularity-drift extractions, then the long tail of P1/P2 regex bugs\nin individual adapters.\n\nAudit report and pattern catalogue are in conversation context (not\ncommitted). This TODO is intended to be self-contained enough to start\nimplementation, but every work unit must still confirm the cited snippet\nagainst the current file before editing because line numbers were captured\nat commit 73f3a582e.", "id": "address-cross-adapter-13-pattern-audit-findings", "preserves": ["All currently-passing fast tests continue to pass: `uv run -- python -m pytest -m fast -q`", "Existing CLI invocations using legacy `---` flags continue to work where w17 keeps a shim (deprecation must not be a hard break)", "PlatformAdapter ABC method signatures stay stable; w15's conformance test enforces them going forward", "PostgreSQLAdapter.create_schema(self, benchmark, connection) -> float signature stays canonical for the entire PG family", "Spark/lakesail/velox triple continues to pass tests/unit/platforms/test_{spark,lakesail,velox}_*.py \u2014 they are the canonical reference for patterns 1-13", "_strip_balanced_paren_constraints behavior in _spark_helpers.py is unchanged when lifted to base/ \u2014 re-export must keep import paths stable"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/velox.py"}, {"kind": "do_not_modify", "path_glob": "docker/velox/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "docker/questdb/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Address cross-adapter 13-pattern audit findings (3 Critical, 22 Required, 8 Nit/Consider)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests still pass after every work unit", "expected": "all tests pass", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- ruff check benchbox/platforms/ tests/unit/platforms/", "description": "Lint clean", "expected": "All checks passed", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- ty check benchbox/platforms/", "description": "Typecheck clean", "expected": "All checks passed", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_abc_conformance.py -v", "description": "ABC conformance test exists and passes after w15", "expected": "all concrete adapters conform to PlatformAdapter ABC", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_timescaledb_*.py -v -k rollback", "description": "C1 \u2014 timescaledb rolls back on per-statement failure", "expected": "rollback test passes", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_pg_mooncake_*.py -v", "description": "C2 \u2014 pg_mooncake create_schema delegates correctly through parent", "expected": "all tests pass; new test exercises full create_schema flow", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -c 'import inspect; from benchbox.platforms.snowpark_connect import SnowparkConnectAdapter; from benchbox.platforms.base.adapter import PlatformAdapter; print(inspect.signature(SnowparkConnectAdapter.create_schema))'", "description": "C3 \u2014 snowpark_connect ABC conformance", "expected": "(self, benchmark, connection) (or compatible)", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 7}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'log_level or destructive'", "description": "P11 cluster \u2014 destructive ops visible at INFO", "expected": "all destructive ops emit INFO-level log entries", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 8}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_cloud_*.py -v -k throughput", "description": "N1 \u2014 clickhouse_cloud throughput regression test", "expected": "total_rows_loaded matches sum of per-file row counts only", "item_id": "address-cross-adapter-13-pattern-audit-findings", "last_result": null, "last_run": null, "seq": 9}], "work": [{"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "timescaledb.py:378-393 \u2014 psycopg2 keeps the txn open after a failed\ncursor.execute. Add `connection.rollback()` in the except branch\n(mirror postgresql.py:440). Optional improvement: adopt the parent's\nFK-strip retry path instead of replacing it.\nTrigger: any benchmark whose first DDL stmt fails (extension-dependent\ntype, missing role) \u2014 every subsequent stmt then raises\nInFailedSqlTransaction and warning-logs, so the whole schema silently\nfails.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "C1 \u2014 Add per-statement rollback to timescaledb create_schema DDL loop", "wid": "w1"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w7"], "notes": "clickhouse_cloud.py:579-712 (S3, 133 lines) vs :714-851 (GCS, 137\nlines) are 95% duplicate. Extract\n`_load_data_via_object_storage(provider_kind, ...)` that takes the\nstorage SDK and the `INSERT FROM s3()`/`INSERT FROM gcs()` table\nfunction as parameters. Sequenced after w7 so the COUNT(*) fix\npropagates correctly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Consolidate clickhouse_cloud S3 + GCS load paths into one helper", "wid": "w10"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w5"], "notes": "First balanced-walker migration slice. Sites and triggers:\n- athena.py:685 `\\\\s+WITH\\\\s*\\\\([^)]*\\\\)` \u2192 breaks on\n `WITH (partitioning = ARRAY['bucket(ws_sold_date_sk, 16)'])`\n- presto.py:852 + trino.py:852 (`WITH ([^)]*)`) \u2192 breaks on\n `WITH (partitioning = ARRAY[date_trunc('day', ts)])`\n\nAdd unit tests covering each trigger. Keep adapter-specific PK/FK and\nClickHouse/Fabric cases in w22/w23 so failures isolate cleanly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P1 cluster A \u2014 replace WITH/DDL option regex strippers with the balanced walker from w5", "wid": "w11"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w11"], "notes": "Three asymmetric drift items between presto.py and trino.py:\n1. presto.py:847-854 strips NOT NULL for memory catalog; trino.py\n does NOT. Trino memory catalog also rejects NOT NULL.\n2. presto.py:633-635 uses bare `raise`; trino.py:640-647 calls\n `_build_friendly_connection_error`. Lift the helper to\n presto_trino_utils.py and use in both.\n3. trino.py:681-696 _load_table_data sends unqualified\n `INSERT INTO orders`; presto.py:669-697 prepends `catalog.schema.`.\n Promote to a shared helper that always qualifies.\n\nStarburst inherits via TrinoAdapter \u2014 no separate fix needed there.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Restore Presto/Trino symmetry: NOT NULL stripping, friendly connect error, qualified _load_table_data", "wid": "w12"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "motherduck.py:56,116,135-138 \u2014 module docstring at :56 advertises\n`--platform-option token=`, but `add_cli_arguments` registers\n`--motherduck-token`/`--motherduck-database` and `from_config` reads\n`motherduck_token`/`motherduck_database` keys (not `token`/`database`\nfrom options). Either (a) make from_config read both option keys\nAND the legacy ones, or (b) align the docstring with the actual flag.\nPrefer (a) \u2014 it's the migration path other adapters took.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P3 \u2014 Fix motherduck `--platform-option token=` advertised in docstring but not wired", "wid": "w13"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "snowflake.py:441-453 \u2014 `database` is referenced in the except branch\nbut bound only after `_create_admin_connection(...)`. If\nconnection-build raises, the handler raises NameError and masks the\noriginal. Move\n`database = connection_config.get(\"database\", self.database)` to the\ntop of the function before any callable that can raise.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P15 \u2014 Fix snowflake.drop_database reference-before-assignment in exception handler", "wid": "w14"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w3"], "notes": "Defensive \u2014 prevents future N2 (ABC contract divergence). New test\n`tests/unit/platforms/test_abc_conformance.py` walks every concrete\nsubclass of PlatformAdapter, asserts that\n{create_schema, load_data, execute_query, drop_database} match the\nABC signature using `inspect.signature`. Snowpark Connect (after w3),\npg_mooncake (after w2) should pass; failures mean someone diverged\nagain.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ABC-conformance test that introspects every concrete adapter's method signatures", "wid": "w15"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w12", "w21"], "notes": "Smaller correctness/visibility items batched as one PR:\n- postgresql.py:444-451 \u2014 FK-stripped retry catches the second\n exception, warning-logs, and proceeds. Promote to raise (or push\n onto critical_failures list like SingleStore) when the second\n attempt also fails on a CREATE TABLE. (P9 Required.)\n- presto.py:633-635 \u2014 verify the bare raise was replaced by w12's\n shared `_build_friendly_connection_error`; do not duplicate the fix here.\n- cedardb.py:243-247 \u2014 multi-table DROP loop emits no per-table log\n lines until final summary.\n- doris.py:643-665 \u2014 wrap bare `raise` to include host/user context.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P11/P9 cluster \u2014 postgresql FK-strip silent retry + presto bare raise + cedardb per-table drop logs", "wid": "w16"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w13"], "notes": "PG-family adapters wire BOTH `register_option_specs` AND\n`add_cli_arguments` for the same fields, with undefined precedence:\npostgresql.py:55-118, pg_duckdb.py:64-122, timescaledb.py:76-107,\ncedardb.py:56-70.\n\nPer adapter: either (a) make add_cli_arguments a thin shim that\nenumerates option-spec keys and registers them as legacy flags\npointing at the same dest, or (b) delete the duplicates after a\ndeprecation cycle. Preserve existing legacy invocations.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P14 cluster A \u2014 normalize PG-family legacy flag shims against option specs", "wid": "w17"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w4"], "notes": "Following the canonical _spark_helpers.py pattern, add the same\ndocstring paragraph to:\n- benchbox/platforms/cloud_shared.py:1-9\n- benchbox/platforms/presto_trino_utils.py:1\n- benchbox/platforms/questdb_rewriter.py:1-110\n- benchbox/platforms/datafusion_query_transformer.py\n- benchbox/platforms/datafusion_write_transformer.py\n\nOne-paragraph note: \"Adapters import these functions by name; tests\nmust monkeypatch on the adapter module\n(e.g. benchbox.platforms.snowflake.validate_session_cache_control),\nnot on this module, or the patch silently no-ops.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P7 \u2014 Add import-by-name discipline docstring to remaining shared helpers", "wid": "w18"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w23", "w25"], "notes": "Small Firebolt/ClickHouse/QuestDB cleanup slice:\n- firebolt.py:1056 \u2014 re-enable inline NOT NULL stripper (or document\n the deliberate keep)\n- firebolt.py:1275-1315 \u2014 refactor validate_session_cache_control to\n delegate to cloud_shared (parameterise query string)\n- clickhouse_cloud.py:75,79 \u2014 clean up docstring that mixes\n `--clickhouse-cloud-*` and `--platform-option` forms\n- questdb.py \u2014 set `_max_identifier_length = 127` on the adapter;\n move the override out of the mixin docstring\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Nit/Consider tail A \u2014 firebolt/clickhouse_cloud/questdb docs and helper cleanup", "wid": "w19"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "pg_mooncake.py:193-207 \u2014 current signature\n`create_schema(self, connection, schema_ddl, **kwargs)` does not match\nPostgreSQLAdapter.create_schema(self, benchmark, connection) -> float\n(verified at postgresql.py:422 and base/adapter.py:254).\n\nRefactor to override `_transform_create_statement` (the SingleStore\npattern) so the columnstore USING clause is injected per-statement\ninside the parent's normal flow. Delete the broken create_schema\noverride. Update tests/unit/platforms/test_pg_mooncake_*.py \u2014 current\ntests acknowledge they \"can't call create_schema directly without a\nreal connection,\" which is what masked the regression.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "C2 \u2014 Fix pg_mooncake.create_schema signature mismatch with parent", "wid": "w2"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "docker/questdb/docker-compose.yml:23-47 \u2014 comment claims \"Data is\nephemeral - no named volume,\" but with no `tmpfs:` block and\n`restart: unless-stopped`, state actually sticks to the container\ninstance across `docker compose restart`.\nEither (a) add `tmpfs: - ` to make it RAM-backed,\nor (b) rewrite the comment to \"no host-mounted volume; container\nrecreate clears state.\" Mirror docker/velox/docker-compose.yml\ncanonical pattern.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P5 \u2014 Make docker/questdb truly ephemeral (or fix the misleading comment)", "wid": "w20"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w6"], "notes": "Mechanical, low-risk shared/local cleanup visibility changes. Promote\ndestructive operation logging at:\n- base/connection_lifecycle.py:255-275 (unlink/rmtree/drop_database)\n- datafusion.py:1450-1459 (shutil.rmtree(working_dir))\n- polars_platform.py:657-666 (mirror of datafusion)\n- presto_trino_utils.py:198-205 (add INFO between DROP and recreate\n in execute_schema_statements \"already exists\" handler)\n- base/psycopg2_mixin.py:67-85 (warning, not debug, on\n _get_existing_tables exception)\n\nSingle PR, no behavior change beyond log output. QuestDB failed\npre-create DROP attempts must stay warning-level per w6.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P11 cluster B \u2014 make shared-helper and local-dir destructive ops visible", "wid": "w21"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w11"], "notes": "Second balanced-walker migration slice. Sites and triggers:\n- duckdb.py:733-739 (table-level + inline FK) \u2192 breaks on\n `FOREIGN KEY (...) REFERENCES date_dim(d_date_sk) ON DELETE CASCADE`\n- firebolt.py:1055,1059-1064 (PK + FK) \u2192 breaks on composite PK with\n expression members `PRIMARY KEY (\"a\", coalesce(b,0))`\n\nAdd unit tests covering each trigger.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P1 cluster B \u2014 replace PK/FK regex strippers with the balanced walker from w5", "wid": "w22"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w22"], "notes": "Third balanced-walker migration slice. Sites and triggers:\n- clickhouse/workload.py:65-70 `Nullable\\\\([^)]+\\\\)\\\\s+NOT\\\\s+NULL`\n \u2192 silently breaks decimal-bearing TPC-DS tables for ALL clickhouse\n adapters (`Nullable(Decimal(38,0)) NOT NULL`)\n- clickhouse/workload.py:116 inline PK regex on parametrized types\n- fabric_warehouse.py:1218-1226 `_optimize_table_definition` uses\n str.replace after re.search \u2192 munges `CREATE TABLE IF NOT EXISTS orders`\n\nAdd unit tests covering each trigger.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P1 cluster C \u2014 fix ClickHouse nullable and Fabric table-definition regex edge cases", "wid": "w23"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w17"], "notes": "Follow the w17 shim/deprecation pattern for:\n- singlestore.py:137-175\n- snowpark_connect.py:503-538\n\nSnowpark Connect is urgent because its unprefixed `--user`/`--password`/\n`--account` collide with other adapters' flags in shared parsers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P14 cluster B \u2014 normalize SingleStore and Snowpark Connect legacy flag shims", "wid": "w24"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w24"], "notes": "Inverse case: sqlite.py has only legacy flags and no option-spec\nregistration.\n\nEither register sqlite under\nbenchbox/platforms/__init__.py or document that sqlite intentionally\nbypasses the registry (and update the error message at sqlite.py:124-127\naccordingly).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P14 cluster C \u2014 settle sqlite option-spec registration policy", "wid": "w25"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w19"], "notes": "Small SingleStore/Doris/Fabric cleanup slice:\n- singlestore.py:713-798 \u2014 extract `_extract_create_table_name(stmt)`\n helper; replace 4 copies of the regex\n- doris.py:579-586 \u2014 delegate `_validate_identifier` to\n benchbox.utils.sql_identifier.is_valid_sql_identifier (with\n max_length=128)\n- fabric_lakehouse.py:230,266 \u2014 pick a single error model\n (raise everywhere, or return-as-dict everywhere); align with\n sibling fabric_warehouse.py\n- fabric_spark_adapter.py:730 \u2014 wrap CLI args in a parser group for\n consistency with sibling Fabric adapters\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Nit/Consider tail B \u2014 singlestore/doris/fabric cleanup", "wid": "w26"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "snowpark_connect.py:291,314,385 vs spark.py:590,632,663.\ncreate_schema(self, schema_name=None) \u2192 must accept (benchmark, connection).\nload_data(self, tables, source_dir, file_format=...) \u2192 must accept\n(benchmark, connection, data_dir).\nexecute_query(self, query, **kwargs) \u2192 must accept (connection, query,\nquery_id, ...) and return the standard dict\n{query_id, stream_id, status, execution_time_seconds, ...}.\n\nIf Snowpark Session replaces the \"connection\" arg, accept and ignore\nit positionally. The orchestrator's standard call site will TypeError\non the current code.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "C3 \u2014 Conform snowpark_connect.{create_schema,load_data,execute_query} to PlatformAdapter ABC", "wid": "w3"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w1", "w2"], "notes": "base/ddl_helpers.py:25-69 \u2014 `_FK_CLAUSE_RE` requires the table-level\n`FOREIGN KEY (...)` header. Add a second regex pass for inline\n`\\s+REFERENCES\\s+...` (mirroring questdb.py:670-675), normalize\ntrailing commas, then DELETE questdb._strip_fk_constraints since the\nbase helper now covers both forms.\n\nAlso add the \"Import discipline\" paragraph from\n_spark_helpers.py:7-13 to the module docstring (callers:\npostgresql:19, singlestore:27, doris:35, questdb:28,\ndatabend/adapter:53, starrocks/workload:12 \u2014 all import by name).\n\nVerify against TPC-DS-style inline FK form\n(`cs_sold_date_sk INTEGER REFERENCES date_dim(d_date_sk)`).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Shared helper P2 + P7 \u2014 base/ddl_helpers.py inline-REFERENCES + import-discipline docstring", "wid": "w4"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w4"], "notes": "`_strip_balanced_paren_constraints` in _spark_helpers.py is the\ncanonical fix for Pattern 1 (greedy `[^)]*` over balanced parens).\nMove it into benchbox/platforms/base/ddl_helpers.py (or a new\nbenchbox/platforms/base/sql_paren_walker.py) and re-export from\n_spark_helpers.py for backward compat. This unblocks w11 \u2014 every\nadapter rewriter can then call the same balanced walker instead of\nrolling its own broken `\\([^)]*\\)` regex.\n\nAdd unit tests with at least: `CHECK ((a>0) AND (b>0))`,\n`DECIMAL(10,2)`, `PARTITION BY (date_trunc('day',ts))`,\n`DEFAULT (now())`, `WITH (partitioning = ARRAY['bucket(x, 16)'])`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Lift balanced-paren constraint walker out of _spark_helpers.py into shared module", "wid": "w5"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "Mechanical, low-risk adapter-local visibility changes. Promote\n`logger.debug` \u2192 `logger.info` (or log_operation_complete) at:\n- athena.py:1054 (DROP staging table)\n- athena.py:1070 (S3 staging delete)\n- redshift.py:1156-1158 (DROP TABLE)\n- azure_synapse.py:452-462 (DROP TABLE in schema recreate)\n- fabric_warehouse.py:741-742 (DROP TABLE)\n- presto.py:559 (DROP TABLE)\n- questdb.py:476-480 (pre-create DROP failures \u2192 warning, not info)\n\nKeep this PR limited to adapter-local logging sites. Shared helper\nand local-working-dir destructive operations are covered by w21.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P11 cluster A \u2014 promote adapter destructive-op debug logs to info/warning", "wid": "w6"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "clickhouse_cloud.py:676-682 (S3) and :816-822 (GCS) \u2014 loop assigns\n`total_rows_loaded = SELECT COUNT(*) FROM table` per iteration; the\nsource comment acknowledges the bug. On rerun-into-non-empty-table,\nthe count includes pre-existing rows and silently inflates throughput\nin published BenchmarkResults.\n\nEither TRUNCATE before the loop, or take COUNT(*) once after the\nloop and assign once. Add a regression test that loads two files\ninto a non-empty table and asserts total_rows_loaded == sum of new\nfile row counts only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "N1 \u2014 Fix ClickHouse Cloud COUNT(*) throughput inflation in S3/GCS load loops", "wid": "w7"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": ["w5"], "notes": "cudf.py:605-636 \u2245 polars_platform.py:618-645 \u2245 datafusion.py:1400-1435\ntriplicate `apply_constraint_configuration` and\n`apply_platform_optimizations`; only the platform-name string differs.\n\nAdd benchbox/platforms/base/no_constraint_enforcement_mixin.py with\nthe parameterised methods (read self.platform_name). Update the three\nadapters to inherit. Update or delete the three duplicate test files.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Extract NoConstraintEnforcementMixin (cudf, datafusion, polars_platform)", "wid": "w8"}, {"evidence": null, "item_id": "address-cross-adapter-13-pattern-audit-findings", "needs": [], "notes": "timescaledb.py:110-159 \u2245 postgresql.py:154-201 \u2245 pg_duckdb.py:124-181 \u2245\npg_mooncake.py:67-116 \u2245 cedardb.py:73-119 \u2014 ~30 identical lines for\nhost/port/username/password/schema/sslmode + database-naming.\n\nAdd a class method `_build_postgres_connection_kwargs(cls, config)`\non PostgreSQLAdapter that the four subclasses call. Concrete drift\nrisk: pg_mooncake/cedardb already lack `enable_timescale` and\n`statement_timeout` while sharing the same parent __init__.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "P12 \u2014 Extract _build_postgres_connection_kwargs helper for PG family from_config", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (COMPLETED)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: Completed \u2705\"\n- \"**Description**: Hardened resource monitoring and performance summary support in `benchbox/platforms/base.py`, removing the final TODO placeholders before release.\"\n- \"**Context**: The base adapter previously exposed minimal metrics and left TODO markers, making benchmark outputs inconsistent and limiting result diagnostics.\"\n- \"**Tasks**:\"\n- \"**Files Affected**:\"\n- \"**Success Metrics**:\"\n- \"**Impact**: HIGH - Provides actionable telemetry in benchmark results, clears lingering TODOs, and gives users immediate insight into performance and resource profiles.\"", "id": "address-outstanding-todos", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Address Outstanding TODOs", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "keep it opt-in with a clear skip message.", "dont": "make ci-linux a hard prerequisite of pr-open or any CI job", "why": "most contributors are not on Apple-silicon macOS 26"}, {"INSTEAD": "(not specified)", "dont": "rebuild the agent image per run", "why": "create the machine once and reuse it; rebuild only when the toolchain changes."}, {"INSTEAD": "(not specified)", "dont": "route routine `pytest -m fast` through the container", "why": "it already runs natively on macOS; the VM hop only adds latency."}], "approach": "Treat Apple container as a *parity sandbox*, not a general test runner --\nroutine fast tests stay native on macOS (faster, no VM hop). The only\nreason to pay the VM cost is to reproduce Linux-specific behavior the Mac\ncannot. Build benchbox-agent once, reuse the persistent machine across\nsessions (home-mounted), and gate the whole adoption on the w4 finding so\nwe do not add a wrapper target that earns its keep only marginally.\n", "blocked_reason": null, "category": "Developer Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-03T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "adopt-apple-container-ci-parity", "id": 301, "reason": "Needs a database service, which is the Docker Compose problem tracked in migrate-test-docker-stacks-to-mocker. postgres-integration is also a non-blocking sample, not the hard gate.\n", "resolution": "promoted", "resolved_item": "reproduce-postgres-integration-needs-a-live-postgres-inside", "resolved_reason": null, "summary": "Reproduce postgres-integration (needs a live Postgres) inside container"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "adopt-apple-container-ci-parity", "id": 302, "reason": "Apple container is Apple-silicon + macOS-26 only; it cannot be a required step for contributors on other hardware, and CI already runs real ubuntu. Local opt-in only.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Reason itself concludes ci-linux can't be a required gate (Apple-silicon+macOS-26 only); CI already runs real ubuntu. Nothing to promote.", "summary": "Make ci-linux a required/enforced step or run it in CI"}], "deps": [], "description": "Every PR-gating job in .github/workflows/pr.yml runs on `ubuntu-latest`,\nbut the project is developed on Apple-silicon macOS. So every PR is a\nmacOS -> Linux leap of faith: a pure-Python check can pass locally on\nmacOS and then fail in CI on locale, path-case, float formatting, dict/\nhash ordering, line endings, or `uv` lock/arch resolution differences.\nThe correctness-gate and the *-cross-surface-equivalence-report jobs are\nthe sharp edge here -- they compare result digests against pinned Linux\nreference results, exactly the class of value that can format/round\ndifferently across OS.\n\nApple's `container` (v1.0.0, GA) runs Linux containers as lightweight VMs\non Apple silicon and ships a `container machine` mode: a persistent Linux\nenvironment with the Mac home directory mounted in, so you \"edit on the\nMac, build inside Linux\". This lets us reproduce the *pure-Python* Linux\nPR gate locally before submitting -- the one capability native macOS\ncannot provide.\n\nScope of THIS TODO is the runtime/agent substrate only: stand up Apple\n`container` + a `benchbox-agent` machine, run the pure-Python gate inside\nit, decide whether in-Linux digests actually diverge from macOS-local,\nand (if warranted) add an opt-in `make ci-linux` wrapper. It explicitly\ndoes NOT cover Docker Compose / `make test-docker-*` (no Compose support\nin `container`); that is tracked separately in\nmigrate-test-docker-stacks-to-mocker.\n\nSpike evidence (2026-06-30, host macOS 26.5 / arm64): `brew install\ncontainer` -> container CLI 1.0.0; `container system start` +\n`container system kernel set --recommended` (kata-static 3.28.0) brought\nthe runtime up; a single Linux container published ports to the Mac\n`localhost` and served real traffic (questdb REST `localhost:19000/exec`\n-> result; postgres `localhost:5432` psycopg round-trip -> PostgreSQL\n18.4, RW returned 42). `container machine` mode is documented and GA but\nwas NOT itself exercised in the spike -- w0/w1 must confirm it.\n\nFINDINGS (2026-07-03, container 1.0.0 / macOS 26.5 / arm64):\n- w0 PASS: container 1.0.0, apiserver running, arm64/macOS 26.5, mocker 0.5.4.\n- w1/w2 PASS: `container machine` boots ONLY with an init in the rootfs\n (ubuntu:24.04 needs systemd-sysv for /sbin/init); home-mount exposes the\n worktree rw at its Mac path via virtiofs. Image: docker/benchbox-agent.\n- w3: correctness-gate + ssb equivalence ran inside the machine on a targeted\n lock-pinned env (core + duckdb 1.3.2 + polars); full `uv sync --group dev` +\n `make ci-local` exceeded this host's ~6.6 GiB free disk (the VM sparse disk\n only grows). Digest is DuckDB-build-tied, so the subset does not weaken w4.\n- w4 = DIVERGES (central finding): identical DuckDB 1.3.2/arm64, TPC-H SF=1 --\n Linux correctness-gate PASS (18/18 match pinned ref); macOS FAIL on Q2/Q10/\n Q15 (same cardinality, different value digests, deterministic over 2 runs).\n ssb cross-surface equivalence PASS on both (self-referential -> OS-insensitive).\n The pinned references are effectively Linux-specific, so `container` parity is\n the only way to validate the correctness-gate pre-PR from a Mac -> shipped the\n opt-in `make ci-linux` wrapper (w5) + AGENTS.md (w6).", "id": "adopt-apple-container-ci-parity", "preserves": ["No PR-gating CI job gains a dependency on Apple container; .github/workflows/ stays ubuntu-latest.", "make ci-local and the correctness-gate / *-cross-surface-equivalence-report targets keep working unchanged when run natively on macOS (ci-linux is purely additive).", "test-docker-* targets are untouched by this TODO."], "prior_art": [{"concept": "Makefile:1065 agent-write-preflight \u2014 pattern: agent-/ci- prefixed Make targets that guard/parameterize the dev loop; follow its naming + guard style for ci-linux.", "decision": "reuse", "path": "Makefile:1065"}, {"concept": "Makefile:794 ci-local \u2014 extend: ci-linux wraps the same gate bundle but executes it inside `container machine run`.", "decision": "extend", "path": "Makefile:794"}, {"concept": "docker/lakesail/Dockerfile, docker/velox/Dockerfile \u2014 reuse as reference for the benchbox-agent image build; do not reuse the DB stack compose files.", "decision": "reuse", "path": "docker/lakesail/Dockerfile"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "do_not_modify", "path_glob": "docker/*/docker-compose.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docker/benchbox-agent/Dockerfile"}], "state": "done", "title": "Adopt Apple container for local Linux CI-parity validation before PRs", "verifications": [{"command": "container system status", "description": "Apple container runtime is up", "expected": "apiserver reported running", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 1}, {"command": "container machine run -n benchbox-agent -- bash -lc 'uname -a && uv --version'", "description": "benchbox-agent machine reaches a Linux shell that can sync deps", "expected": "Linux ... aarch64; uv version prints", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 2}, {"command": "container machine run -n benchbox-agent -- bash -lc 'cd && uv sync --group dev && make ci-local'", "description": "Pure-Python gate runs to completion inside the machine", "expected": "ci-local completes; pass/fail recorded for w3", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 3}, {"command": "make test-correctness-gate # native macOS vs inside machine, diff outputs", "description": "Digest divergence finding captured (w4)", "expected": "documented whether macOS-local digests match pinned Linux references", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": [], "notes": "Evidence-durability gate per _project/verification-logs/README.md.\nThe description cites container CLI 1.0.0 on macOS 26.5/arm64 and\nobserved localhost reachability. Before w1, confirm:\n - `container --version` (record the version actually installed now).\n - `container system status` reports the apiserver running after\n `container system start`.\n - host is Apple silicon + macOS 26+ (`uname -m`, `sw_vers`).\nCapture raw stdout to `/tmp/adopt-apple-container-ci-parity-w0.log`\nor a CI artifact; commit only a compact summary (command, version,\nPASS/FAIL, key lines) into this TODO or the PR body. If `container`\nno longer installs/starts on the current macOS, re-scope before w1.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate cited spike evidence against current container version", "wid": "w0"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w0", "w2"], "notes": "`container machine create --name benchbox-agent` with the Mac\nhome mounted; `container machine set -n benchbox-agent cpus/memory`;\nverify `container machine run -n benchbox-agent -- bash` lands in a\nLinux shell that can see a claimed pool worktree path. THIS is the\nstep the spike did not exercise -- prove machine mode + home-mount\nwork, not just `container run`.\nCONFIRMED (also flagged on develop): the machine is created FROM the w2\nimage, so w1 depends on w2; and `container machine` boots only when the\nrootfs has /sbin/init (systemd-sysv), which minimal ubuntu lacks.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stand up a benchbox-agent container machine with home-mount", "wid": "w1"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w0"], "notes": "ubuntu:24.04 + git, build-essential, curl, ca-certificates, uv\n(and the system libs BenchBox's deps need to `uv sync --group dev`).\nBuild with `container build --arch arm64 --tag local/benchbox-agent`.\narm64 only unless a check specifically needs x86 parity.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build the benchbox-agent OCI image (ubuntu + uv + build toolchain)", "wid": "w2"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w1", "w2"], "notes": "Inside a claimed worktree, run the Linux-reproducible subset of\npr.yml: `uv sync --group dev`, `make ci-local`, `make\ntest-correctness-gate`, and the *-cross-surface-equivalence-report\ntargets the correctness-gate job runs. Record pass/fail + any digest\noutput. clickhouse-integration is in-process chDB (no service) so it\nis in-scope here; postgres-integration needs a DB and is out of scope\n(see deps).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the pure-Python PR gate inside the machine; capture results", "wid": "w3"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w3"], "notes": "Run the same correctness-gate + equivalence targets natively on macOS\nAND inside the Linux machine; diff the digests/outputs. This decides\nthe tool's value:\n - If macOS-local already matches the pinned Linux references, the\n parity win is marginal -> stop at \"documented optional tool\".\n - If they diverge, container parity is the ONLY way to validate\n these gates pre-PR -> proceed to w5 and recommend it.\nDocument the observed divergence (or lack of it) explicitly in the\nTODO/PR -- this is the central finding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DECISION GATE: do correctness/equivalence digests diverge macOS vs Linux?", "wid": "w4"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w4"], "notes": "Only if w4 shows divergence (or as a convenience even if not, at\nreviewer discretion). Add a `ci-linux` target that runs the gate\nbundle inside `container machine run -n benchbox-agent`. Keep it\nadditive and clearly named -- distinct from `test-docker-*`. Must be\na no-op-with-clear-message on non-Apple-silicon / non-macOS-26 hosts,\nnever a hard CI dependency.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add opt-in `make ci-linux` wrapper (gated on w4 = diverges)", "wid": "w5"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w5"], "notes": "One dense block: prerequisites (Apple silicon, macOS 26), one-time\nsetup (install/start/kernel/machine-create), and the pre-PR command.\nMatch the existing terse documentation style; no bash tutorials that\nrestate `container --help`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document setup + usage in AGENTS.md (terse, house style)", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "make ci-linux a hard prerequisite of pr-open or any CI job", "instead": "keep it opt-in with a clear skip message.", "why": "most contributors are not on Apple-silicon macOS 26"}, {"dont": "rebuild the agent image per run", "instead": "(not specified)", "why": "create the machine once and reuse it; rebuild only when the toolchain changes."}, {"dont": "route routine `pytest -m fast` through the container", "instead": "(not specified)", "why": "it already runs natively on macOS; the VM hop only adds latency."}], "approach": "Treat Apple container as a *parity sandbox*, not a general test runner --\nroutine fast tests stay native on macOS (faster, no VM hop). The only\nreason to pay the VM cost is to reproduce Linux-specific behavior the Mac\ncannot. Build benchbox-agent once, reuse the persistent machine across\nsessions (home-mounted), and gate the whole adoption on the w4 finding so\nwe do not add a wrapper target that earns its keep only marginally.\n", "blocked_reason": null, "category": "Developer Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-03T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "adopt-apple-container-ci-parity", "id": 301, "reason": "Needs a database service, which is the Docker Compose problem tracked in migrate-test-docker-stacks-to-mocker. postgres-integration is also a non-blocking sample, not the hard gate.\n", "resolution": "promoted", "resolved_item": "reproduce-postgres-integration-needs-a-live-postgres-inside", "resolved_reason": null, "summary": "Reproduce postgres-integration (needs a live Postgres) inside container"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "adopt-apple-container-ci-parity", "id": 302, "reason": "Apple container is Apple-silicon + macOS-26 only; it cannot be a required step for contributors on other hardware, and CI already runs real ubuntu. Local opt-in only.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Reason itself concludes ci-linux can't be a required gate (Apple-silicon+macOS-26 only); CI already runs real ubuntu. Nothing to promote.", "summary": "Make ci-linux a required/enforced step or run it in CI"}], "deps": [], "description": "Every PR-gating job in .github/workflows/pr.yml runs on `ubuntu-latest`,\nbut the project is developed on Apple-silicon macOS. So every PR is a\nmacOS -> Linux leap of faith: a pure-Python check can pass locally on\nmacOS and then fail in CI on locale, path-case, float formatting, dict/\nhash ordering, line endings, or `uv` lock/arch resolution differences.\nThe correctness-gate and the *-cross-surface-equivalence-report jobs are\nthe sharp edge here -- they compare result digests against pinned Linux\nreference results, exactly the class of value that can format/round\ndifferently across OS.\n\nApple's `container` (v1.0.0, GA) runs Linux containers as lightweight VMs\non Apple silicon and ships a `container machine` mode: a persistent Linux\nenvironment with the Mac home directory mounted in, so you \"edit on the\nMac, build inside Linux\". This lets us reproduce the *pure-Python* Linux\nPR gate locally before submitting -- the one capability native macOS\ncannot provide.\n\nScope of THIS TODO is the runtime/agent substrate only: stand up Apple\n`container` + a `benchbox-agent` machine, run the pure-Python gate inside\nit, decide whether in-Linux digests actually diverge from macOS-local,\nand (if warranted) add an opt-in `make ci-linux` wrapper. It explicitly\ndoes NOT cover Docker Compose / `make test-docker-*` (no Compose support\nin `container`); that is tracked separately in\nmigrate-test-docker-stacks-to-mocker.\n\nSpike evidence (2026-06-30, host macOS 26.5 / arm64): `brew install\ncontainer` -> container CLI 1.0.0; `container system start` +\n`container system kernel set --recommended` (kata-static 3.28.0) brought\nthe runtime up; a single Linux container published ports to the Mac\n`localhost` and served real traffic (questdb REST `localhost:19000/exec`\n-> result; postgres `localhost:5432` psycopg round-trip -> PostgreSQL\n18.4, RW returned 42). `container machine` mode is documented and GA but\nwas NOT itself exercised in the spike -- w0/w1 must confirm it.\n\nFINDINGS (2026-07-03, container 1.0.0 / macOS 26.5 / arm64):\n- w0 PASS: container 1.0.0, apiserver running, arm64/macOS 26.5, mocker 0.5.4.\n- w1/w2 PASS: `container machine` boots ONLY with an init in the rootfs\n (ubuntu:24.04 needs systemd-sysv for /sbin/init); home-mount exposes the\n worktree rw at its Mac path via virtiofs. Image: docker/benchbox-agent.\n- w3: correctness-gate + ssb equivalence ran inside the machine on a targeted\n lock-pinned env (core + duckdb 1.3.2 + polars); full `uv sync --group dev` +\n `make ci-local` exceeded this host's ~6.6 GiB free disk (the VM sparse disk\n only grows). Digest is DuckDB-build-tied, so the subset does not weaken w4.\n- w4 = DIVERGES (central finding): identical DuckDB 1.3.2/arm64, TPC-H SF=1 --\n Linux correctness-gate PASS (18/18 match pinned ref); macOS FAIL on Q2/Q10/\n Q15 (same cardinality, different value digests, deterministic over 2 runs).\n ssb cross-surface equivalence PASS on both (self-referential -> OS-insensitive).\n The pinned references are effectively Linux-specific, so `container` parity is\n the only way to validate the correctness-gate pre-PR from a Mac -> shipped the\n opt-in `make ci-linux` wrapper (w5) + AGENTS.md (w6).", "id": "adopt-apple-container-ci-parity", "preserves": ["No PR-gating CI job gains a dependency on Apple container; .github/workflows/ stays ubuntu-latest.", "make ci-local and the correctness-gate / *-cross-surface-equivalence-report targets keep working unchanged when run natively on macOS (ci-linux is purely additive).", "test-docker-* targets are untouched by this TODO."], "prior_art": [{"concept": "Makefile:1065 agent-write-preflight \u2014 pattern: agent-/ci- prefixed Make targets that guard/parameterize the dev loop; follow its naming + guard style for ci-linux.", "decision": "reuse", "path": "Makefile:1065"}, {"concept": "Makefile:794 ci-local \u2014 extend: ci-linux wraps the same gate bundle but executes it inside `container machine run`.", "decision": "extend", "path": "Makefile:794"}, {"concept": "docker/lakesail/Dockerfile, docker/velox/Dockerfile \u2014 reuse as reference for the benchbox-agent image build; do not reuse the DB stack compose files.", "decision": "reuse", "path": "docker/lakesail/Dockerfile"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "do_not_modify", "path_glob": "docker/*/docker-compose.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docker/benchbox-agent/Dockerfile"}], "state": "done", "title": "Adopt Apple container for local Linux CI-parity validation before PRs", "verifications": [{"command": "container system status", "description": "Apple container runtime is up", "expected": "apiserver reported running", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 1}, {"command": "container machine run -n benchbox-agent -- bash -lc 'uname -a && uv --version'", "description": "benchbox-agent machine reaches a Linux shell that can sync deps", "expected": "Linux ... aarch64; uv version prints", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 2}, {"command": "container machine run -n benchbox-agent -- bash -lc 'cd && uv sync --group dev && make ci-local'", "description": "Pure-Python gate runs to completion inside the machine", "expected": "ci-local completes; pass/fail recorded for w3", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 3}, {"command": "make test-correctness-gate # native macOS vs inside machine, diff outputs", "description": "Digest divergence finding captured (w4)", "expected": "documented whether macOS-local digests match pinned Linux references", "item_id": "adopt-apple-container-ci-parity", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": [], "notes": "Evidence-durability gate per _project/verification-logs/README.md.\nThe description cites container CLI 1.0.0 on macOS 26.5/arm64 and\nobserved localhost reachability. Before w1, confirm:\n - `container --version` (record the version actually installed now).\n - `container system status` reports the apiserver running after\n `container system start`.\n - host is Apple silicon + macOS 26+ (`uname -m`, `sw_vers`).\nCapture raw stdout to `/tmp/adopt-apple-container-ci-parity-w0.log`\nor a CI artifact; commit only a compact summary (command, version,\nPASS/FAIL, key lines) into this TODO or the PR body. If `container`\nno longer installs/starts on the current macOS, re-scope before w1.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate cited spike evidence against current container version", "wid": "w0"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w0", "w2"], "notes": "`container machine create --name benchbox-agent` with the Mac\nhome mounted; `container machine set -n benchbox-agent cpus/memory`;\nverify `container machine run -n benchbox-agent -- bash` lands in a\nLinux shell that can see a claimed pool worktree path. THIS is the\nstep the spike did not exercise -- prove machine mode + home-mount\nwork, not just `container run`.\nCONFIRMED (also flagged on develop): the machine is created FROM the w2\nimage, so w1 depends on w2; and `container machine` boots only when the\nrootfs has /sbin/init (systemd-sysv), which minimal ubuntu lacks.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stand up a benchbox-agent container machine with home-mount", "wid": "w1"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w0"], "notes": "ubuntu:24.04 + git, build-essential, curl, ca-certificates, uv\n(and the system libs BenchBox's deps need to `uv sync --group dev`).\nBuild with `container build --arch arm64 --tag local/benchbox-agent`.\narm64 only unless a check specifically needs x86 parity.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build the benchbox-agent OCI image (ubuntu + uv + build toolchain)", "wid": "w2"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w1", "w2"], "notes": "Inside a claimed worktree, run the Linux-reproducible subset of\npr.yml: `uv sync --group dev`, `make ci-local`, `make\ntest-correctness-gate`, and the *-cross-surface-equivalence-report\ntargets the correctness-gate job runs. Record pass/fail + any digest\noutput. clickhouse-integration is in-process chDB (no service) so it\nis in-scope here; postgres-integration needs a DB and is out of scope\n(see deps).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the pure-Python PR gate inside the machine; capture results", "wid": "w3"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w3"], "notes": "Run the same correctness-gate + equivalence targets natively on macOS\nAND inside the Linux machine; diff the digests/outputs. This decides\nthe tool's value:\n - If macOS-local already matches the pinned Linux references, the\n parity win is marginal -> stop at \"documented optional tool\".\n - If they diverge, container parity is the ONLY way to validate\n these gates pre-PR -> proceed to w5 and recommend it.\nDocument the observed divergence (or lack of it) explicitly in the\nTODO/PR -- this is the central finding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DECISION GATE: do correctness/equivalence digests diverge macOS vs Linux?", "wid": "w4"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w4"], "notes": "Only if w4 shows divergence (or as a convenience even if not, at\nreviewer discretion). Add a `ci-linux` target that runs the gate\nbundle inside `container machine run -n benchbox-agent`. Keep it\nadditive and clearly named -- distinct from `test-docker-*`. Must be\na no-op-with-clear-message on non-Apple-silicon / non-macOS-26 hosts,\nnever a hard CI dependency.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add opt-in `make ci-linux` wrapper (gated on w4 = diverges)", "wid": "w5"}, {"evidence": null, "item_id": "adopt-apple-container-ci-parity", "needs": ["w5"], "notes": "One dense block: prerequisites (Apple silicon, macOS 26), one-time\nsetup (install/start/kernel/machine-create), and the pre-PR command.\nMatch the existing terse documentation style; no bash tutorials that\nrestate `container --help`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document setup + usage in AGENTS.md (terse, house style)", "wid": "w6"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Cost Intelligence", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Extend cost analysis to support multi-region pricing, spot instances,\ncommitted use discounts, and data transfer costs. Enable TCO comparisons\nand cost optimization recommendations.", "id": "advanced-cost-analysis-framework-multi-region-pricing-models", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Advanced Cost Analysis Framework - Multi-Region & Pricing Models", "verifications": [], "work": [], "worktree": "cost-intelligence-advanced"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Cost Estimation and Transparency", "claimed_at": null, "claimed_by": null, "completed_at": "2025-11-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Conducted adversarial review of cost estimation framework and implemented 4 phases of improvements to address transparency and accuracy gaps. Added cost_model field to distinguish marginal vs actual costs, implemented warnings system for user education, added pricing version tracking with staleness detection, enriched phase costs with timing context for budgeting, and created storage cost estimation module. Updated comprehensive documentation with output schema examples.", "id": "adversarial-review-cost-framework-enhancement-warnings-metadata-timing-storage", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Adversarial Review - Cost Framework Enhancement (Warnings, Metadata, Timing, Storage)", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Conducted comprehensive adversarial review to identify and remove all placeholder stub implementations. Completed 3-phase remediation plan removing unused legacy methods (TPC-DS), NotImplementedError execution methods (TPC-Havoc), and dead code (TPC-DI). Added comprehensive usage documentation and integration tests. Maintained architectural consistency across all benchmarks using platform adapter pattern.", "id": "adversarial-review-notimplementederror-stub-removal-tpc-ds-tpc-havoc-tpc-di", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Adversarial Review - NotImplementedError Stub Removal (TPC-DS, TPC-Havoc, TPC-DI)", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Benchmark Development", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Create a new AI/ML Primitives benchmark to test cross-platform SQL-based AI/ML\nfunctions. This benchmark covers built-in AI functions that execute within the\ndata warehouse, NOT external ML frameworks or vector search (covered separately).\n\nFocus areas:\n- LLM/Generative AI functions (Snowflake Cortex, BigQuery ML, Databricks AI)\n- Sentiment analysis, text classification, summarization\n- Entity extraction, translation\n- SQL-invokable ML predictions (not model training)\n\nThis differs from the existing Vector Search benchmark which focuses on\nembedding storage and similarity search. This benchmark tests AI function\ninvocation performance and output quality consistency.", "id": "ai-ml-primitives-benchmark", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "AI/ML Primitives Benchmark - SQL-Based AI Functions", "verifications": [], "work": [], "worktree": "benchmark-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "AI/ML Capabilities", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Benchmark AI/ML SQL functions (sentiment analysis, classification, summarization,\nLLM completion) across Snowflake Cortex, BigQuery ML, Databricks ML.", "id": "ai-ml-sql-function-performance-testing", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "AI/ML SQL Function Performance Testing", "verifications": [], "work": [], "worktree": "ai-ml-functions"} -{"anti_patterns": [{"INSTEAD": "adopt when BenchBox adds throughput or Power@Size chart rendering", "dont": "adopt value_formatter or lower_is_better without a concrete use case", "why": "defaults are correct for latency metrics"}, {"INSTEAD": "(not specified)", "dont": "change any explicit title= or metric_label= values in ascii_runtime.py", "why": "they are already correct and the audit confirmed no reliance on defaults"}, {"INSTEAD": "(not specified)", "dont": "remove deprecated factory names from the compatibility shim", "why": "external callers using old BenchBox import paths need both old and new names available"}], "approach": "Low-risk mechanical migration. The audit confirms BenchBox passes all labels\nand titles explicitly - no call site relies on changed defaults. The work is:\n5 factory function renames, 3 dead import removals, 2 verifications, and an\noptional subject= adoption evaluation. Golden tests are the safety net.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "textcharts completed its full de-benchboxing initiative (5 change groups).\nAll changes are backward-compatible - old APIs still work but emit\nDeprecationWarning. BenchBox should update call sites to use the new APIs.\n\nAudit findings (2026-03-09):\n- All 14+ constructor calls in ascii_runtime.py and exporters.py pass\n title=, metric_label=, y_label= etc. explicitly - zero reliance on\n changed defaults. No output changes from Group 1.\n- 5 deprecated factory function calls in ascii_runtime.py need renaming.\n- 3 unused deprecated imports in test_ascii_charts.py need removal.\n- Compatibility shim (ascii/__init__.py) does `from textcharts import *`\n so new exports are automatically available.\n- DEFAULT_MAX_QUERIES is not referenced in BenchBox - no action needed.\n- lower_is_better defaults to True - matches BenchBox latency semantics.\n- subject= parameter available but optional - could simplify title passing.\n\nSupersedes: adapt-benchbox-to-textcharts-neutral-defaults (archived)", "id": "align-benchbox-with-textcharts-de-benchboxing", "preserves": ["All 15 golden snapshot tests pass unchanged - BenchBox passes explicit values everywhere", "BenchBox CLI and MCP chart output byte-identical to pre-migration", "Compatibility shim re-exports both old and new names for external callers", "Zero DeprecationWarning from textcharts during normal BenchBox operation"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "packages/textcharts/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_api.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_runtime.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/contracts/visualization/"}, {"kind": "only_modify", "path_glob": "tests/fixtures/golden/ascii/"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/"}], "state": "done", "title": "Align BenchBox call sites with textcharts de-benchboxed API", "verifications": [{"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v", "description": "Golden snapshot parity maintained", "expected": "All 15 snapshot tests pass", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q", "description": "Zero DeprecationWarning from textcharts in visualization tests", "expected": "All tests pass without DeprecationWarning", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -q", "description": "Full test suite passes", "expected": "All tests pass", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run ruff check benchbox/core/visualization/ tests/unit/visualization/", "description": "Lint clean on modified files", "expected": "No errors", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": [], "notes": "1. Update textcharts version constraint in pyproject.toml\n2. Run: `uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v`\n3. All 15 golden tests should pass unchanged - BenchBox passes all labels\n and titles explicitly, so the default changes in Group 1 are invisible.\n4. If any fail, inspect: the only legitimate failures would be from test\n fixtures that construct charts without explicit labels/titles.\n Regenerate with `--update-golden` after visual inspection.\n5. Run full test suite: `uv run -- python -m pytest -q`\n6. Verify new parameter defaults match BenchBox semantics:\n - `lower_is_better: bool = True` on ASCIIComparisonBar, ASCIIDivergingBar,\n SummaryStats - default True correct for latency (lower = better).\n - `outlier_cap: float | None = None` on ASCIIChartOptions - None means\n auto P95\u00d72 capping, unchanged behavior.\n - Visual spot-check: comparison_bar and diverging_bar golden output\n shows correct green (improvement) / red (regression) colors.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Bump textcharts version and verify golden test parity", "wid": "w1"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w1"], "notes": "These are the ONLY deprecated factory calls in production code. Each is\nimported from a submodule inside a render function. Update both the import\nand the call site.\n\n| Line | Current import | New import | New call |\n|------|---------------|------------|----------|\n| 379 | `from ...percentile_ladder import from_query_results` | `from ...percentile_ladder import from_series` | `from_series(...)` |\n| 395 | `from ...normalized_speedup import from_normalized_results` | `from ...normalized_speedup import from_ratios` | `from_ratios(...)` |\n| 428 | `from ...sparkline_table import from_metrics` | `from ...sparkline_table import from_data` | `from_data(...)` |\n| 453 | `from ...cdf_chart import from_query_results` | `from ...cdf_chart import from_series` | `from_series(...)` |\n| 465 | `from ...rank_table import from_heatmap_data` | `from ...rank_table import from_matrix` | `from_matrix(...)` |\n\nNote: BenchBox imports from `benchbox.core.visualization.ascii.`,\nnot from `textcharts` directly. The compatibility shim re-exports both\nold and new names, so the new names are already importable.\n\nAfter changes, verify: `uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q`\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rename 5 deprecated factory function calls in ascii_runtime.py", "wid": "w2"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w1"], "notes": "Three deprecated factory functions are imported but never called in\n`tests/unit/visualization/ascii/test_ascii_charts.py`:\n\n- Line 13: `from_bar_data` - unused\n- Line 26: `from_normalized_results` - unused\n- Line 31: `from_query_results` - unused\n\nRemove these dead imports. Run lint to confirm: `uv run ruff check tests/`\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove 3 unused deprecated imports in test_ascii_charts.py", "wid": "w3"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w2"], "notes": "textcharts added `subject: str | None = None` to all chart constructors.\nWhen set, the title is composed as f\"{subject} {default_title}\".\n\nExample: `subject=\"Query Latency\"` on Histogram \u2192 \"Query Latency Histogram\"\n\nCurrent BenchBox pattern:\n `ASCIIQueryHistogram(data=data, title=\"Query Latency Histogram\", ...)`\nCould become:\n `ASCIIQueryHistogram(data=data, subject=\"Query Latency\", ...)`\n\nEvaluate for each of the ~14 constructor calls in ascii_runtime.py whether\nsubject= would be cleaner. The benefit is avoiding title string duplication\nwith the chart type name. The risk is titles that don't compose well with\nthe generic default (e.g., subject=\"Performance\" + \"Bar Chart\" = \"Performance\nBar Chart\" - fine, but subject=\"Phase Breakdown by Platform\" + \"Stacked\nBreakdown\" = nonsense).\n\nDecision criteria:\n- Use subject= when the domain qualifier composes naturally with the generic\n chart type name (e.g., \"Query Latency\" + \"Histogram\" \u2192 good)\n- Keep explicit title= when the desired title doesn't match the pattern\n (e.g., \"Cost vs Performance\" doesn't compose with \"Scatter Plot\")\n- Keep explicit title= for dynamic titles (f-strings with platform names)\n\nDone criteria: Adopt subject= if 5+ of the ~14 call sites compose cleanly.\nIf fewer than 5 compose well, skip - the consistency benefit is too small\nto justify a mixed approach (some subject=, some title=). Document the\nevaluation result either way in a commit message or code comment.\n\nScope: ascii_runtime.py constructor calls only. Do NOT touch MCP dispatch\nor exporters.py - those pass title= from external input and must remain\nexplicit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Evaluate adopting subject= parameter for cleaner title composition", "wid": "w5"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w2", "w3"], "notes": "Final gate after w2 (factory renames) and w3 (dead import removal):\n1. `uv run ruff check benchbox/core/visualization/ tests/unit/visualization/` - lint clean\n2. `uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q` - zero deprecation warnings\nNote: full suite and golden parity already verified in w1 and w2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Final lint + deprecation warning gate", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "adopt value_formatter or lower_is_better without a concrete use case", "instead": "adopt when BenchBox adds throughput or Power@Size chart rendering", "why": "defaults are correct for latency metrics"}, {"dont": "change any explicit title= or metric_label= values in ascii_runtime.py", "instead": "(not specified)", "why": "they are already correct and the audit confirmed no reliance on defaults"}, {"dont": "remove deprecated factory names from the compatibility shim", "instead": "(not specified)", "why": "external callers using old BenchBox import paths need both old and new names available"}], "approach": "Low-risk mechanical migration. The audit confirms BenchBox passes all labels\nand titles explicitly - no call site relies on changed defaults. The work is:\n5 factory function renames, 3 dead import removals, 2 verifications, and an\noptional subject= adoption evaluation. Golden tests are the safety net.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "textcharts completed its full de-benchboxing initiative (5 change groups).\nAll changes are backward-compatible - old APIs still work but emit\nDeprecationWarning. BenchBox should update call sites to use the new APIs.\n\nAudit findings (2026-03-09):\n- All 14+ constructor calls in ascii_runtime.py and exporters.py pass\n title=, metric_label=, y_label= etc. explicitly - zero reliance on\n changed defaults. No output changes from Group 1.\n- 5 deprecated factory function calls in ascii_runtime.py need renaming.\n- 3 unused deprecated imports in test_ascii_charts.py need removal.\n- Compatibility shim (ascii/__init__.py) does `from textcharts import *`\n so new exports are automatically available.\n- DEFAULT_MAX_QUERIES is not referenced in BenchBox - no action needed.\n- lower_is_better defaults to True - matches BenchBox latency semantics.\n- subject= parameter available but optional - could simplify title passing.\n\nSupersedes: adapt-benchbox-to-textcharts-neutral-defaults (archived)", "id": "align-benchbox-with-textcharts-de-benchboxing", "preserves": ["All 15 golden snapshot tests pass unchanged - BenchBox passes explicit values everywhere", "BenchBox CLI and MCP chart output byte-identical to pre-migration", "Compatibility shim re-exports both old and new names for external callers", "Zero DeprecationWarning from textcharts during normal BenchBox operation"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "packages/textcharts/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_api.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii_runtime.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/contracts/visualization/"}, {"kind": "only_modify", "path_glob": "tests/fixtures/golden/ascii/"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/"}], "state": "done", "title": "Align BenchBox call sites with textcharts de-benchboxed API", "verifications": [{"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v", "description": "Golden snapshot parity maintained", "expected": "All 15 snapshot tests pass", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q", "description": "Zero DeprecationWarning from textcharts in visualization tests", "expected": "All tests pass without DeprecationWarning", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -q", "description": "Full test suite passes", "expected": "All tests pass", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run ruff check benchbox/core/visualization/ tests/unit/visualization/", "description": "Lint clean on modified files", "expected": "No errors", "item_id": "align-benchbox-with-textcharts-de-benchboxing", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": [], "notes": "1. Update textcharts version constraint in pyproject.toml\n2. Run: `uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v`\n3. All 15 golden tests should pass unchanged - BenchBox passes all labels\n and titles explicitly, so the default changes in Group 1 are invisible.\n4. If any fail, inspect: the only legitimate failures would be from test\n fixtures that construct charts without explicit labels/titles.\n Regenerate with `--update-golden` after visual inspection.\n5. Run full test suite: `uv run -- python -m pytest -q`\n6. Verify new parameter defaults match BenchBox semantics:\n - `lower_is_better: bool = True` on ASCIIComparisonBar, ASCIIDivergingBar,\n SummaryStats - default True correct for latency (lower = better).\n - `outlier_cap: float | None = None` on ASCIIChartOptions - None means\n auto P95\u00d72 capping, unchanged behavior.\n - Visual spot-check: comparison_bar and diverging_bar golden output\n shows correct green (improvement) / red (regression) colors.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Bump textcharts version and verify golden test parity", "wid": "w1"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w1"], "notes": "These are the ONLY deprecated factory calls in production code. Each is\nimported from a submodule inside a render function. Update both the import\nand the call site.\n\n| Line | Current import | New import | New call |\n|------|---------------|------------|----------|\n| 379 | `from ...percentile_ladder import from_query_results` | `from ...percentile_ladder import from_series` | `from_series(...)` |\n| 395 | `from ...normalized_speedup import from_normalized_results` | `from ...normalized_speedup import from_ratios` | `from_ratios(...)` |\n| 428 | `from ...sparkline_table import from_metrics` | `from ...sparkline_table import from_data` | `from_data(...)` |\n| 453 | `from ...cdf_chart import from_query_results` | `from ...cdf_chart import from_series` | `from_series(...)` |\n| 465 | `from ...rank_table import from_heatmap_data` | `from ...rank_table import from_matrix` | `from_matrix(...)` |\n\nNote: BenchBox imports from `benchbox.core.visualization.ascii.`,\nnot from `textcharts` directly. The compatibility shim re-exports both\nold and new names, so the new names are already importable.\n\nAfter changes, verify: `uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q`\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rename 5 deprecated factory function calls in ascii_runtime.py", "wid": "w2"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w1"], "notes": "Three deprecated factory functions are imported but never called in\n`tests/unit/visualization/ascii/test_ascii_charts.py`:\n\n- Line 13: `from_bar_data` - unused\n- Line 26: `from_normalized_results` - unused\n- Line 31: `from_query_results` - unused\n\nRemove these dead imports. Run lint to confirm: `uv run ruff check tests/`\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove 3 unused deprecated imports in test_ascii_charts.py", "wid": "w3"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w2"], "notes": "textcharts added `subject: str | None = None` to all chart constructors.\nWhen set, the title is composed as f\"{subject} {default_title}\".\n\nExample: `subject=\"Query Latency\"` on Histogram \u2192 \"Query Latency Histogram\"\n\nCurrent BenchBox pattern:\n `ASCIIQueryHistogram(data=data, title=\"Query Latency Histogram\", ...)`\nCould become:\n `ASCIIQueryHistogram(data=data, subject=\"Query Latency\", ...)`\n\nEvaluate for each of the ~14 constructor calls in ascii_runtime.py whether\nsubject= would be cleaner. The benefit is avoiding title string duplication\nwith the chart type name. The risk is titles that don't compose well with\nthe generic default (e.g., subject=\"Performance\" + \"Bar Chart\" = \"Performance\nBar Chart\" - fine, but subject=\"Phase Breakdown by Platform\" + \"Stacked\nBreakdown\" = nonsense).\n\nDecision criteria:\n- Use subject= when the domain qualifier composes naturally with the generic\n chart type name (e.g., \"Query Latency\" + \"Histogram\" \u2192 good)\n- Keep explicit title= when the desired title doesn't match the pattern\n (e.g., \"Cost vs Performance\" doesn't compose with \"Scatter Plot\")\n- Keep explicit title= for dynamic titles (f-strings with platform names)\n\nDone criteria: Adopt subject= if 5+ of the ~14 call sites compose cleanly.\nIf fewer than 5 compose well, skip - the consistency benefit is too small\nto justify a mixed approach (some subject=, some title=). Document the\nevaluation result either way in a commit message or code comment.\n\nScope: ascii_runtime.py constructor calls only. Do NOT touch MCP dispatch\nor exporters.py - those pass title= from external input and must remain\nexplicit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Evaluate adopting subject= parameter for cleaner title composition", "wid": "w5"}, {"evidence": null, "item_id": "align-benchbox-with-textcharts-de-benchboxing", "needs": ["w2", "w3"], "notes": "Final gate after w2 (factory renames) and w3 (dead import removal):\n1. `uv run ruff check benchbox/core/visualization/ tests/unit/visualization/` - lint clean\n2. `uv run -- python -W error::DeprecationWarning -m pytest tests/unit/visualization/ -q` - zero deprecation warnings\nNote: full suite and golden parity already verified in w1 and w2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Final lint + deprecation warning gate", "wid": "w6"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (Should-Fix Before Release)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Replaced the six-table CoffeeShop implementation with the reference three-table model, rebuilt the generator with deterministic weighting and order-line explosion, refreshed the query suite, and updated docs/CLI/examples accordingly.", "id": "align-coffeeshop-benchmark-with-reference-generator", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Align CoffeeShop Benchmark with Reference Generator", "verifications": [], "work": [], "worktree": "coffeeshop-parity"} -{"anti_patterns": [{"INSTEAD": "ensure connection configuration is fully isolated", "dont": "conflict with ClickHouse's PyMySQL usage", "why": "both platforms use PyMySQL on different ports"}, {"INSTEAD": "implement independently", "dont": "create a shared DorisStarRocksBase class", "why": "the projects have diverged and coupling creates fragile inheritance"}, {"INSTEAD": "create a new doris.py file", "dont": "modify existing adapter files", "why": "Doris is additive"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-17T00:00:00Z", "completed_pr": null, "created_at": "2026-01-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Integrate Apache Doris as a supported BenchBox platform. Apache Doris is an\nApache Top-Level Project providing a high-performance real-time analytical\ndatabase. It's the upstream project from which StarRocks was forked, with\ncontinued active development and strong adoption (12K+ GitHub stars, used by\nXiaomi, ByteDance, Baidu, JD.com, and 4000+ enterprises globally).\n\nTechnical fit analysis:\n- SQLGlot has official 'doris' dialect support (verified)\n- MySQL protocol connectivity via PyMySQL (port 9030)\n- Full TPC-H (22 queries) and TPC-DS (99 queries) compatibility\n- Competitive performance in OLAP workloads\n- Multiple data loading options (Stream Load, Broker Load, INSERT)\n- Vectorized execution engine and columnar storage\n\nStrategic value:\n- Major Apache project with strong community and enterprise adoption\n- Enables side-by-side comparison with StarRocks (shared heritage)\n- Real-time analytics focus complements batch OLAP platforms\n- Multiple managed cloud options: VeloDB Cloud, SelectDB Cloud, ApsaraDB for SelectDB\n\nNote: Apache Doris and StarRocks share common ancestry but have diverged\nsignificantly. Both are valuable benchmarking targets for different user bases.", "id": "apache-doris-olap-platform", "preserves": ["All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "ClickHouse adapter's PyMySQL usage unaffected by Doris's PyMySQL dependency", "Platform registry autodiscovery continues working for all existing platforms"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/starrocks.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/doris.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/doris.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_doris_adapter.py"}], "state": "done", "title": "Add Apache Doris MPP OLAP platform support", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.doris import DorisAdapter; print('import OK')\"", "description": "New adapter imports correctly", "expected": "import OK", "item_id": "apache-doris-olap-platform", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_doris_adapter.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "apache-doris-olap-platform", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "apache-doris-olap-platform", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create benchbox/platforms/doris.py adapter", "wid": "w1"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement schema creation with Doris-specific DDL", "wid": "w10"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support Duplicate/Aggregate/Unique Key table models", "wid": "w11"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DISTRIBUTED BY HASH clause generation", "wid": "w12"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support PARTITION BY RANGE for large tables", "wid": "w13"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register platform in platform_registry.py", "wid": "w14"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add pymysql to optional dependencies in pyproject.toml", "wid": "w15"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement query cache control", "wid": "w16"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Disable query cache via SET enable_query_cache = false", "wid": "w17"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add validation that cache is disabled", "wid": "w18"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement platform optimizations", "wid": "w19"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DorisAdapter class inheriting from PlatformAdapter", "wid": "w2"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w19"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Bloom filter index support for high-cardinality columns", "wid": "w20"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w19"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Bitmap index support for low-cardinality columns", "wid": "w21"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w19"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ANALYZE for statistics collection", "wid": "w22"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create tuning profile for Doris", "wid": "w23"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define TPC-H specific tuning (distribution keys, indexes)", "wid": "w24"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define TPC-DS specific tuning", "wid": "w25"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document colocate join configuration for multi-table joins", "wid": "w26"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create unit tests for Doris adapter", "wid": "w27"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w27"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test connection handling and configuration", "wid": "w28"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w27"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test SQL generation and dialect handling", "wid": "w29"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set _dialect = 'doris' for SQLGlot transpilation", "wid": "w3"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w27"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Stream Load request formatting", "wid": "w30"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create integration tests", "wid": "w31"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w31"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Docker Compose for local Doris cluster", "wid": "w32"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w31"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-H SF=0.01 end-to-end", "wid": "w33"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w31"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-DS SF=1 end-to-end", "wid": "w34"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create documentation", "wid": "w35"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w35"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs/platforms/doris.md with connection guide", "wid": "w36"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w35"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document Stream Load data loading process", "wid": "w37"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w35"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add deployment options (self-hosted, VeloDB Cloud, SelectDB Cloud, ApsaraDB)", "wid": "w38"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create example script", "wid": "w39"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement create_connection() using PyMySQL", "wid": "w4"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create examples/getting_started/sql/doris_tpch.py", "wid": "w40"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement execute_query() with timing and row count validation", "wid": "w5"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement data loading via Stream Load HTTP API", "wid": "w6"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create HTTP PUT to FE/BE node for CSV data loading", "wid": "w7"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle chunked loading for large files", "wid": "w8"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement load status checking and error handling", "wid": "w9"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "conflict with ClickHouse's PyMySQL usage", "instead": "ensure connection configuration is fully isolated", "why": "both platforms use PyMySQL on different ports"}, {"dont": "create a shared DorisStarRocksBase class", "instead": "implement independently", "why": "the projects have diverged and coupling creates fragile inheritance"}, {"dont": "modify existing adapter files", "instead": "create a new doris.py file", "why": "Doris is additive"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-17T00:00:00Z", "completed_pr": null, "created_at": "2026-01-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Integrate Apache Doris as a supported BenchBox platform. Apache Doris is an\nApache Top-Level Project providing a high-performance real-time analytical\ndatabase. It's the upstream project from which StarRocks was forked, with\ncontinued active development and strong adoption (12K+ GitHub stars, used by\nXiaomi, ByteDance, Baidu, JD.com, and 4000+ enterprises globally).\n\nTechnical fit analysis:\n- SQLGlot has official 'doris' dialect support (verified)\n- MySQL protocol connectivity via PyMySQL (port 9030)\n- Full TPC-H (22 queries) and TPC-DS (99 queries) compatibility\n- Competitive performance in OLAP workloads\n- Multiple data loading options (Stream Load, Broker Load, INSERT)\n- Vectorized execution engine and columnar storage\n\nStrategic value:\n- Major Apache project with strong community and enterprise adoption\n- Enables side-by-side comparison with StarRocks (shared heritage)\n- Real-time analytics focus complements batch OLAP platforms\n- Multiple managed cloud options: VeloDB Cloud, SelectDB Cloud, ApsaraDB for SelectDB\n\nNote: Apache Doris and StarRocks share common ancestry but have diverged\nsignificantly. Both are valuable benchmarking targets for different user bases.", "id": "apache-doris-olap-platform", "preserves": ["All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "ClickHouse adapter's PyMySQL usage unaffected by Doris's PyMySQL dependency", "Platform registry autodiscovery continues working for all existing platforms"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/starrocks.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/doris.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/doris.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_doris_adapter.py"}], "state": "done", "title": "Add Apache Doris MPP OLAP platform support", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.doris import DorisAdapter; print('import OK')\"", "description": "New adapter imports correctly", "expected": "import OK", "item_id": "apache-doris-olap-platform", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_doris_adapter.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "apache-doris-olap-platform", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "apache-doris-olap-platform", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create benchbox/platforms/doris.py adapter", "wid": "w1"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement schema creation with Doris-specific DDL", "wid": "w10"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support Duplicate/Aggregate/Unique Key table models", "wid": "w11"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DISTRIBUTED BY HASH clause generation", "wid": "w12"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support PARTITION BY RANGE for large tables", "wid": "w13"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register platform in platform_registry.py", "wid": "w14"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add pymysql to optional dependencies in pyproject.toml", "wid": "w15"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement query cache control", "wid": "w16"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Disable query cache via SET enable_query_cache = false", "wid": "w17"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add validation that cache is disabled", "wid": "w18"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement platform optimizations", "wid": "w19"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DorisAdapter class inheriting from PlatformAdapter", "wid": "w2"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w19"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Bloom filter index support for high-cardinality columns", "wid": "w20"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w19"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Bitmap index support for low-cardinality columns", "wid": "w21"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w19"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ANALYZE for statistics collection", "wid": "w22"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create tuning profile for Doris", "wid": "w23"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define TPC-H specific tuning (distribution keys, indexes)", "wid": "w24"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define TPC-DS specific tuning", "wid": "w25"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document colocate join configuration for multi-table joins", "wid": "w26"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create unit tests for Doris adapter", "wid": "w27"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w27"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test connection handling and configuration", "wid": "w28"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w27"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test SQL generation and dialect handling", "wid": "w29"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set _dialect = 'doris' for SQLGlot transpilation", "wid": "w3"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w27"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Stream Load request formatting", "wid": "w30"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create integration tests", "wid": "w31"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w31"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Docker Compose for local Doris cluster", "wid": "w32"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w31"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-H SF=0.01 end-to-end", "wid": "w33"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w31"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-DS SF=1 end-to-end", "wid": "w34"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create documentation", "wid": "w35"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w35"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs/platforms/doris.md with connection guide", "wid": "w36"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w35"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document Stream Load data loading process", "wid": "w37"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w35"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add deployment options (self-hosted, VeloDB Cloud, SelectDB Cloud, ApsaraDB)", "wid": "w38"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create example script", "wid": "w39"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement create_connection() using PyMySQL", "wid": "w4"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create examples/getting_started/sql/doris_tpch.py", "wid": "w40"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement execute_query() with timing and row count validation", "wid": "w5"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement data loading via Stream Load HTTP API", "wid": "w6"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create HTTP PUT to FE/BE node for CSV data loading", "wid": "w7"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle chunked loading for large files", "wid": "w8"}, {"evidence": null, "item_id": "apache-doris-olap-platform", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement load status checking and error handling", "wid": "w9"}], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Implement Apache Spark adapter for the most widely deployed distributed\nSQL engine (35K+ GitHub stars, industry standard).", "id": "apache-spark-platform-adapter", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Apache Spark Platform Adapter", "verifications": [], "work": [], "worktree": "platform-expansion-spark"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "commit raw stdout logs, benchmark_runs result bundles, generated TBL/Parquet data, or screenshot/chart batches; summarize durable facts in docs instead.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "fix SQL translation, platform adapters, catalog operations, or packaging defects in this survey; file separate TODOs for those implementation gaps.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "include sketch_df_hll_persist_merge or sketch_df_topk_persist_merge in SQL-mode cloud runs; they are DataFrame aggregate-state operations.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "run live cloud benchmarks, upload cloud data, or generate SF=10 data until w2 records explicit approval, max runtime, log path, output root, disk threshold, and stop conditions.", "why": "(unstated)"}], "approach": "Run this as a two-stage survey: cheap platform/driver/dry-run validation\nfirst, then approved live cloud execution. Use only SQL-mode query IDs in\nthe cloud SQL survey; keep DataFrame aggregate-state operations for the\ndeferred DataFrame survey. Use BENCHBOX_OUTPUT_DIR=$HOME/Developer/benchmark_runs\nfor generated data, result bundles, analysis artifacts, and long-run logs.\nCommit durable markdown summaries and follow-up TODOs only; keep raw logs,\ncharts, and result JSONs outside git.\n", "blocked_reason": null, "category": "Benchmark Development", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-25T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 1, "reason": "SQL mode establishes the baseline; DataFrame mode includes the sketch_df_* aggregate-state operations and has known exact-fallback caveats on Polars/Dask that make cross-engine comparison less clean. Separate survey.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: DataFrame-mode approx survey is explicitly sequenced after the SQL-mode survey; parent approx-analytics-cloud-survey is itself still open/planning, not yet actionable.", "summary": "DataFrame mode survey (Polars, PySpark, DataFusion) for approx queries"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 2, "reason": "Power phase (single session) is sufficient for capability verification and latency measurement. Throughput adds cloud cost and complexity; defer to a dedicated concurrency study.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'dedicated concurrency study' with no scope/owner defined; parent item itself still open/planning, not ripe to track separately.", "summary": "Throughput phase benchmarking (concurrent sessions)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 3, "reason": "BenchBox measures latency, not approximation error. Accuracy belongs in a separate accuracy study with known-answer datasets.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly outside BenchBox's latency-only scope; accuracy studies are deferred permanently to a separate project, not a pending TODO.", "summary": "Error-rate analysis (actual approximation accuracy vs exact)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 4, "reason": "pyathen/pyodbc drivers and IAM setup are significant additional overhead. Start with the 8 primary platforms; add these if they surface unique sketch behavior.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Parent survey item approx-analytics-cloud-survey is still open (planning); adding Athena/Synapse/Fabric is a natural extension of that tracked item.", "summary": "Amazon Athena SQL, Azure Synapse, Fabric Warehouse"}], "deps": [], "description": "Survey approximate analytics support and performance across all major cloud SQL\nplatforms by running the read_primitives (one-shot approx aggregates) and\nwrite_primitives SQL sketch category (persist + merge + requery) benchmarks at three\nscale factors (SF=0.1, SF=1, SF=10).\n\nThe goal is a complete support matrix \u2014 which platform supports which sketch family,\nat what latency, and how latency scales with data volume \u2014 plus identification of\nany gaps or regressions not yet captured in the cross-engine docs.\n\nTarget platforms (SQL mode):\n- Databricks SQL (enabled, credentials available)\n- ClickHouse Cloud (enabled)\n- Firebolt Cloud (enabled)\n- Starburst/Trino Cloud (enabled)\n- MotherDuck (enabled)\n- Snowflake (driver install required)\n- Amazon Redshift (driver install required)\n- Google BigQuery (driver install required)\n\nread_primitives queries in scope (5):\n approx_count_distinct_simple, approx_count_distinct_groupby,\n approx_quantile_groupby, approx_quantiles_array, approx_top_k_lineitem\n\nSQL write_primitives sketch ops in scope (22 sketch_* operations):\n sketch_ddl_create_persistent_table through sketch_query_topk_combine_lgmm10.\n Exclude sketch_df_hll_persist_merge and sketch_df_topk_persist_merge because\n those are DataFrame aggregate-state operations, not SQL cloud-platform operations.\n\nKnown platform limitations (from docs \u2014 verify, don't assume still current):\n- Redshift: HLL substitution only; KLL and Top-K skipped (no native equivalent)\n- BigQuery: Theta substituted with HLL; Top-K skipped (no native accumulator)\n- DataFusion: sketches skipped entirely\n- DuckDB (not in scope here): 4 theta/topk ops blocked by extension `2e38607`\n", "id": "approx-analytics-cloud-survey", "preserves": ["Existing result bundles in benchmark_runs/results/ and $BENCHBOX_OUTPUT_DIR/results/ \u2014 never overwrite or delete", "benchmark_runs/datagen/tpch_sf001/ and $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf001/ \u2014 existing SF=0.01 data; do not force-regenerate", "docs/benchmarks/read-primitives-approximate-functions.md and write-primitives-sketch-functions.md \u2014 update only, never truncate"], "prior_art": [{"concept": "benchbox/cli/commands/run.py \u2014 canonical cloud platform keys include clickhouse-cloud and firebolt:cloud; reuse exact keys", "decision": "reuse", "path": "benchbox/cli/commands/run.py"}, {"concept": "benchbox/core/read_primitives/catalog/queries.yaml \u2014 5 approx query IDs already defined; no new catalog work needed", "decision": "reuse", "path": "benchbox/core/read_primitives/catalog/queries.yaml"}, {"concept": "benchbox/core/write_primitives/catalog/operations.yaml \u2014 22 SQL sketch op IDs plus 2 DataFrame sketch ops already defined; reuse only SQL ops here", "decision": "reuse", "path": "benchbox/core/write_primitives/catalog/operations.yaml"}, {"concept": "benchmark_runs/datagen/tpch_sf001/ \u2014 SF=0.01 datagen exists; extend, don't regenerate", "decision": "extend", "path": "benchmark_runs/datagen/tpch_sf001/"}, {"concept": "docs/benchmarks/read-primitives-approximate-functions.md \u2014 cross-engine function reference; reuse as ground truth for expected dialect rewrites", "decision": "reuse", "path": "docs/benchmarks/read-primitives-approximate-functions.md"}, {"concept": "docs/benchmarks/write-primitives-sketch-functions.md \u2014 sketch family \u00d7 engine support matrix; reuse and extend with Firebolt + Starburst findings", "decision": "extend", "path": "docs/benchmarks/write-primitives-sketch-functions.md"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/platform-expansion/planning/"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/read-primitives-approximate-functions.md"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/write-primitives-sketch-functions.md"}], "state": "planning", "title": "Approximate Analytics Cloud SQL Survey \u2014 read_primitives + SQL write_primitives at SF=0.1/1/10", "verifications": [{"command": "uv run -- python - <<'PY'\nfrom pathlib import Path\nimport yaml\n\nitem = yaml.safe_load(Path(\"_project/TODO/benchmark-expansion/planning/approx-analytics-cloud-survey.yaml\").read_text())\nwork_notes = \"\\n\".join(str(unit.get(\"notes\", \"\")) for unit in item[\"work\"])\nassert \"sketch_df_hll_persist_merge\" not in work_notes\nassert \"sketch_df_topk_persist_merge\" not in work_notes\nprint(\"sql-only sketch scope verified\")\nPY\n", "description": "TODO query scope stays SQL-only and excludes DataFrame aggregate-state sketch ops", "expected": "sql-only sketch scope verified", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 1}, {"command": "BENCHBOX_OUTPUT_DIR=\"${BENCHBOX_OUTPUT_DIR:-$HOME/Developer/benchmark_runs}\" uv run -- python - <<'PY'\nimport json\nimport os\nfrom pathlib import Path\n\nroot = Path(os.environ[\"BENCHBOX_OUTPUT_DIR\"]) / \"datagen\"\nexpected = {\"sf01\": 0.1, \"sf1\": 1.0, \"sf10\": 10.0}\nfor sf, value in expected.items():\n path = root / f\"tpch_{sf}\" / \"_datagen_manifest.json\"\n manifest = json.loads(path.read_text())\n actual = float(manifest[\"scale_factor\"])\n assert actual == value, f\"{path}: expected {value}, got {actual}\"\n print(f\"tpch_{sf}: scale_factor={actual}\")\nPY\n", "description": "SF=0.1, SF=1, SF=10 datagen manifests exist under BENCHBOX_OUTPUT_DIR with correct scale_factor", "expected": "tpch_sf01: scale_factor=0.1\ntpch_sf1: scale_factor=1.0\ntpch_sf10: scale_factor=10.0\n", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 2}, {"command": "BENCHBOX_OUTPUT_DIR=\"${BENCHBOX_OUTPUT_DIR:-$HOME/Developer/benchmark_runs}\" uv run -- python - <<'PY'\nimport json\nimport os\nfrom pathlib import Path\n\nroot = Path(os.environ[\"BENCHBOX_OUTPUT_DIR\"]) / \"results\"\nplatforms = {\"databricks\", \"clickhouse-cloud\", \"firebolt\", \"starburst\", \"motherduck\", \"snowflake\", \"bigquery\", \"redshift\"}\nbenchmarks = {\"read_primitives\", \"write_primitives\"}\naliases = {\n \"amazon-redshift\": \"redshift\",\n \"clickhouse-cloud\": \"clickhouse-cloud\",\n \"databricks-sql\": \"databricks\",\n \"google-bigquery\": \"bigquery\",\n}\nfound: set[tuple[str, str]] = set()\nfor path in root.glob(\"*.json\"):\n data = json.loads(path.read_text())\n benchmark = data.get(\"benchmark\", {})\n benchmark_id = str(benchmark.get(\"id\", \"\"))\n if benchmark_id not in benchmarks or float(benchmark.get(\"scale_factor\", -1)) != 1.0:\n continue\n platform_name = str(data.get(\"platform\", {}).get(\"name\", \"\")).lower().replace(\" \", \"-\")\n platform_key = aliases.get(platform_name, platform_name)\n filename = path.name.replace(\"_\", \"-\").lower()\n for expected in platforms:\n if platform_key == expected or expected in filename:\n queries = data.get(\"queries\", [])\n assert queries, f\"{path}: missing query statuses\"\n assert all(\"status\" in query for query in queries), f\"{path}: query without status\"\n found.add((expected, benchmark_id))\nmissing = sorted((platform, benchmark) for platform in platforms for benchmark in benchmarks if (platform, benchmark) not in found)\nassert not missing, f\"missing result bundles: {missing}\"\nprint(f\"verified {len(found)} platform/benchmark SF=1 result bundles\")\nPY\n", "description": "SF=1 result bundles have structured query statuses for each target platform and both SQL benchmarks", "expected": "verified 16 platform/benchmark SF=1 result bundles", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python - <<'PY'\nfrom pathlib import Path\n\ndocs = [\n Path(\"docs/benchmarks/write-primitives-sketch-functions.md\"),\n Path(\"docs/benchmarks/read-primitives-approximate-functions.md\"),\n]\nfor path in docs:\n text = path.read_text()\n for platform in (\"Firebolt\", \"Starburst\"):\n assert f\"| {platform}\" in text or f\"| `{platform}`\" in text, f\"{path}: missing {platform} table row\"\nprint(\"Firebolt and Starburst rows present in approximate analytics docs\")\nPY\n", "description": "Reference docs contain explicit Firebolt and Starburst rows after the survey", "expected": "Firebolt and Starburst rows present in approximate analytics docs", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": [], "notes": "Use exact platform keys for this survey:\n databricks, clickhouse-cloud, firebolt:cloud, starburst, motherduck,\n snowflake, bigquery, redshift\nDependency-check keys differ for deployment aliases:\n firebolt:cloud -> firebolt\n motherduck -> verify via platforms list/status (no dedicated check-deps key)\n\nRun:\n uv run -- benchbox platforms list\n uv run -- benchbox check-deps --platform \n\nFor each target platform, record a compact evidence summary for the final\ndocs update:\n - checked BenchBox SHA\n - command(s) run\n - platform status (enabled / missing driver / missing credentials / blocked)\n - driver package and version when available\n - credential signal checked, with secrets redacted\n\nKeep raw stdout under /tmp or $BENCHBOX_OUTPUT_DIR/logs/approx-survey-*/\nonly. Do not commit raw logs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify platform keys, driver availability, credentials, and evidence freshness", "wid": "w0"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w0"], "notes": "For each missing platform identified in w0, prefer existing optional extras:\n uv sync --extra snowflake\n uv sync --extra bigquery\n uv sync --extra redshift\n\nIf the local environment still needs one-off tooling, use uv-managed\ncommands only; never pip install. Do not commit pyproject.toml or uv.lock\nchanges for driver activation in this survey. If an optional extra is\nmissing or incorrect, file a separate follow-up TODO instead of fixing\npackaging here.\n\nRe-run:\n uv run -- benchbox platforms list\n uv run -- benchbox check-deps --platform \n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Install or activate missing optional cloud drivers without widening project scope", "wid": "w1"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w1", "w2", "w3", "w4"], "notes": "Platform: snowflake. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nUse --table-mode native unless the approved run envelope chooses a different\nmode. Full DataSketches binary support is expected; verify actual behavior\nfrom query statuses and platform errors.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Snowflake at SF=0.1, SF=1, SF=10", "wid": "w10"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w1", "w2", "w3", "w4"], "notes": "Platform: bigquery. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nCloud storage URI: gs:///benchbox or approved BigQuery native load.\nExpected substitutions: Theta to HLL_COUNT.INIT/MERGE; KLL through\nKLL_QUANTILES; Top-K skipped. Verify instead of assuming.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Google BigQuery at SF=0.1, SF=1, SF=10", "wid": "w11"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w1", "w2", "w3", "w4"], "notes": "Platform: redshift. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nCloud storage URI: s3:///benchbox with IAM COPY permissions.\nExpected substitutions: Theta to HLLSKETCH/HLL_CREATE_SKETCH; KLL and\nTop-K skipped. Verify instead of assuming.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Amazon Redshift at SF=0.1, SF=1, SF=10", "wid": "w12"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w10", "w11", "w12", "w5", "w6", "w7", "w8", "w9"], "notes": "For each platform x benchmark x query x scale factor, extract from result bundles:\n - Pass / Fail / Skipped status\n - execution_time_seconds, converted to ms only for display\n - Sketch family mapping actually used (native / substitution / skipped / failed)\n - Result bundle path and checked BenchBox SHA\n\nProduce a compact summary, not raw stdout:\n 1. Support matrix table: platforms x sketch families\n 2. Performance comparison: SF=1 latency across platforms per sketch op\n 3. Scale analysis: SF=0.1/1/10 per platform, highlighting O(1) sketch\n merge behavior versus O(n) exact/substituted fallback\n\nUse:\n uv run -- benchbox compare ...\n uv run -- benchbox visualize --chart-type performance_bar\n\nSave generated charts/raw analysis under $BENCHBOX_OUTPUT_DIR or /tmp.\nCommit only durable markdown summaries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Compile cross-platform support matrix and performance comparison", "wid": "w13"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w13"], "notes": "Update docs/benchmarks/write-primitives-sketch-functions.md with:\n - Firebolt and Starburst rows in the sketch family x engine support matrix\n - MotherDuck extension version observation\n - Current compact evidence summary: commands, checked SHA, driver versions,\n PASS/FAIL/SKIPPED counts, and result-bundle identifiers\n - Any changed platform behavior versus the pre-survey docs\n\nUpdate docs/benchmarks/read-primitives-approximate-functions.md with:\n - Firebolt and Starburst rows in the function reference tables\n - Current compact evidence summary for read_primitives query support\n\nFor each unsupported or broken sketch family that requires code or adapter\nwork, file a separate follow-up TODO under platform-expansion/planning/.\nThis survey commit may edit docs and follow-up TODO YAML only; no runtime\ncode or catalog changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Document findings and file focused platform-gap TODOs", "wid": "w14"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w0"], "notes": "Before any live cloud benchmark, cloud upload, or SF=10 generation, present\nthe operator with:\n - target platforms and exact commands\n - estimated data volume and cloud services touched\n - max runtime per platform and overall stop condition\n - cost/budget ceiling, if one applies\n - output root and log root\n - disk-free threshold; minimum 25 GiB because SF=10 TPC-H data is large\n\nRequire an explicit approval reply in the implementation thread before\ncontinuing. Use:\n export BENCHBOX_OUTPUT_DIR=\"${BENCHBOX_OUTPUT_DIR:-$HOME/Developer/benchmark_runs}\"\n export APPROX_SURVEY_LOG_DIR=\"$BENCHBOX_OUTPUT_DIR/logs/approx-survey-\"\n\nLive runs are sequential by platform. Stop new work if disk space drops\nbelow the approved threshold, if cloud credentials are invalid, or if a\ncommand exceeds the approved runtime cap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Get explicit live-run approval and define the run envelope", "wid": "w2"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w0", "w1"], "notes": "Use dry-run to catch query selection, dialect, and config errors before\ngenerating SF=10 data or touching live warehouses.\n\nRead query set:\n approx_count_distinct_simple,approx_count_distinct_groupby,approx_quantile_groupby,approx_quantiles_array,approx_top_k_lineitem\n\nSQL sketch query set (22 operations; intentionally excludes sketch_df_*):\n sketch_ddl_create_persistent_table,sketch_insert_theta_per_partition,sketch_insert_kll_per_partition,sketch_insert_topk_per_shard,sketch_query_theta_union_merge,sketch_query_kll_quantiles_merge,sketch_query_topk_combine,sketch_drop_persistent_table,sketch_cpc_create_persistent_table,sketch_cpc_insert_per_partition,sketch_cpc_query_union_merge,sketch_cpc_drop_persistent_table,sketch_req_create_persistent_table,sketch_req_insert_per_partition,sketch_req_query_quantile_merge,sketch_req_drop_persistent_table,sketch_query_theta_union_merge_lgk10,sketch_query_theta_union_merge_lgk14,sketch_query_kll_quantiles_merge_k100,sketch_query_kll_quantiles_merge_k1000,sketch_query_topk_combine_lgmm8,sketch_query_topk_combine_lgmm10\n\nFor each platform:\n uv run -- benchbox run --platform --benchmark read_primitives --scale 0.1 \\\n --phases load,power \\\n --queries \\\n --non-interactive \\\n --dry-run /tmp/approx-survey-dryrun/-read\n uv run -- benchbox run --platform --benchmark write_primitives --scale 0.1 \\\n --phases load,power \\\n --queries \\\n --non-interactive \\\n --dry-run /tmp/approx-survey-dryrun/-write\n\nFix no code here. If dry-run exposes a translation or adapter defect,\nfile a separate TODO and mark that platform blocked for this survey.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Dry-run all 8 platforms x 2 SQL benchmarks before data generation or live runs", "wid": "w3"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3"], "notes": "Use the approved BENCHBOX_OUTPUT_DIR from w2. Run sequentially:\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark read_primitives --scale 0.1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark write_primitives --scale 0.1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark read_primitives --scale 1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark write_primitives --scale 1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark read_primitives --scale 10\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark write_primitives --scale 10\n\nVerify manifests under:\n $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf01/_datagen_manifest.json\n $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf1/_datagen_manifest.json\n $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf10/_datagen_manifest.json\n\nEstimated disk: ~200MB (SF=0.1), ~2GB (SF=1), ~20GB (SF=10)\nuncompressed TBL. Announce the SF=10 command, max runtime, log path, and\nstop condition immediately before launch.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Generate TPC-H data at SF=0.1, SF=1, SF=10 in the shared UAT output root", "wid": "w4"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: databricks. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nOutput/staging: dbfs:/Volumes////benchbox\nRecord result bundle paths and query status counts for w13.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Databricks SQL at SF=0.1, SF=1, SF=10", "wid": "w5"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: clickhouse-cloud. Use phases load,power, --non-interactive, the\nread query set, and the 22-op SQL sketch query set from w3.\nClickHouse uses native -State/-Merge combinators for sketches, not\nDataSketches binary portability. Record result bundle paths and query\nstatus counts for w13.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: ClickHouse Cloud at SF=0.1, SF=1, SF=10", "wid": "w6"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: firebolt:cloud. Use phases load,power, --non-interactive, the\nread query set, and the 22-op SQL sketch query set from w3.\nFirebolt approximate-analytics coverage is not yet in the cross-engine\nreference docs. Record which sketch ops succeed, fail, or skip and which\ndialect rewrites Firebolt applies.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Firebolt Cloud at SF=0.1, SF=1, SF=10", "wid": "w7"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: starburst. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nTrino's one-shot approx functions include approx_distinct, approx_percentile,\nand approx_most_frequent. Record whether persisted sketch operations have a\nnative, substituted, skipped, or failed path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Starburst/Trino Cloud at SF=0.1, SF=1, SF=10", "wid": "w8"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: motherduck. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nVerify the DuckDB extension version MotherDuck exposes before assuming the\nlocal DuckDB extension limitation at `2e38607` still applies.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: MotherDuck at SF=0.1, SF=1, SF=10", "wid": "w9"}], "worktree": "benchmark-expansion"} -{"anti_patterns": [{"INSTEAD": "classify them first, then decide deprecation separately.", "dont": "collapse wrappers into internals just because they duplicate CLI reachability", "why": "tests assert wrapper facades as public behavior"}, {"INSTEAD": "generate docs from registry metadata or fail the drift check.", "dont": "generate platform counts from docs", "why": "docs are the drifting surface"}, {"INSTEAD": "use the finite support_status vocabulary.", "dont": "introduce vague tiers like supported-ish", "why": "ambiguous tiers recreate the same blind spot"}], "approach": "Treat this as a taxonomy and governance change first. Start from existing\ndocumented commitments, then reconcile code evidence against those\ncommitments. Prefer a small, enforceable contract map over aspirational prose.\n", "blocked_reason": null, "category": "Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "architecture-contract-map-and-support-taxonomy", "id": 340, "reason": "Completed by platform-support-status-and-extension-health after the taxonomy was accepted.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Reason states it was already completed by platform-support-status-and-extension-health; that item is not in open_items (done).", "summary": "Fully migrate every platform registry entry to support_status"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "architecture-contract-map-and-support-taxonomy", "id": 341, "reason": "Belongs in benchmark-support-status-and-discovery-policy; platform migration did not cover benchmark metadata.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Target item benchmark-support-status-and-discovery-policy (DONE) states support_status was added to every BENCHMARK_METADATA entry; migration already completed.", "summary": "Fully migrate every benchmark registry entry to support_status"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "architecture-contract-map-and-support-taxonomy", "id": 342, "reason": "Do after this map lands and downstream TODOs resolve their product-surface decisions.", "resolution": "promoted", "resolved_item": "rewrite-architecture-md-around-the-new-contract-map", "resolved_reason": null, "summary": "Rewrite architecture.md around the new contract map"}], "deps": [], "description": "BenchBox has several real product contracts that are only partially declared:\nCLI behavior, Python wrappers, BaseBenchmark.run_with_platform, adapter\nsubclassing, MCP tools, result JSON, explorer input, generated compatibility\ndocs, platform availability, and experimental modules. The architecture is\nstill sound, but future work is making contract decisions implicitly.\n\nThis item creates one authoritative map of public/stable, beta-public,\ninternal, experimental, deprecated, generated, and repo-only surfaces. It also\ndefines support status vocabulary for platforms and benchmarks so docs,\nregistry metadata, packaging, MCP, and PR review use the same language.", "id": "architecture-contract-map-and-support-taxonomy", "preserves": ["Do not change runtime behavior except for narrowly scoped docs/checklist validation needed by this TODO.", "Do not downgrade existing public wrapper tests or imply wrappers are unsupported without an explicit deprecation decision.", "Do not mark experimental code as stable merely because it is packaged."], "prior_art": [{"concept": "Makefile: compat docs generation targets -- reuse generated-docs checks where feasible.", "decision": "reuse", "path": "Makefile"}, {"concept": "docs/reference/backward-compatibility.md: existing shim/deprecation registry -- extend, do not duplicate.", "decision": "extend", "path": "docs/reference/backward-compatibility.md"}, {"concept": "docs/reference/hosted-results-contract.md: existing product-contract style -- reuse the crisp contract language.", "decision": "reuse", "path": "docs/reference/hosted-results-contract.md"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": ".github/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "README.md"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/design/future-state/"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}], "state": "done", "title": "Architecture hardening: create the public contract map and support taxonomy", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate", "description": "Validate TODO graph and schema after adding the contract-map item.", "expected": "Validation passes or reports only pre-existing unrelated TODO issues.", "item_id": "architecture-contract-map-and-support-taxonomy", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/core/test_platform_registry.py tests/unit/core/results/test_schema_policy.py tests/unit/mcp/test_run_surface_contract.py --tb=short", "description": "Run targeted docs checks added by this work.", "expected": "Contract, count, support-status, schema-policy, and MCP run-surface checks pass or skip only for missing optional MCP SDK.", "item_id": "architecture-contract-map-and-support-taxonomy", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": [], "notes": "Capture a compact committed summary of checked SHA, inspected files, and\nany line-number drift. Do not commit raw command transcripts.\n\nMinimum evidence to re-check:\n - README support disclaimer and platform counts: README.md:35-48.\n - Backward-compat registry: docs/reference/backward-compatibility.md:24-84.\n - Wrapper facade tests: tests/unit/test_wrapper_facades_fast.py.\n - MCP public API decision: _project/DONE/mcp-integration/active/refactor-mcp-use-public-api.yaml.\n - Existing future-state architecture docs under docs/design/future-state/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate contract-surface evidence against the current branch before editing", "wid": "w0"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w0"], "notes": "The table must cover at least: CLI commands/options, Python wrapper\nfacades, benchbox.base.BaseBenchmark, BaseBenchmark.run_with_platform,\ncore.base_benchmark, adapter subclassing hooks, platform registry\nmetadata, MCP tools, result JSON bundles, explorer read model,\nsubmission validator behavior, sql_compat rule catalog, generated docs,\nbenchbox.experimental, _project scripts, and TODO/ADR docs.\n\nRequired columns: surface, current tier, owner, compatibility promise,\ndeprecation path, verification gate, and source of truth.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs/reference/public-contracts.md with a complete surface classification table", "wid": "w1"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w1"], "notes": "Define allowed statuses: stable, beta, experimental, repo_only,\ndeprecated, document_only. Record what each status means for packaging,\ndocs, registry visibility, MCP exposure, CI coverage, and breakage policy.\n\nThis work unit defines the vocabulary and contract. It may add minimal\nschema/constants only if that avoids duplicating strings in follow-up\nitems. Full registry migration belongs to the platform-extension TODO.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a support_status taxonomy for platforms and benchmarks", "wid": "w2"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w1"], "notes": "Add or update the project PR checklist so any new public, beta-public,\ndeprecated, generated, or experimental surface must update the contract\nmap or explicitly state why no contract-map change is needed.\n\nThe checklist must mention result schema, MCP params, platform support\nstatus, wrapper facades, adapter subclassing hooks, and generated docs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a PR checklist gate for public/deprecated surface changes", "wid": "w3"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w1"], "notes": "Promote active architecture decisions from scattered TODOs and\nfuture-state docs into one index. Include status, owner, blocking gates,\nand the current implementation surface.\n\nAt minimum index:\n - docs/design/future-state/formalize-mcp-internal-apis/README.md.\n - docs/design/future-state/isolate-experimental-core-subsystems/README.md.\n - docs/reference/backward-compatibility.md entries for wrappers and core.base_benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create an ADR/contract index for scattered future-state decisions", "wid": "w4"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w2"], "notes": "Add generated or tested checks for platform counts, benchmark counts,\nsupport-status tables, result-schema version claims, and adapter\navailability statements. The check should fail on contradictory claims\nlike docs saying a platform is both unavailable and supported.\n\nPrefer generating tables from registry metadata where possible. If a\nsection is intentionally editorial, mark it as non-authoritative in the\npublic contract map.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add docs drift checks for current-count and support-claim statements", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "CDF computation is trivial with sorted arrays", "dont": "add numpy/scipy dependency", "why": "the visualization module has zero external deps"}, {"INSTEAD": "create a dedicated class that borrows rendering patterns", "dont": "reuse ASCIILineChart directly", "why": "CDF has fixed 0-100% Y-axis and different data preparation (sorted cumulative)"}], "approach": "Reuse the grid-based rendering pattern from benchbox/core/visualization/ascii/line_chart.py\n(grid creation, Bresenham-like line drawing, axis rendering) but with CDF-specific logic:\n\n1. Create CDFSeriesData (name, values list[float]) dataclass.\n2. Create ASCIICDFChart(ASCIIChartBase) with render().\n3. For each series: sort values, compute cumulative proportions (i/n for each value).\n4. Plot on a grid where X = execution time, Y = cumulative proportion (0-100%).\n5. Reuse the marker system from ASCIILineChart (*, +, o, x, ^, v, #, @) for multi-series.\n6. Add percentile annotations: vertical dashed lines at P50, P95, P99 positions.\n7. Add from_query_results() factory.\n\nKeep Y-axis fixed at 0-100% (no auto-scaling). X-axis scales to data range.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that renders a cumulative distribution function (CDF) plot\nshowing the proportion of queries completing within a given time threshold. The X-axis\nis execution time, the Y-axis is the cumulative proportion (0% to 100%), and each\nplatform is a separate line/curve.\n\nCDF plots are the standard academic and industry method for visualizing latency\ndistributions, used extensively in TailBench, YCSB, and database vendor whitepapers.\nThey answer: \"what percentage of my queries finish within T milliseconds?\" which maps\ndirectly to SLA compliance analysis.\n\nExample output:\n```\nCumulative Distribution of Query Latency\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n100%\u2502 ****+++++++++++\n \u2502 ****++++\n 75%\u2502 ****+++\n \u2502 ***++++\n 50%\u2502 **+++\n \u2502 *+++\n 25%\u2502 *++\n \u2502 *+\n 0%\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n 0 50 100 150 200 300 ms\n * DuckDB + Polars\n```\n\nSteeper curves indicate more consistent performance. Curves that rise quickly and\nplateau show tight latency distributions; long tails that slowly reach 100% reveal\nproblematic outlier queries.\n\nThis is distinct from the Box Plot (quartile summary statistics that collapse\ndistribution shape), the Histogram (per-query latency, not cumulative), and the\nLine Chart (time-series trends over runs, not distribution functions). CDF reveals\nbimodal distributions, heavy tails, and performance cliffs that quartile summaries hide.", "id": "ascii-chart-cdf", "preserves": ["ASCIILineChart behavior unchanged (CDF is a distribution chart, line chart is time-series)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII CDF chart (cumulative distribution of query latency)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.cdf_chart import ASCIICDFChart, CDFSeriesData; data = [CDFSeriesData('DuckDB', [10, 20, 30, 50, 120]), CDFSeriesData('Polars', [15, 25, 45, 80, 310])]; chart = ASCIICDFChart(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with two CDF curves, Y-axis 0-100%", "item_id": "ascii-chart-cdf", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-cdf", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-cdf", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create CDFSeriesData dataclass and ASCIICDFChart class with grid-based render()", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement CDF computation (sort values, compute cumulative proportions) and grid plotting", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add optional percentile annotation lines (P50, P95, P99 vertical markers on X-axis)", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_query_results() factory for creating CDF from NormalizedResult data", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'cdf_chart' case to render_ascii_chart() in exporters.py", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (single series, multi-series, identical values, single value, large dataset)", "wid": "w6"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cdf_chart to flagship template in templates.py", "wid": "w7"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "implement as a new class", "dont": "modify ASCIIBarChart or ASCIIDivergingBar", "why": "they serve different purposes (absolute values, pct change)"}, {"INSTEAD": "use log2 scaling so 2x and 0.5x are symmetric around the baseline", "dont": "use linear scaling for bar widths", "why": "ratios can span 0.01x to 100x"}], "approach": "Follow the pattern in benchbox/core/visualization/ascii/diverging_bar.py for the\ncentered-zero layout with bars extending left and right from a reference line:\n\n1. Create SpeedupData dataclass (name, ratio, is_baseline).\n2. Create ASCIINormalizedSpeedup(ASCIIChartBase) with a render() method.\n3. Draw a vertical baseline reference line at 1.0x in the center.\n4. Ratios > 1.0 extend right (faster, green), ratios < 1.0 extend left (slower, red).\n5. Add from_normalized_results() factory that computes ratios given results + baseline name.\n6. Use logarithmic scaling for the bar width to handle large ratio ranges (e.g., 0.1x to 50x).\n\nThe baseline platform selection should accept either a platform name or \"slowest\"/\"fastest\"\nas convenience aliases.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that shows platform performance as a ratio relative to a\nuser-selected baseline platform (e.g., \"DuckDB is 3.2x faster than SQLite\"). The\nbaseline is rendered at 1.0x with a vertical reference line. Platforms faster than\nbaseline extend right, slower ones extend left.\n\nThis is the standard way TPC and SPEC results are formally reported. When comparing\nplatforms that span orders of magnitude (50ms to 50,000ms), absolute bar charts make\nfast platforms invisible slivers. Normalized ratios make relative performance\nimmediately clear regardless of absolute magnitude.\n\nExample output:\n```\nNormalized Performance (baseline: SQLite = 1.0x)\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n Slower \u2502 Faster\nSQLite \u2502\u2588 1.0x\nPandas \u2588\u2588\u2588\u2588\u2588\u2588\u2502 0.4x\nDuckDB \u2502\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 8.2x\nPolars \u2502\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 7.5x\nClickHouse \u2502\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 12.1x\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n```\n\nThis is distinct from the existing bar chart (absolute values) and the diverging bar\nchart (percentage change between two runs of the same platform). The normalized speedup\nchart shows cross-platform comparison ratios with a centered baseline reference.", "id": "ascii-chart-normalized-speedup", "preserves": ["ASCIIBarChart behavior unchanged (this is a new chart type, not a modification)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Normalized Speedup chart (relative to baseline platform)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.normalized_speedup import ASCIINormalizedSpeedup, SpeedupData; data = [SpeedupData('SQLite', 1.0, True), SpeedupData('DuckDB', 8.2, False), SpeedupData('Pandas', 0.4, False)]; chart = ASCIINormalizedSpeedup(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with baseline at center and bars extending left/right", "item_id": "ascii-chart-normalized-speedup", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-normalized-speedup", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create SpeedupData dataclass and ASCIINormalizedSpeedup class with render() using log2-scaled bars", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement centered baseline reference line with left (slower) / right (faster) bar layout", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_normalized_results() factory that computes ratios from NormalizedResult data and baseline name", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'normalized_speedup' case to render_ascii_chart() in exporters.py", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (symmetry of log scaling, edge cases, single platform, many platforms)", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add normalized_speedup to flagship and head_to_head templates in templates.py", "wid": "w6"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "compute percentiles with sorted arrays and linear interpolation", "dont": "add numpy/scipy as a dependency", "why": "the visualization module has zero external deps"}, {"INSTEAD": "implement as a new chart class", "dont": "modify ASCIIBoxPlot", "why": "box plots and percentile ladders serve different purposes (IQR vs tail-latency thresholds)"}], "approach": "Follow the established pattern in benchbox/core/visualization/ascii/bar_chart.py:\n\n1. Create a dataclass for input data (PercentileData with name, p50, p90, p95, p99).\n2. Create ASCIIPercentileLadder(ASCIIChartBase) with a render() method.\n3. Use layered horizontal bars with different block/intensity characters for each band.\n4. Use the same colorblind-friendly palette (DEFAULT_PALETTE) for percentile bands.\n5. Add a from_query_results() factory that computes percentiles from raw query timings.\n6. Register in __init__.py, exporters.py (render_ascii_chart), and add to templates.\n\nCompute percentiles using Python stdlib statistics or sorted-array interpolation\n(no numpy dependency). The chart should handle single-platform and multi-platform modes.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that visualizes percentile latency bands (P50, P90, P95, P99)\nfor each platform. Each platform gets a single row showing nested/layered bar segments\nwhere each percentile extends further right, creating a \"ladder\" appearance.\n\nThis chart answers: \"what fraction of queries finish within X ms?\" which is the gold\nstandard for tail-latency analysis in database benchmarks. The difference between P50\nand P99 reveals consistency - a platform with a tight spread is more predictable than\none with a wide gap.\n\nExample output:\n```\nPercentile Latency by Platform (ms)\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n P50 P90 P95 P99\nDuckDB \u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2588\u2588 12 | 45 | 78 | 120\nPolars \u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588 15 | 52 | 95 | 310\nPandas \u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2588\u2588\u2588\u2588 25 | 85 | 140 | 280\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2591 P50 \u2592 P90 \u2593 P95 \u2588 P99\n```\n\nThis is distinct from the existing Box Plot which shows IQR-based quartiles with\noutlier whiskers. The Percentile Ladder emphasizes specific tail-latency thresholds\n(P90/P95/P99) that are standard SLA metrics, rendered as layered horizontal bars\nrather than box-and-whisker diagrams.", "id": "ascii-chart-percentile-ladder", "preserves": ["ASCIIChartBase interface contract (render() method, options, metadata) unchanged", "All 9 existing chart types continue to render identically (no changes to their logic)", "render_ascii_chart() in exporters.py continues to handle all existing chart_type strings"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Percentile Ladder chart (P50/P90/P95/P99)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.percentile_ladder import ASCIIPercentileLadder, PercentileData; data = [PercentileData('DuckDB', 12, 45, 78, 120), PercentileData('Polars', 15, 52, 95, 310)]; chart = ASCIIPercentileLadder(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with two platform rows showing layered bars", "item_id": "ascii-chart-percentile-ladder", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-percentile-ladder", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.core.visualization.exporters import render_ascii_chart; print('OK')\"", "description": "New chart registered in exporters", "expected": "OK", "item_id": "ascii-chart-percentile-ladder", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create PercentileData dataclass and ASCIIPercentileLadder class with render() method", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement percentile computation helper (from raw query times to P50/P90/P95/P99)", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_query_results() factory function for creating chart from NormalizedResult data", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'percentile_ladder' case to render_ascii_chart() in exporters.py", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for percentile ladder (single platform, multi-platform, edge cases)", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add percentile_ladder to relevant templates in templates.py (flagship, default)", "wid": "w6"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "add a separate subtitle line instead", "dont": "add metadata to the chart title string itself", "why": "titles are already long and metadata should be visually distinct"}, {"INSTEAD": "use generic detection of same-platform/different-mode pairs", "dont": "hardcode platform names or mode strings in the naming logic", "why": "new platforms and modes are added regularly"}, {"INSTEAD": "add a color_scheme parameter defaulting to 'diverging'", "dont": "make the blue\u2192red heatmap palette the ONLY option", "why": "some users may prefer the sequential green\u2192red palette"}, {"INSTEAD": "use simple IQR calculation with sorted() and len()", "dont": "use statistical outlier detection that requires numpy", "why": "BenchBox has zero numpy dependency for core"}, {"INSTEAD": "only wrap at natural boundaries like spaces or parentheses", "dont": "wrap labels by splitting platform names mid-word", "why": "'DataFu\\nsion' is unreadable"}], "approach": "Work chart-by-chart, not feature-by-feature. Each phase is self-contained and testable.\n\nPhase 1 (naming) modifies result_plotter.py only - add a _disambiguate_modes() method similar\nto existing _disambiguate_platforms(). Call it after _disambiguate_platforms() in __init__().\n\nPhase 2 (truncation) modifies individual chart files. Replace hardcoded caps with dynamic\ncalculation: `min(max(len(label) for label in labels), 30)`. Adjust the bar/data area width\nby subtracting the label width from the total width, with a floor of 40 chars for data.\n\nPhase 3 (metadata) adds a new _render_subtitle() to base.py, similar to _render_title().\nThread metadata as an optional dict through chart constructors. Modify _render_single_ascii_chart()\nin visualization.py to extract metadata from NormalizedResult and pass to chart.\n\nPhases 4-7 are localized to individual chart files - follow existing patterns in each file.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": [], "description": "The ASCII charting module (benchbox/core/visualization/ascii/, ~2,976 lines across 11 files) is\nfunctionally complete with 11 chart types, but real-world usage reveals several UX issues that make\ncharts harder to read and interpret. These are cosmetic/readability issues, not functional bugs -\nall chart types render correctly and return valid strings.\n\nIssues identified from head_to_head and comparison template output with DataFusion SQL vs DataFrame:\n\n1. **Asymmetric series naming**: When comparing SQL vs DataFrame, one series is labeled\n \"DataFusion\" and the other \"DataFusion (dataframe)\". Both should carry their mode suffix\n for symmetry - e.g., \"DataFusion (sql)\" vs \"DataFusion (df)\". Prefer abbreviated mode\n labels: \"df\" not \"dataframe\", \"sql\" stays \"sql\".\n\n2. **Aggressive label truncation**: Platform names are truncated to as few as 8 characters\n (comparison_bar.py:81), causing \"DataFusion\" \u2192 \"DataFu..\". Labels should accommodate full\n names by shrinking the data area or wrapping to a second line.\n\n3. **No chart metadata**: Scale factor, platform version, and tuning configuration are available\n in NormalizedResult but never displayed. Charts should include a subtitle line showing key\n run parameters (e.g., \"SF=1 | DataFusion 44.0.0 | tuned\").\n\n4. **Inconsistent color application**: Bar chart labels are not colored (bar_chart.py:141) while\n histogram bars and box plot lines are colored. Colors should be applied uniformly across all\n chart types that display series data.\n\n5. **Missing axis labels**: Charts lack axis title annotations (e.g., \"Query ID \u2192\" on x-axis,\n \"\u2191 Execution Time (ms)\" on y-axis). Only tick values and query IDs are shown.\n\n6. **Heatmap readability**: Column headers are severely truncated. The green\u2192yellow\u2192red color\n scale (heatmap.py:32-38) is hard to distinguish in many terminals. User requests red\u2192blue\n diverging palette for better contrast.\n\n7. **Histogram outlier bars**: All bars use the same \u2588 block character regardless of whether\n they represent outlier values. Outlier columns should use a visually distinct character\n (e.g., \u2593 or hatched pattern) so extreme values are immediately recognizable.\n\n8. **Summary box dot-leader gap**: In comparison summary boxes, the percentage change is\n right-justified against the border while metric text is left-aligned, leaving a wide blank\n gap (80+ chars of whitespace). Replace with dot-leader pattern (\u00b7\u00b7\u00b7\u00b7\u00b7) so the eye can trace\n from the value to its percentage.", "id": "ascii-chart-polish-and-ux-refinements", "preserves": ["4-tier color degradation chain (truecolor -> 256 -> 16 -> none) works for all changes", "ASCIIChartBase.render() -> str contract unchanged", "ASCIIChartOptions(use_color=False) produces clean output without ANSI codes", "All 6 original chart types + 3 comparison chart types render correctly", "All existing factory functions (from_bar_data, from_query_latency_data, from_comparison_data, etc.) remain backward-compatible", "Chart width respects terminal width detection and ASCIIChartOptions.width override", "MCP generate_chart tool continues to work for all existing chart_type values", "Unicode -> ASCII fallback works for new characters (outlier bars, axis labels)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/bar_chart.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/box_plot.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/comparison_bar.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/diverging_bar.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/heatmap.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/histogram.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/line_chart.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/scatter_plot.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/summary_box.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/result_plotter.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/test_ascii_charts.py"}], "state": "done", "title": "Polish ASCII chart output: series naming, label truncation, metadata, colors, axes, heatmap, and histogram outliers", "verifications": [{"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -q", "description": "All existing ASCII chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'visualization or ascii' -q", "description": "No regressions in visualization module", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check benchbox/core/visualization/", "description": "No lint errors in modified files", "expected": "All checks passed or no output (clean)", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.visualization.ascii import ASCIIBarChart, ASCIIBoxPlot, ASCIIHeatmap, ASCIIQueryHistogram, ASCIIComparisonBar, ASCIIDivergingBar, ASCIISummaryBox; print('imports ok')\"", "description": "MCP generate_chart still works for all chart types", "expected": "imports ok", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run benchbox visualize --template head_to_head benchmark_runs/results/tpch_sf1_datafusion_df_20260206_141251_1d7dbb77.json benchmark_runs/results/tpch_sf1_datafusion_sql_20260206_141231_c20da81e.json", "description": "Visual regression check with real results", "expected": "Charts render without errors; series names are symmetric and untruncated", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 5}], "work": [], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "implement as a new class that can accept the same input data", "dont": "modify ASCIIHeatmap", "why": "the heatmap shows absolute values with intensity encoding, while the rank table shows ordinal positions"}, {"INSTEAD": "show '1st', '2nd', '3rd' text, not '1.0', '2.0'", "dont": "use floating-point rank display", "why": "ranks are ordinal"}], "approach": "Build as a table renderer (similar approach to ASCIIHeatmap) with ranking logic:\n\n1. Create RankTableData (queries list[str], platforms list[str],\n times dict[tuple[platform, query], float]) dataclass.\n2. Create ASCIIRankTable(ASCIIChartBase) with render().\n3. For each query, rank platforms by execution time (lower is better). Handle ties\n by assigning the same rank (1st, 1st, 3rd, 4th for a two-way tie).\n4. Color 1st place green (#1b9e77), last place red (#d95f02), middle ranks dim.\n5. Footer: count 1st-place wins per platform, compute geometric mean of ranks.\n6. Add from_heatmap_data() factory that reuses the same matrix data as ASCIIHeatmap.\n7. Auto-split into multiple charts when queries exceed terminal height (similar to\n histogram chunking pattern).\n\nUse natural query sort order (Q1, Q2, Q10 not Q1, Q10, Q2) matching\nASCIIQueryHistogram._natural_sort_key().\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that shows a ranked leaderboard table with each platform's\nrank position per query, plus aggregate win/loss/tie counts and an overall geometric\nmean rank. Each cell shows a compact rank indicator (1st, 2nd, 3rd...) with color\ncoding. The footer summarizes total wins, losses, and the overall ranking.\n\nThis is how ClickBench presents its results. When comparing 4+ platforms across 22+\nqueries, it is hard to determine an overall winner from raw timing data. A rank table\nimmediately shows: \"DuckDB wins 18 of 22 queries, Polars wins 3, SQLite wins 1.\"\n\nExample output:\n```\nQuery Rankings (1st = fastest)\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nQuery DuckDB Polars SQLite Pandas\nQ1 1st 2nd 3rd 4th\nQ2 2nd 1st 3rd 4th\nQ3 1st 1st 3rd 4th\nQ4 1st 2nd 4th 3rd\n...\nQ22 1st 3rd 2nd 4th\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n1st wins 15 5 2 0\nGeoRank 1.2 1.8 2.8 3.9\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n```\n\nThis is distinct from the Heatmap (shows absolute times color-coded by magnitude,\nnot relative rankings), the Comparison Bar (handles only two platforms), and the\nBar Chart (one aggregate metric, not per-query ranking). Rank tables answer\n\"which platform wins the most queries?\" rather than \"how fast is each platform?\"", "id": "ascii-chart-rank-table", "preserves": ["ASCIIHeatmap behavior unchanged (heatmap shows absolute values, rank table shows ordinal positions)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Rank Table chart (per-query competitive ranking with win/loss tally)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.rank_table import ASCIIRankTable, RankTableData; data = RankTableData(['Q1','Q2'], ['DuckDB','Polars'], {('DuckDB','Q1'): 10, ('Polars','Q1'): 20, ('DuckDB','Q2'): 30, ('Polars','Q2'): 15}); chart = ASCIIRankTable(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered table with rank positions and win/loss tally", "item_id": "ascii-chart-rank-table", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-rank-table", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-rank-table", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create RankTableData dataclass and ASCIIRankTable class with render()", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement per-query ranking with tie handling (1st, 1st, 3rd) and color coding", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement footer with win tallies and geometric mean rank computation", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_heatmap_data() factory that accepts the same matrix format as ASCIIHeatmap", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'rank_table' case to render_ascii_chart() in exporters.py", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (ties, single platform, many platforms, auto-split for many queries)", "wid": "w6"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add rank_table to head_to_head and flagship templates in templates.py", "wid": "w7"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "accept a flexible list of SparklineColumn instances", "dont": "hardcode metric columns", "why": "different benchmarks expose different metrics"}, {"INSTEAD": "use 1-4 character sparkline bars only", "dont": "use full-width bar charts per cell", "why": "the value of this chart is compactness"}], "approach": "Build as a table renderer with inline mini-bars rather than a traditional chart:\n\n1. Create SparklineColumn (name, values dict[platform, float], higher_is_better) and\n SparklineTableData (platforms, columns) dataclasses.\n2. Create ASCIISparklineTable(ASCIIChartBase) with render().\n3. For each column, normalize values to 0.0-1.0 range and render as a 1-4 character\n vertical block using LOWER_BLOCK_CHARS (\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588).\n4. Respect higher_is_better flag: for latency metrics (lower is better), invert the\n normalization so the best platform gets the tallest bar.\n5. Color the best value green (#1b9e77) and worst red (#d95f02) per column.\n6. Add from_normalized_results() factory that populates columns from standard metrics.\n\nColumn width calculation should be dynamic: metric name width + sparkline bar width +\nvalue width, constrained to fit within terminal width. If too many columns, truncate\nlower-priority columns with \"...\" indicator.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that displays a compact leaderboard table where each row\nis a platform and each column is a metric, with inline sparkline-style mini-bars\nshowing relative values. This combines the information density of a table with\nvisual comparison cues.\n\nThis is the format used by ClickBench's dashboard and most database comparison sites.\nWhen comparing many platforms across many dimensions (total time, geometric mean, P99,\ncost, success rate, data load time), a pure table is hard to scan visually, while\nindividual charts for each metric create information overload.\n\nExample output:\n```\nPlatform Comparison Overview\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nPlatform Total(ms) GeoMean P99(ms) Load(s) Success\nDuckDB \u2587 1,240 \u2587 56 \u2585 120 \u2582 2.1 \u2588\u2588\u2588\u2588 100%\nPolars \u2586 1,580 \u2586 72 \u2587 310 \u2581 1.2 \u2588\u2588\u2588\u2588 100%\nSQLite \u2583 4,820 \u2583 219 \u2583 450 \u2585 8.4 \u2588\u2588\u2588\u2591 95%\nPandas \u2581 12,300 \u2581 559 \u2581 980 \u2583 5.2 \u2588\u2588\u2591\u2591 82%\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2587=best \u2581=worst (bars scaled per column)\n```\n\nThis is distinct from the Summary Box (single run stats), the Heatmap (query-level\ndetail), and the Bar Chart (one metric at a time). The sparkline table uniquely\ncombines multiple aggregate metrics in a single compact view with embedded\nvisual encoding.", "id": "ascii-chart-sparkline-table", "preserves": ["ASCIISummaryBox behavior unchanged (sparkline table is multi-platform overview, summary box is single-run aggregate stats)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Sparkline Table chart (compact multi-metric leaderboard)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.sparkline_table import ASCIISparklineTable, SparklineColumn, SparklineTableData; cols = [SparklineColumn('Total(ms)', {'DuckDB': 1240, 'Polars': 1580}, False), SparklineColumn('Success', {'DuckDB': 100, 'Polars': 100}, True)]; data = SparklineTableData(['DuckDB', 'Polars'], cols); chart = ASCIISparklineTable(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered table with sparkline bars and values", "item_id": "ascii-chart-sparkline-table", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-sparkline-table", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create SparklineColumn, SparklineTableData dataclasses and ASCIISparklineTable class", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement per-column normalization with higher_is_better inversion and sparkline bar rendering", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement dynamic column width calculation with truncation for narrow terminals", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_normalized_results() factory that populates standard benchmark metrics", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'sparkline_table' case to render_ascii_chart() in exporters.py", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (multi-platform, mixed higher/lower-is-better, edge cases, narrow terminal)", "wid": "w6"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add sparkline_table to flagship template in templates.py", "wid": "w7"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "only use ExecutionPhases data", "dont": "import or depend on specific platform adapters", "why": "the chart should be platform-agnostic"}, {"INSTEAD": "skip them entirely", "dont": "render phases with zero duration as empty segments", "why": "they add visual noise"}], "approach": "Follow the pattern in benchbox/core/visualization/ascii/bar_chart.py for the\nhorizontal bar layout, extended with stacked segments:\n\n1. Create StackedBarSegment (phase_name, value, color) and StackedBarData (label,\n segments, total) dataclasses.\n2. Create ASCIIStackedBar(ASCIIChartBase) with a render() method.\n3. Each bar is composed of multiple segments with distinct fill characters or colors\n from DEFAULT_PALETTE. Use unicode block chars for consistent segment boundaries.\n4. Add a from_execution_phases() factory that extracts phase durations from\n BenchmarkResults or ExecutionPhases data.\n5. Show total time at the end of each bar.\n6. Include a phase-color legend below the chart.\n\nPhase ordering should be chronological: DataGen \u2192 Schema \u2192 Load \u2192 Warmup \u2192 Power \u2192\nThroughput \u2192 Maintenance. Phases with zero duration are omitted.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that shows how total benchmark time is distributed across\nexecution phases (data generation, schema creation, data loading, warmup, power test,\nthroughput test, maintenance) as a horizontal stacked bar for each platform.\n\nBenchBox already captures detailed phase timing in its ExecutionPhases model but has\nno dedicated visualization for this data. This chart exposes data that is already\ncollected but currently invisible.\n\nExample output:\n```\nPhase Breakdown by Platform\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nDuckDB \u2591\u2591\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588 12.4s\nPolars \u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 15.2s\nSQLite \u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 28.7s\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2591 DataGen \u2592 Load \u2593 Power \u2588 Throughput\n```\n\nThis reveals insights like \"Platform X is 2x faster at queries but 5x slower at\ndata loading\" which are invisible in query-only charts. TPC benchmarks explicitly\nseparate phases because they measure different capabilities.", "id": "ascii-chart-stacked-phase-breakdown", "preserves": ["All 9 existing chart types continue to render identically", "ExecutionPhases model is read-only for this chart - no modifications to the model", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Stacked Phase Breakdown chart (datagen/load/power/throughput)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.stacked_bar import ASCIIStackedBar, StackedBarData, StackedBarSegment; segs = [StackedBarSegment('Load', 2.0), StackedBarSegment('Power', 8.0), StackedBarSegment('Throughput', 4.0)]; data = [StackedBarData('DuckDB', segs, 14.0)]; chart = ASCIIStackedBar(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with one platform row showing stacked segments", "item_id": "ascii-chart-stacked-phase-breakdown", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-stacked-phase-breakdown", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create StackedBarSegment, StackedBarData dataclasses and ASCIIStackedBar class with render()", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement stacked segment rendering with per-phase colors/fill characters and total label", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_execution_phases() factory to extract phase durations from BenchmarkResults", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'stacked_phase' case to render_ascii_chart() in exporters.py", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (single platform, multi-platform, empty phases, all-zero edge case)", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add stacked_phase to default and flagship templates in templates.py", "wid": "w6"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "commit raw stdout logs, benchmark_runs result bundles, generated TBL/Parquet data, or screenshot/chart batches; summarize durable facts in docs instead.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "fix SQL translation, platform adapters, catalog operations, or packaging defects in this survey; file separate TODOs for those implementation gaps.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "include sketch_df_hll_persist_merge or sketch_df_topk_persist_merge in SQL-mode cloud runs; they are DataFrame aggregate-state operations.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "run live cloud benchmarks, upload cloud data, or generate SF=10 data until w2 records explicit approval, max runtime, log path, output root, disk threshold, and stop conditions.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Run this as a two-stage survey: cheap platform/driver/dry-run validation\nfirst, then approved live cloud execution. Use only SQL-mode query IDs in\nthe cloud SQL survey; keep DataFrame aggregate-state operations for the\ndeferred DataFrame survey. Use BENCHBOX_OUTPUT_DIR=$HOME/Developer/benchmark_runs\nfor generated data, result bundles, analysis artifacts, and long-run logs.\nCommit durable markdown summaries and follow-up TODOs only; keep raw logs,\ncharts, and result JSONs outside git.\n", "blocked_reason": null, "category": "Benchmark Development", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-25T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 1, "reason": "SQL mode establishes the baseline; DataFrame mode includes the sketch_df_* aggregate-state operations and has known exact-fallback caveats on Polars/Dask that make cross-engine comparison less clean. Separate survey.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: DataFrame-mode approx survey is explicitly sequenced after the SQL-mode survey; parent approx-analytics-cloud-survey is itself still open/planning, not yet actionable.", "summary": "DataFrame mode survey (Polars, PySpark, DataFusion) for approx queries"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 2, "reason": "Power phase (single session) is sufficient for capability verification and latency measurement. Throughput adds cloud cost and complexity; defer to a dedicated concurrency study.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'dedicated concurrency study' with no scope/owner defined; parent item itself still open/planning, not ripe to track separately.", "summary": "Throughput phase benchmarking (concurrent sessions)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 3, "reason": "BenchBox measures latency, not approximation error. Accuracy belongs in a separate accuracy study with known-answer datasets.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly outside BenchBox's latency-only scope; accuracy studies are deferred permanently to a separate project, not a pending TODO.", "summary": "Error-rate analysis (actual approximation accuracy vs exact)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "approx-analytics-cloud-survey", "id": 4, "reason": "pyathen/pyodbc drivers and IAM setup are significant additional overhead. Start with the 8 primary platforms; add these if they surface unique sketch behavior.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Parent survey item approx-analytics-cloud-survey is still open (planning); adding Athena/Synapse/Fabric is a natural extension of that tracked item.", "summary": "Amazon Athena SQL, Azure Synapse, Fabric Warehouse"}], "deps": [], "description": "Survey approximate analytics support and performance across all major cloud SQL\nplatforms by running the read_primitives (one-shot approx aggregates) and\nwrite_primitives SQL sketch category (persist + merge + requery) benchmarks at three\nscale factors (SF=0.1, SF=1, SF=10).\n\nThe goal is a complete support matrix \u2014 which platform supports which sketch family,\nat what latency, and how latency scales with data volume \u2014 plus identification of\nany gaps or regressions not yet captured in the cross-engine docs.\n\nTarget platforms (SQL mode):\n- Databricks SQL (enabled, credentials available)\n- ClickHouse Cloud (enabled)\n- Firebolt Cloud (enabled)\n- Starburst/Trino Cloud (enabled)\n- MotherDuck (enabled)\n- Snowflake (driver install required)\n- Amazon Redshift (driver install required)\n- Google BigQuery (driver install required)\n\nread_primitives queries in scope (5):\n approx_count_distinct_simple, approx_count_distinct_groupby,\n approx_quantile_groupby, approx_quantiles_array, approx_top_k_lineitem\n\nSQL write_primitives sketch ops in scope (22 sketch_* operations):\n sketch_ddl_create_persistent_table through sketch_query_topk_combine_lgmm10.\n Exclude sketch_df_hll_persist_merge and sketch_df_topk_persist_merge because\n those are DataFrame aggregate-state operations, not SQL cloud-platform operations.\n\nKnown platform limitations (from docs \u2014 verify, don't assume still current):\n- Redshift: HLL substitution only; KLL and Top-K skipped (no native equivalent)\n- BigQuery: Theta substituted with HLL; Top-K skipped (no native accumulator)\n- DataFusion: sketches skipped entirely\n- DuckDB (not in scope here): 4 theta/topk ops blocked by extension `2e38607`\n", "id": "approx-analytics-cloud-survey", "preserves": ["Existing result bundles in benchmark_runs/results/ and $BENCHBOX_OUTPUT_DIR/results/ \u2014 never overwrite or delete", "benchmark_runs/datagen/tpch_sf001/ and $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf001/ \u2014 existing SF=0.01 data; do not force-regenerate", "docs/benchmarks/read-primitives-approximate-functions.md and write-primitives-sketch-functions.md \u2014 update only, never truncate"], "prior_art": [{"concept": "benchbox/cli/commands/run.py \u2014 canonical cloud platform keys include clickhouse-cloud and firebolt:cloud; reuse exact keys", "decision": "reuse", "path": "benchbox/cli/commands/run.py"}, {"concept": "benchbox/core/read_primitives/catalog/queries.yaml \u2014 5 approx query IDs already defined; no new catalog work needed", "decision": "reuse", "path": "benchbox/core/read_primitives/catalog/queries.yaml"}, {"concept": "benchbox/core/write_primitives/catalog/operations.yaml \u2014 22 SQL sketch op IDs plus 2 DataFrame sketch ops already defined; reuse only SQL ops here", "decision": "reuse", "path": "benchbox/core/write_primitives/catalog/operations.yaml"}, {"concept": "benchmark_runs/datagen/tpch_sf001/ \u2014 SF=0.01 datagen exists; extend, don't regenerate", "decision": "extend", "path": "benchmark_runs/datagen/tpch_sf001/"}, {"concept": "docs/benchmarks/read-primitives-approximate-functions.md \u2014 cross-engine function reference; reuse as ground truth for expected dialect rewrites", "decision": "reuse", "path": "docs/benchmarks/read-primitives-approximate-functions.md"}, {"concept": "docs/benchmarks/write-primitives-sketch-functions.md \u2014 sketch family \u00d7 engine support matrix; reuse and extend with Firebolt + Starburst findings", "decision": "extend", "path": "docs/benchmarks/write-primitives-sketch-functions.md"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/platform-expansion/planning/"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/read-primitives-approximate-functions.md"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/write-primitives-sketch-functions.md"}], "state": "planning", "title": "Approximate Analytics Cloud SQL Survey \u2014 read_primitives + SQL write_primitives at SF=0.1/1/10", "verifications": [{"command": "uv run -- python - <<'PY'\nfrom pathlib import Path\nimport yaml\n\nitem = yaml.safe_load(Path(\"_project/TODO/benchmark-expansion/planning/approx-analytics-cloud-survey.yaml\").read_text())\nwork_notes = \"\\n\".join(str(unit.get(\"notes\", \"\")) for unit in item[\"work\"])\nassert \"sketch_df_hll_persist_merge\" not in work_notes\nassert \"sketch_df_topk_persist_merge\" not in work_notes\nprint(\"sql-only sketch scope verified\")\nPY\n", "description": "TODO query scope stays SQL-only and excludes DataFrame aggregate-state sketch ops", "expected": "sql-only sketch scope verified", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 1}, {"command": "BENCHBOX_OUTPUT_DIR=\"${BENCHBOX_OUTPUT_DIR:-$HOME/Developer/benchmark_runs}\" uv run -- python - <<'PY'\nimport json\nimport os\nfrom pathlib import Path\n\nroot = Path(os.environ[\"BENCHBOX_OUTPUT_DIR\"]) / \"datagen\"\nexpected = {\"sf01\": 0.1, \"sf1\": 1.0, \"sf10\": 10.0}\nfor sf, value in expected.items():\n path = root / f\"tpch_{sf}\" / \"_datagen_manifest.json\"\n manifest = json.loads(path.read_text())\n actual = float(manifest[\"scale_factor\"])\n assert actual == value, f\"{path}: expected {value}, got {actual}\"\n print(f\"tpch_{sf}: scale_factor={actual}\")\nPY\n", "description": "SF=0.1, SF=1, SF=10 datagen manifests exist under BENCHBOX_OUTPUT_DIR with correct scale_factor", "expected": "tpch_sf01: scale_factor=0.1\ntpch_sf1: scale_factor=1.0\ntpch_sf10: scale_factor=10.0\n", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 2}, {"command": "BENCHBOX_OUTPUT_DIR=\"${BENCHBOX_OUTPUT_DIR:-$HOME/Developer/benchmark_runs}\" uv run -- python - <<'PY'\nimport json\nimport os\nfrom pathlib import Path\n\nroot = Path(os.environ[\"BENCHBOX_OUTPUT_DIR\"]) / \"results\"\nplatforms = {\"databricks\", \"clickhouse-cloud\", \"firebolt\", \"starburst\", \"motherduck\", \"snowflake\", \"bigquery\", \"redshift\"}\nbenchmarks = {\"read_primitives\", \"write_primitives\"}\naliases = {\n \"amazon-redshift\": \"redshift\",\n \"clickhouse-cloud\": \"clickhouse-cloud\",\n \"databricks-sql\": \"databricks\",\n \"google-bigquery\": \"bigquery\",\n}\nfound: set[tuple[str, str]] = set()\nfor path in root.glob(\"*.json\"):\n data = json.loads(path.read_text())\n benchmark = data.get(\"benchmark\", {})\n benchmark_id = str(benchmark.get(\"id\", \"\"))\n if benchmark_id not in benchmarks or float(benchmark.get(\"scale_factor\", -1)) != 1.0:\n continue\n platform_name = str(data.get(\"platform\", {}).get(\"name\", \"\")).lower().replace(\" \", \"-\")\n platform_key = aliases.get(platform_name, platform_name)\n filename = path.name.replace(\"_\", \"-\").lower()\n for expected in platforms:\n if platform_key == expected or expected in filename:\n queries = data.get(\"queries\", [])\n assert queries, f\"{path}: missing query statuses\"\n assert all(\"status\" in query for query in queries), f\"{path}: query without status\"\n found.add((expected, benchmark_id))\nmissing = sorted((platform, benchmark) for platform in platforms for benchmark in benchmarks if (platform, benchmark) not in found)\nassert not missing, f\"missing result bundles: {missing}\"\nprint(f\"verified {len(found)} platform/benchmark SF=1 result bundles\")\nPY\n", "description": "SF=1 result bundles have structured query statuses for each target platform and both SQL benchmarks", "expected": "verified 16 platform/benchmark SF=1 result bundles", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python - <<'PY'\nfrom pathlib import Path\n\ndocs = [\n Path(\"docs/benchmarks/write-primitives-sketch-functions.md\"),\n Path(\"docs/benchmarks/read-primitives-approximate-functions.md\"),\n]\nfor path in docs:\n text = path.read_text()\n for platform in (\"Firebolt\", \"Starburst\"):\n assert f\"| {platform}\" in text or f\"| `{platform}`\" in text, f\"{path}: missing {platform} table row\"\nprint(\"Firebolt and Starburst rows present in approximate analytics docs\")\nPY\n", "description": "Reference docs contain explicit Firebolt and Starburst rows after the survey", "expected": "Firebolt and Starburst rows present in approximate analytics docs", "item_id": "approx-analytics-cloud-survey", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": [], "notes": "Use exact platform keys for this survey:\n databricks, clickhouse-cloud, firebolt:cloud, starburst, motherduck,\n snowflake, bigquery, redshift\nDependency-check keys differ for deployment aliases:\n firebolt:cloud -> firebolt\n motherduck -> verify via platforms list/status (no dedicated check-deps key)\n\nRun:\n uv run -- benchbox platforms list\n uv run -- benchbox check-deps --platform \n\nFor each target platform, record a compact evidence summary for the final\ndocs update:\n - checked BenchBox SHA\n - command(s) run\n - platform status (enabled / missing driver / missing credentials / blocked)\n - driver package and version when available\n - credential signal checked, with secrets redacted\n\nKeep raw stdout under /tmp or $BENCHBOX_OUTPUT_DIR/logs/approx-survey-*/\nonly. Do not commit raw logs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify platform keys, driver availability, credentials, and evidence freshness", "wid": "w0"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w0"], "notes": "For each missing platform identified in w0, prefer existing optional extras:\n uv sync --extra snowflake\n uv sync --extra bigquery\n uv sync --extra redshift\n\nIf the local environment still needs one-off tooling, use uv-managed\ncommands only; never pip install. Do not commit pyproject.toml or uv.lock\nchanges for driver activation in this survey. If an optional extra is\nmissing or incorrect, file a separate follow-up TODO instead of fixing\npackaging here.\n\nRe-run:\n uv run -- benchbox platforms list\n uv run -- benchbox check-deps --platform \n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Install or activate missing optional cloud drivers without widening project scope", "wid": "w1"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w1", "w2", "w3", "w4"], "notes": "Platform: snowflake. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nUse --table-mode native unless the approved run envelope chooses a different\nmode. Full DataSketches binary support is expected; verify actual behavior\nfrom query statuses and platform errors.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Snowflake at SF=0.1, SF=1, SF=10", "wid": "w10"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w1", "w2", "w3", "w4"], "notes": "Platform: bigquery. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nCloud storage URI: gs:///benchbox or approved BigQuery native load.\nExpected substitutions: Theta to HLL_COUNT.INIT/MERGE; KLL through\nKLL_QUANTILES; Top-K skipped. Verify instead of assuming.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Google BigQuery at SF=0.1, SF=1, SF=10", "wid": "w11"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w1", "w2", "w3", "w4"], "notes": "Platform: redshift. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nCloud storage URI: s3:///benchbox with IAM COPY permissions.\nExpected substitutions: Theta to HLLSKETCH/HLL_CREATE_SKETCH; KLL and\nTop-K skipped. Verify instead of assuming.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Amazon Redshift at SF=0.1, SF=1, SF=10", "wid": "w12"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w10", "w11", "w12", "w5", "w6", "w7", "w8", "w9"], "notes": "For each platform x benchmark x query x scale factor, extract from result bundles:\n - Pass / Fail / Skipped status\n - execution_time_seconds, converted to ms only for display\n - Sketch family mapping actually used (native / substitution / skipped / failed)\n - Result bundle path and checked BenchBox SHA\n\nProduce a compact summary, not raw stdout:\n 1. Support matrix table: platforms x sketch families\n 2. Performance comparison: SF=1 latency across platforms per sketch op\n 3. Scale analysis: SF=0.1/1/10 per platform, highlighting O(1) sketch\n merge behavior versus O(n) exact/substituted fallback\n\nUse:\n uv run -- benchbox compare ...\n uv run -- benchbox visualize --chart-type performance_bar\n\nSave generated charts/raw analysis under $BENCHBOX_OUTPUT_DIR or /tmp.\nCommit only durable markdown summaries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Compile cross-platform support matrix and performance comparison", "wid": "w13"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w13"], "notes": "Update docs/benchmarks/write-primitives-sketch-functions.md with:\n - Firebolt and Starburst rows in the sketch family x engine support matrix\n - MotherDuck extension version observation\n - Current compact evidence summary: commands, checked SHA, driver versions,\n PASS/FAIL/SKIPPED counts, and result-bundle identifiers\n - Any changed platform behavior versus the pre-survey docs\n\nUpdate docs/benchmarks/read-primitives-approximate-functions.md with:\n - Firebolt and Starburst rows in the function reference tables\n - Current compact evidence summary for read_primitives query support\n\nFor each unsupported or broken sketch family that requires code or adapter\nwork, file a separate follow-up TODO under platform-expansion/planning/.\nThis survey commit may edit docs and follow-up TODO YAML only; no runtime\ncode or catalog changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Document findings and file focused platform-gap TODOs", "wid": "w14"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w0"], "notes": "Before any live cloud benchmark, cloud upload, or SF=10 generation, present\nthe operator with:\n - target platforms and exact commands\n - estimated data volume and cloud services touched\n - max runtime per platform and overall stop condition\n - cost/budget ceiling, if one applies\n - output root and log root\n - disk-free threshold; minimum 25 GiB because SF=10 TPC-H data is large\n\nRequire an explicit approval reply in the implementation thread before\ncontinuing. Use:\n export BENCHBOX_OUTPUT_DIR=\"${BENCHBOX_OUTPUT_DIR:-$HOME/Developer/benchmark_runs}\"\n export APPROX_SURVEY_LOG_DIR=\"$BENCHBOX_OUTPUT_DIR/logs/approx-survey-\"\n\nLive runs are sequential by platform. Stop new work if disk space drops\nbelow the approved threshold, if cloud credentials are invalid, or if a\ncommand exceeds the approved runtime cap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Get explicit live-run approval and define the run envelope", "wid": "w2"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w0", "w1"], "notes": "Use dry-run to catch query selection, dialect, and config errors before\ngenerating SF=10 data or touching live warehouses.\n\nRead query set:\n approx_count_distinct_simple,approx_count_distinct_groupby,approx_quantile_groupby,approx_quantiles_array,approx_top_k_lineitem\n\nSQL sketch query set (22 operations; intentionally excludes sketch_df_*):\n sketch_ddl_create_persistent_table,sketch_insert_theta_per_partition,sketch_insert_kll_per_partition,sketch_insert_topk_per_shard,sketch_query_theta_union_merge,sketch_query_kll_quantiles_merge,sketch_query_topk_combine,sketch_drop_persistent_table,sketch_cpc_create_persistent_table,sketch_cpc_insert_per_partition,sketch_cpc_query_union_merge,sketch_cpc_drop_persistent_table,sketch_req_create_persistent_table,sketch_req_insert_per_partition,sketch_req_query_quantile_merge,sketch_req_drop_persistent_table,sketch_query_theta_union_merge_lgk10,sketch_query_theta_union_merge_lgk14,sketch_query_kll_quantiles_merge_k100,sketch_query_kll_quantiles_merge_k1000,sketch_query_topk_combine_lgmm8,sketch_query_topk_combine_lgmm10\n\nFor each platform:\n uv run -- benchbox run --platform --benchmark read_primitives --scale 0.1 \\\n --phases load,power \\\n --queries \\\n --non-interactive \\\n --dry-run /tmp/approx-survey-dryrun/-read\n uv run -- benchbox run --platform --benchmark write_primitives --scale 0.1 \\\n --phases load,power \\\n --queries \\\n --non-interactive \\\n --dry-run /tmp/approx-survey-dryrun/-write\n\nFix no code here. If dry-run exposes a translation or adapter defect,\nfile a separate TODO and mark that platform blocked for this survey.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Dry-run all 8 platforms x 2 SQL benchmarks before data generation or live runs", "wid": "w3"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3"], "notes": "Use the approved BENCHBOX_OUTPUT_DIR from w2. Run sequentially:\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark read_primitives --scale 0.1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark write_primitives --scale 0.1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark read_primitives --scale 1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark write_primitives --scale 1\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark read_primitives --scale 10\n BENCHBOX_OUTPUT_DIR=\"$BENCHBOX_OUTPUT_DIR\" uv run -- benchbox datagen --benchmark write_primitives --scale 10\n\nVerify manifests under:\n $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf01/_datagen_manifest.json\n $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf1/_datagen_manifest.json\n $BENCHBOX_OUTPUT_DIR/datagen/tpch_sf10/_datagen_manifest.json\n\nEstimated disk: ~200MB (SF=0.1), ~2GB (SF=1), ~20GB (SF=10)\nuncompressed TBL. Announce the SF=10 command, max runtime, log path, and\nstop condition immediately before launch.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Generate TPC-H data at SF=0.1, SF=1, SF=10 in the shared UAT output root", "wid": "w4"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: databricks. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nOutput/staging: dbfs:/Volumes////benchbox\nRecord result bundle paths and query status counts for w13.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Databricks SQL at SF=0.1, SF=1, SF=10", "wid": "w5"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: clickhouse-cloud. Use phases load,power, --non-interactive, the\nread query set, and the 22-op SQL sketch query set from w3.\nClickHouse uses native -State/-Merge combinators for sketches, not\nDataSketches binary portability. Record result bundle paths and query\nstatus counts for w13.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: ClickHouse Cloud at SF=0.1, SF=1, SF=10", "wid": "w6"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: firebolt:cloud. Use phases load,power, --non-interactive, the\nread query set, and the 22-op SQL sketch query set from w3.\nFirebolt approximate-analytics coverage is not yet in the cross-engine\nreference docs. Record which sketch ops succeed, fail, or skip and which\ndialect rewrites Firebolt applies.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Firebolt Cloud at SF=0.1, SF=1, SF=10", "wid": "w7"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: starburst. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nTrino's one-shot approx functions include approx_distinct, approx_percentile,\nand approx_most_frequent. Record whether persisted sketch operations have a\nnative, substituted, skipped, or failed path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: Starburst/Trino Cloud at SF=0.1, SF=1, SF=10", "wid": "w8"}, {"evidence": null, "item_id": "approx-analytics-cloud-survey", "needs": ["w2", "w3", "w4"], "notes": "Platform: motherduck. Use phases load,power, --non-interactive, the read\nquery set, and the 22-op SQL sketch query set from w3.\nVerify the DuckDB extension version MotherDuck exposes before assuming the\nlocal DuckDB extension limitation at `2e38607` still applies.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Live run: MotherDuck at SF=0.1, SF=1, SF=10", "wid": "w9"}], "worktree": "benchmark-expansion"} +{"anti_patterns": [{"dont": "collapse wrappers into internals just because they duplicate CLI reachability", "instead": "classify them first, then decide deprecation separately.", "why": "tests assert wrapper facades as public behavior"}, {"dont": "generate platform counts from docs", "instead": "generate docs from registry metadata or fail the drift check.", "why": "docs are the drifting surface"}, {"dont": "introduce vague tiers like supported-ish", "instead": "use the finite support_status vocabulary.", "why": "ambiguous tiers recreate the same blind spot"}], "approach": "Treat this as a taxonomy and governance change first. Start from existing\ndocumented commitments, then reconcile code evidence against those\ncommitments. Prefer a small, enforceable contract map over aspirational prose.\n", "blocked_reason": null, "category": "Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "architecture-contract-map-and-support-taxonomy", "id": 340, "reason": "Completed by platform-support-status-and-extension-health after the taxonomy was accepted.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Reason states it was already completed by platform-support-status-and-extension-health; that item is not in open_items (done).", "summary": "Fully migrate every platform registry entry to support_status"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "architecture-contract-map-and-support-taxonomy", "id": 341, "reason": "Belongs in benchmark-support-status-and-discovery-policy; platform migration did not cover benchmark metadata.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Target item benchmark-support-status-and-discovery-policy (DONE) states support_status was added to every BENCHMARK_METADATA entry; migration already completed.", "summary": "Fully migrate every benchmark registry entry to support_status"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "architecture-contract-map-and-support-taxonomy", "id": 342, "reason": "Do after this map lands and downstream TODOs resolve their product-surface decisions.", "resolution": "promoted", "resolved_item": "rewrite-architecture-md-around-the-new-contract-map", "resolved_reason": null, "summary": "Rewrite architecture.md around the new contract map"}], "deps": [], "description": "BenchBox has several real product contracts that are only partially declared:\nCLI behavior, Python wrappers, BaseBenchmark.run_with_platform, adapter\nsubclassing, MCP tools, result JSON, explorer input, generated compatibility\ndocs, platform availability, and experimental modules. The architecture is\nstill sound, but future work is making contract decisions implicitly.\n\nThis item creates one authoritative map of public/stable, beta-public,\ninternal, experimental, deprecated, generated, and repo-only surfaces. It also\ndefines support status vocabulary for platforms and benchmarks so docs,\nregistry metadata, packaging, MCP, and PR review use the same language.", "id": "architecture-contract-map-and-support-taxonomy", "preserves": ["Do not change runtime behavior except for narrowly scoped docs/checklist validation needed by this TODO.", "Do not downgrade existing public wrapper tests or imply wrappers are unsupported without an explicit deprecation decision.", "Do not mark experimental code as stable merely because it is packaged."], "prior_art": [{"concept": "Makefile: compat docs generation targets -- reuse generated-docs checks where feasible.", "decision": "reuse", "path": "Makefile"}, {"concept": "docs/reference/backward-compatibility.md: existing shim/deprecation registry -- extend, do not duplicate.", "decision": "extend", "path": "docs/reference/backward-compatibility.md"}, {"concept": "docs/reference/hosted-results-contract.md: existing product-contract style -- reuse the crisp contract language.", "decision": "reuse", "path": "docs/reference/hosted-results-contract.md"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": ".github/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "README.md"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/design/future-state/"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}], "state": "done", "title": "Architecture hardening: create the public contract map and support taxonomy", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate", "description": "Validate TODO graph and schema after adding the contract-map item.", "expected": "Validation passes or reports only pre-existing unrelated TODO issues.", "item_id": "architecture-contract-map-and-support-taxonomy", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/core/test_platform_registry.py tests/unit/core/results/test_schema_policy.py tests/unit/mcp/test_run_surface_contract.py --tb=short", "description": "Run targeted docs checks added by this work.", "expected": "Contract, count, support-status, schema-policy, and MCP run-surface checks pass or skip only for missing optional MCP SDK.", "item_id": "architecture-contract-map-and-support-taxonomy", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": [], "notes": "Capture a compact committed summary of checked SHA, inspected files, and\nany line-number drift. Do not commit raw command transcripts.\n\nMinimum evidence to re-check:\n - README support disclaimer and platform counts: README.md:35-48.\n - Backward-compat registry: docs/reference/backward-compatibility.md:24-84.\n - Wrapper facade tests: tests/unit/test_wrapper_facades_fast.py.\n - MCP public API decision: _project/DONE/mcp-integration/active/refactor-mcp-use-public-api.yaml.\n - Existing future-state architecture docs under docs/design/future-state/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate contract-surface evidence against the current branch before editing", "wid": "w0"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w0"], "notes": "The table must cover at least: CLI commands/options, Python wrapper\nfacades, benchbox.base.BaseBenchmark, BaseBenchmark.run_with_platform,\ncore.base_benchmark, adapter subclassing hooks, platform registry\nmetadata, MCP tools, result JSON bundles, explorer read model,\nsubmission validator behavior, sql_compat rule catalog, generated docs,\nbenchbox.experimental, _project scripts, and TODO/ADR docs.\n\nRequired columns: surface, current tier, owner, compatibility promise,\ndeprecation path, verification gate, and source of truth.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs/reference/public-contracts.md with a complete surface classification table", "wid": "w1"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w1"], "notes": "Define allowed statuses: stable, beta, experimental, repo_only,\ndeprecated, document_only. Record what each status means for packaging,\ndocs, registry visibility, MCP exposure, CI coverage, and breakage policy.\n\nThis work unit defines the vocabulary and contract. It may add minimal\nschema/constants only if that avoids duplicating strings in follow-up\nitems. Full registry migration belongs to the platform-extension TODO.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a support_status taxonomy for platforms and benchmarks", "wid": "w2"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w1"], "notes": "Add or update the project PR checklist so any new public, beta-public,\ndeprecated, generated, or experimental surface must update the contract\nmap or explicitly state why no contract-map change is needed.\n\nThe checklist must mention result schema, MCP params, platform support\nstatus, wrapper facades, adapter subclassing hooks, and generated docs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a PR checklist gate for public/deprecated surface changes", "wid": "w3"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w1"], "notes": "Promote active architecture decisions from scattered TODOs and\nfuture-state docs into one index. Include status, owner, blocking gates,\nand the current implementation surface.\n\nAt minimum index:\n - docs/design/future-state/formalize-mcp-internal-apis/README.md.\n - docs/design/future-state/isolate-experimental-core-subsystems/README.md.\n - docs/reference/backward-compatibility.md entries for wrappers and core.base_benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create an ADR/contract index for scattered future-state decisions", "wid": "w4"}, {"evidence": null, "item_id": "architecture-contract-map-and-support-taxonomy", "needs": ["w2"], "notes": "Add generated or tested checks for platform counts, benchmark counts,\nsupport-status tables, result-schema version claims, and adapter\navailability statements. The check should fail on contradictory claims\nlike docs saying a platform is both unavailable and supported.\n\nPrefer generating tables from registry metadata where possible. If a\nsection is intentionally editorial, mark it as non-authoritative in the\npublic contract map.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add docs drift checks for current-count and support-claim statements", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add numpy/scipy dependency", "instead": "CDF computation is trivial with sorted arrays", "why": "the visualization module has zero external deps"}, {"dont": "reuse ASCIILineChart directly", "instead": "create a dedicated class that borrows rendering patterns", "why": "CDF has fixed 0-100% Y-axis and different data preparation (sorted cumulative)"}], "approach": "Reuse the grid-based rendering pattern from benchbox/core/visualization/ascii/line_chart.py\n(grid creation, Bresenham-like line drawing, axis rendering) but with CDF-specific logic:\n\n1. Create CDFSeriesData (name, values list[float]) dataclass.\n2. Create ASCIICDFChart(ASCIIChartBase) with render().\n3. For each series: sort values, compute cumulative proportions (i/n for each value).\n4. Plot on a grid where X = execution time, Y = cumulative proportion (0-100%).\n5. Reuse the marker system from ASCIILineChart (*, +, o, x, ^, v, #, @) for multi-series.\n6. Add percentile annotations: vertical dashed lines at P50, P95, P99 positions.\n7. Add from_query_results() factory.\n\nKeep Y-axis fixed at 0-100% (no auto-scaling). X-axis scales to data range.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that renders a cumulative distribution function (CDF) plot\nshowing the proportion of queries completing within a given time threshold. The X-axis\nis execution time, the Y-axis is the cumulative proportion (0% to 100%), and each\nplatform is a separate line/curve.\n\nCDF plots are the standard academic and industry method for visualizing latency\ndistributions, used extensively in TailBench, YCSB, and database vendor whitepapers.\nThey answer: \"what percentage of my queries finish within T milliseconds?\" which maps\ndirectly to SLA compliance analysis.\n\nExample output:\n```\nCumulative Distribution of Query Latency\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n100%\u2502 ****+++++++++++\n \u2502 ****++++\n 75%\u2502 ****+++\n \u2502 ***++++\n 50%\u2502 **+++\n \u2502 *+++\n 25%\u2502 *++\n \u2502 *+\n 0%\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n 0 50 100 150 200 300 ms\n * DuckDB + Polars\n```\n\nSteeper curves indicate more consistent performance. Curves that rise quickly and\nplateau show tight latency distributions; long tails that slowly reach 100% reveal\nproblematic outlier queries.\n\nThis is distinct from the Box Plot (quartile summary statistics that collapse\ndistribution shape), the Histogram (per-query latency, not cumulative), and the\nLine Chart (time-series trends over runs, not distribution functions). CDF reveals\nbimodal distributions, heavy tails, and performance cliffs that quartile summaries hide.", "id": "ascii-chart-cdf", "preserves": ["ASCIILineChart behavior unchanged (CDF is a distribution chart, line chart is time-series)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII CDF chart (cumulative distribution of query latency)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.cdf_chart import ASCIICDFChart, CDFSeriesData; data = [CDFSeriesData('DuckDB', [10, 20, 30, 50, 120]), CDFSeriesData('Polars', [15, 25, 45, 80, 310])]; chart = ASCIICDFChart(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with two CDF curves, Y-axis 0-100%", "item_id": "ascii-chart-cdf", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-cdf", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-cdf", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create CDFSeriesData dataclass and ASCIICDFChart class with grid-based render()", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement CDF computation (sort values, compute cumulative proportions) and grid plotting", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add optional percentile annotation lines (P50, P95, P99 vertical markers on X-axis)", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_query_results() factory for creating CDF from NormalizedResult data", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'cdf_chart' case to render_ascii_chart() in exporters.py", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (single series, multi-series, identical values, single value, large dataset)", "wid": "w6"}, {"evidence": null, "item_id": "ascii-chart-cdf", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cdf_chart to flagship template in templates.py", "wid": "w7"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "modify ASCIIBarChart or ASCIIDivergingBar", "instead": "implement as a new class", "why": "they serve different purposes (absolute values, pct change)"}, {"dont": "use linear scaling for bar widths", "instead": "use log2 scaling so 2x and 0.5x are symmetric around the baseline", "why": "ratios can span 0.01x to 100x"}], "approach": "Follow the pattern in benchbox/core/visualization/ascii/diverging_bar.py for the\ncentered-zero layout with bars extending left and right from a reference line:\n\n1. Create SpeedupData dataclass (name, ratio, is_baseline).\n2. Create ASCIINormalizedSpeedup(ASCIIChartBase) with a render() method.\n3. Draw a vertical baseline reference line at 1.0x in the center.\n4. Ratios > 1.0 extend right (faster, green), ratios < 1.0 extend left (slower, red).\n5. Add from_normalized_results() factory that computes ratios given results + baseline name.\n6. Use logarithmic scaling for the bar width to handle large ratio ranges (e.g., 0.1x to 50x).\n\nThe baseline platform selection should accept either a platform name or \"slowest\"/\"fastest\"\nas convenience aliases.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that shows platform performance as a ratio relative to a\nuser-selected baseline platform (e.g., \"DuckDB is 3.2x faster than SQLite\"). The\nbaseline is rendered at 1.0x with a vertical reference line. Platforms faster than\nbaseline extend right, slower ones extend left.\n\nThis is the standard way TPC and SPEC results are formally reported. When comparing\nplatforms that span orders of magnitude (50ms to 50,000ms), absolute bar charts make\nfast platforms invisible slivers. Normalized ratios make relative performance\nimmediately clear regardless of absolute magnitude.\n\nExample output:\n```\nNormalized Performance (baseline: SQLite = 1.0x)\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n Slower \u2502 Faster\nSQLite \u2502\u2588 1.0x\nPandas \u2588\u2588\u2588\u2588\u2588\u2588\u2502 0.4x\nDuckDB \u2502\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 8.2x\nPolars \u2502\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 7.5x\nClickHouse \u2502\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 12.1x\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n```\n\nThis is distinct from the existing bar chart (absolute values) and the diverging bar\nchart (percentage change between two runs of the same platform). The normalized speedup\nchart shows cross-platform comparison ratios with a centered baseline reference.", "id": "ascii-chart-normalized-speedup", "preserves": ["ASCIIBarChart behavior unchanged (this is a new chart type, not a modification)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Normalized Speedup chart (relative to baseline platform)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.normalized_speedup import ASCIINormalizedSpeedup, SpeedupData; data = [SpeedupData('SQLite', 1.0, True), SpeedupData('DuckDB', 8.2, False), SpeedupData('Pandas', 0.4, False)]; chart = ASCIINormalizedSpeedup(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with baseline at center and bars extending left/right", "item_id": "ascii-chart-normalized-speedup", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-normalized-speedup", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create SpeedupData dataclass and ASCIINormalizedSpeedup class with render() using log2-scaled bars", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement centered baseline reference line with left (slower) / right (faster) bar layout", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_normalized_results() factory that computes ratios from NormalizedResult data and baseline name", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'normalized_speedup' case to render_ascii_chart() in exporters.py", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (symmetry of log scaling, edge cases, single platform, many platforms)", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-normalized-speedup", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add normalized_speedup to flagship and head_to_head templates in templates.py", "wid": "w6"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "add numpy/scipy as a dependency", "instead": "compute percentiles with sorted arrays and linear interpolation", "why": "the visualization module has zero external deps"}, {"dont": "modify ASCIIBoxPlot", "instead": "implement as a new chart class", "why": "box plots and percentile ladders serve different purposes (IQR vs tail-latency thresholds)"}], "approach": "Follow the established pattern in benchbox/core/visualization/ascii/bar_chart.py:\n\n1. Create a dataclass for input data (PercentileData with name, p50, p90, p95, p99).\n2. Create ASCIIPercentileLadder(ASCIIChartBase) with a render() method.\n3. Use layered horizontal bars with different block/intensity characters for each band.\n4. Use the same colorblind-friendly palette (DEFAULT_PALETTE) for percentile bands.\n5. Add a from_query_results() factory that computes percentiles from raw query timings.\n6. Register in __init__.py, exporters.py (render_ascii_chart), and add to templates.\n\nCompute percentiles using Python stdlib statistics or sorted-array interpolation\n(no numpy dependency). The chart should handle single-platform and multi-platform modes.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that visualizes percentile latency bands (P50, P90, P95, P99)\nfor each platform. Each platform gets a single row showing nested/layered bar segments\nwhere each percentile extends further right, creating a \"ladder\" appearance.\n\nThis chart answers: \"what fraction of queries finish within X ms?\" which is the gold\nstandard for tail-latency analysis in database benchmarks. The difference between P50\nand P99 reveals consistency - a platform with a tight spread is more predictable than\none with a wide gap.\n\nExample output:\n```\nPercentile Latency by Platform (ms)\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n P50 P90 P95 P99\nDuckDB \u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2588\u2588 12 | 45 | 78 | 120\nPolars \u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588 15 | 52 | 95 | 310\nPandas \u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2588\u2588\u2588\u2588 25 | 85 | 140 | 280\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2591 P50 \u2592 P90 \u2593 P95 \u2588 P99\n```\n\nThis is distinct from the existing Box Plot which shows IQR-based quartiles with\noutlier whiskers. The Percentile Ladder emphasizes specific tail-latency thresholds\n(P90/P95/P99) that are standard SLA metrics, rendered as layered horizontal bars\nrather than box-and-whisker diagrams.", "id": "ascii-chart-percentile-ladder", "preserves": ["ASCIIChartBase interface contract (render() method, options, metadata) unchanged", "All 9 existing chart types continue to render identically (no changes to their logic)", "render_ascii_chart() in exporters.py continues to handle all existing chart_type strings"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Percentile Ladder chart (P50/P90/P95/P99)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.percentile_ladder import ASCIIPercentileLadder, PercentileData; data = [PercentileData('DuckDB', 12, 45, 78, 120), PercentileData('Polars', 15, 52, 95, 310)]; chart = ASCIIPercentileLadder(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with two platform rows showing layered bars", "item_id": "ascii-chart-percentile-ladder", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-percentile-ladder", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.core.visualization.exporters import render_ascii_chart; print('OK')\"", "description": "New chart registered in exporters", "expected": "OK", "item_id": "ascii-chart-percentile-ladder", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create PercentileData dataclass and ASCIIPercentileLadder class with render() method", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement percentile computation helper (from raw query times to P50/P90/P95/P99)", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_query_results() factory function for creating chart from NormalizedResult data", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'percentile_ladder' case to render_ascii_chart() in exporters.py", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for percentile ladder (single platform, multi-platform, edge cases)", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-percentile-ladder", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add percentile_ladder to relevant templates in templates.py (flagship, default)", "wid": "w6"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "add metadata to the chart title string itself", "instead": "add a separate subtitle line instead", "why": "titles are already long and metadata should be visually distinct"}, {"dont": "hardcode platform names or mode strings in the naming logic", "instead": "use generic detection of same-platform/different-mode pairs", "why": "new platforms and modes are added regularly"}, {"dont": "make the blue\u2192red heatmap palette the ONLY option", "instead": "add a color_scheme parameter defaulting to 'diverging'", "why": "some users may prefer the sequential green\u2192red palette"}, {"dont": "use statistical outlier detection that requires numpy", "instead": "use simple IQR calculation with sorted() and len()", "why": "BenchBox has zero numpy dependency for core"}, {"dont": "wrap labels by splitting platform names mid-word", "instead": "only wrap at natural boundaries like spaces or parentheses", "why": "'DataFu\\nsion' is unreadable"}], "approach": "Work chart-by-chart, not feature-by-feature. Each phase is self-contained and testable.\n\nPhase 1 (naming) modifies result_plotter.py only - add a _disambiguate_modes() method similar\nto existing _disambiguate_platforms(). Call it after _disambiguate_platforms() in __init__().\n\nPhase 2 (truncation) modifies individual chart files. Replace hardcoded caps with dynamic\ncalculation: `min(max(len(label) for label in labels), 30)`. Adjust the bar/data area width\nby subtracting the label width from the total width, with a floor of 40 chars for data.\n\nPhase 3 (metadata) adds a new _render_subtitle() to base.py, similar to _render_title().\nThread metadata as an optional dict through chart constructors. Modify _render_single_ascii_chart()\nin visualization.py to extract metadata from NormalizedResult and pass to chart.\n\nPhases 4-7 are localized to individual chart files - follow existing patterns in each file.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": [], "description": "The ASCII charting module (benchbox/core/visualization/ascii/, ~2,976 lines across 11 files) is\nfunctionally complete with 11 chart types, but real-world usage reveals several UX issues that make\ncharts harder to read and interpret. These are cosmetic/readability issues, not functional bugs -\nall chart types render correctly and return valid strings.\n\nIssues identified from head_to_head and comparison template output with DataFusion SQL vs DataFrame:\n\n1. **Asymmetric series naming**: When comparing SQL vs DataFrame, one series is labeled\n \"DataFusion\" and the other \"DataFusion (dataframe)\". Both should carry their mode suffix\n for symmetry - e.g., \"DataFusion (sql)\" vs \"DataFusion (df)\". Prefer abbreviated mode\n labels: \"df\" not \"dataframe\", \"sql\" stays \"sql\".\n\n2. **Aggressive label truncation**: Platform names are truncated to as few as 8 characters\n (comparison_bar.py:81), causing \"DataFusion\" \u2192 \"DataFu..\". Labels should accommodate full\n names by shrinking the data area or wrapping to a second line.\n\n3. **No chart metadata**: Scale factor, platform version, and tuning configuration are available\n in NormalizedResult but never displayed. Charts should include a subtitle line showing key\n run parameters (e.g., \"SF=1 | DataFusion 44.0.0 | tuned\").\n\n4. **Inconsistent color application**: Bar chart labels are not colored (bar_chart.py:141) while\n histogram bars and box plot lines are colored. Colors should be applied uniformly across all\n chart types that display series data.\n\n5. **Missing axis labels**: Charts lack axis title annotations (e.g., \"Query ID \u2192\" on x-axis,\n \"\u2191 Execution Time (ms)\" on y-axis). Only tick values and query IDs are shown.\n\n6. **Heatmap readability**: Column headers are severely truncated. The green\u2192yellow\u2192red color\n scale (heatmap.py:32-38) is hard to distinguish in many terminals. User requests red\u2192blue\n diverging palette for better contrast.\n\n7. **Histogram outlier bars**: All bars use the same \u2588 block character regardless of whether\n they represent outlier values. Outlier columns should use a visually distinct character\n (e.g., \u2593 or hatched pattern) so extreme values are immediately recognizable.\n\n8. **Summary box dot-leader gap**: In comparison summary boxes, the percentage change is\n right-justified against the border while metric text is left-aligned, leaving a wide blank\n gap (80+ chars of whitespace). Replace with dot-leader pattern (\u00b7\u00b7\u00b7\u00b7\u00b7) so the eye can trace\n from the value to its percentage.", "id": "ascii-chart-polish-and-ux-refinements", "preserves": ["4-tier color degradation chain (truecolor -> 256 -> 16 -> none) works for all changes", "ASCIIChartBase.render() -> str contract unchanged", "ASCIIChartOptions(use_color=False) produces clean output without ANSI codes", "All 6 original chart types + 3 comparison chart types render correctly", "All existing factory functions (from_bar_data, from_query_latency_data, from_comparison_data, etc.) remain backward-compatible", "Chart width respects terminal width detection and ASCIIChartOptions.width override", "MCP generate_chart tool continues to work for all existing chart_type values", "Unicode -> ASCII fallback works for new characters (outlier bars, axis labels)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/bar_chart.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/box_plot.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/comparison_bar.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/diverging_bar.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/heatmap.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/histogram.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/line_chart.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/scatter_plot.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/summary_box.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/result_plotter.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/test_ascii_charts.py"}], "state": "done", "title": "Polish ASCII chart output: series naming, label truncation, metadata, colors, axes, heatmap, and histogram outliers", "verifications": [{"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -q", "description": "All existing ASCII chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'visualization or ascii' -q", "description": "No regressions in visualization module", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check benchbox/core/visualization/", "description": "No lint errors in modified files", "expected": "All checks passed or no output (clean)", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.visualization.ascii import ASCIIBarChart, ASCIIBoxPlot, ASCIIHeatmap, ASCIIQueryHistogram, ASCIIComparisonBar, ASCIIDivergingBar, ASCIISummaryBox; print('imports ok')\"", "description": "MCP generate_chart still works for all chart types", "expected": "imports ok", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run benchbox visualize --template head_to_head benchmark_runs/results/tpch_sf1_datafusion_df_20260206_141251_1d7dbb77.json benchmark_runs/results/tpch_sf1_datafusion_sql_20260206_141231_c20da81e.json", "description": "Visual regression check with real results", "expected": "Charts render without errors; series names are symmetric and untruncated", "item_id": "ascii-chart-polish-and-ux-refinements", "last_result": null, "last_run": null, "seq": 5}], "work": [], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "modify ASCIIHeatmap", "instead": "implement as a new class that can accept the same input data", "why": "the heatmap shows absolute values with intensity encoding, while the rank table shows ordinal positions"}, {"dont": "use floating-point rank display", "instead": "show '1st', '2nd', '3rd' text, not '1.0', '2.0'", "why": "ranks are ordinal"}], "approach": "Build as a table renderer (similar approach to ASCIIHeatmap) with ranking logic:\n\n1. Create RankTableData (queries list[str], platforms list[str],\n times dict[tuple[platform, query], float]) dataclass.\n2. Create ASCIIRankTable(ASCIIChartBase) with render().\n3. For each query, rank platforms by execution time (lower is better). Handle ties\n by assigning the same rank (1st, 1st, 3rd, 4th for a two-way tie).\n4. Color 1st place green (#1b9e77), last place red (#d95f02), middle ranks dim.\n5. Footer: count 1st-place wins per platform, compute geometric mean of ranks.\n6. Add from_heatmap_data() factory that reuses the same matrix data as ASCIIHeatmap.\n7. Auto-split into multiple charts when queries exceed terminal height (similar to\n histogram chunking pattern).\n\nUse natural query sort order (Q1, Q2, Q10 not Q1, Q10, Q2) matching\nASCIIQueryHistogram._natural_sort_key().\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that shows a ranked leaderboard table with each platform's\nrank position per query, plus aggregate win/loss/tie counts and an overall geometric\nmean rank. Each cell shows a compact rank indicator (1st, 2nd, 3rd...) with color\ncoding. The footer summarizes total wins, losses, and the overall ranking.\n\nThis is how ClickBench presents its results. When comparing 4+ platforms across 22+\nqueries, it is hard to determine an overall winner from raw timing data. A rank table\nimmediately shows: \"DuckDB wins 18 of 22 queries, Polars wins 3, SQLite wins 1.\"\n\nExample output:\n```\nQuery Rankings (1st = fastest)\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nQuery DuckDB Polars SQLite Pandas\nQ1 1st 2nd 3rd 4th\nQ2 2nd 1st 3rd 4th\nQ3 1st 1st 3rd 4th\nQ4 1st 2nd 4th 3rd\n...\nQ22 1st 3rd 2nd 4th\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n1st wins 15 5 2 0\nGeoRank 1.2 1.8 2.8 3.9\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n```\n\nThis is distinct from the Heatmap (shows absolute times color-coded by magnitude,\nnot relative rankings), the Comparison Bar (handles only two platforms), and the\nBar Chart (one aggregate metric, not per-query ranking). Rank tables answer\n\"which platform wins the most queries?\" rather than \"how fast is each platform?\"", "id": "ascii-chart-rank-table", "preserves": ["ASCIIHeatmap behavior unchanged (heatmap shows absolute values, rank table shows ordinal positions)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Rank Table chart (per-query competitive ranking with win/loss tally)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.rank_table import ASCIIRankTable, RankTableData; data = RankTableData(['Q1','Q2'], ['DuckDB','Polars'], {('DuckDB','Q1'): 10, ('Polars','Q1'): 20, ('DuckDB','Q2'): 30, ('Polars','Q2'): 15}); chart = ASCIIRankTable(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered table with rank positions and win/loss tally", "item_id": "ascii-chart-rank-table", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-rank-table", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-rank-table", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create RankTableData dataclass and ASCIIRankTable class with render()", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement per-query ranking with tie handling (1st, 1st, 3rd) and color coding", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement footer with win tallies and geometric mean rank computation", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_heatmap_data() factory that accepts the same matrix format as ASCIIHeatmap", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'rank_table' case to render_ascii_chart() in exporters.py", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (ties, single platform, many platforms, auto-split for many queries)", "wid": "w6"}, {"evidence": null, "item_id": "ascii-chart-rank-table", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add rank_table to head_to_head and flagship templates in templates.py", "wid": "w7"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "hardcode metric columns", "instead": "accept a flexible list of SparklineColumn instances", "why": "different benchmarks expose different metrics"}, {"dont": "use full-width bar charts per cell", "instead": "use 1-4 character sparkline bars only", "why": "the value of this chart is compactness"}], "approach": "Build as a table renderer with inline mini-bars rather than a traditional chart:\n\n1. Create SparklineColumn (name, values dict[platform, float], higher_is_better) and\n SparklineTableData (platforms, columns) dataclasses.\n2. Create ASCIISparklineTable(ASCIIChartBase) with render().\n3. For each column, normalize values to 0.0-1.0 range and render as a 1-4 character\n vertical block using LOWER_BLOCK_CHARS (\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588).\n4. Respect higher_is_better flag: for latency metrics (lower is better), invert the\n normalization so the best platform gets the tallest bar.\n5. Color the best value green (#1b9e77) and worst red (#d95f02) per column.\n6. Add from_normalized_results() factory that populates columns from standard metrics.\n\nColumn width calculation should be dynamic: metric name width + sparkline bar width +\nvalue width, constrained to fit within terminal width. If too many columns, truncate\nlower-priority columns with \"...\" indicator.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that displays a compact leaderboard table where each row\nis a platform and each column is a metric, with inline sparkline-style mini-bars\nshowing relative values. This combines the information density of a table with\nvisual comparison cues.\n\nThis is the format used by ClickBench's dashboard and most database comparison sites.\nWhen comparing many platforms across many dimensions (total time, geometric mean, P99,\ncost, success rate, data load time), a pure table is hard to scan visually, while\nindividual charts for each metric create information overload.\n\nExample output:\n```\nPlatform Comparison Overview\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nPlatform Total(ms) GeoMean P99(ms) Load(s) Success\nDuckDB \u2587 1,240 \u2587 56 \u2585 120 \u2582 2.1 \u2588\u2588\u2588\u2588 100%\nPolars \u2586 1,580 \u2586 72 \u2587 310 \u2581 1.2 \u2588\u2588\u2588\u2588 100%\nSQLite \u2583 4,820 \u2583 219 \u2583 450 \u2585 8.4 \u2588\u2588\u2588\u2591 95%\nPandas \u2581 12,300 \u2581 559 \u2581 980 \u2583 5.2 \u2588\u2588\u2591\u2591 82%\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2587=best \u2581=worst (bars scaled per column)\n```\n\nThis is distinct from the Summary Box (single run stats), the Heatmap (query-level\ndetail), and the Bar Chart (one metric at a time). The sparkline table uniquely\ncombines multiple aggregate metrics in a single compact view with embedded\nvisual encoding.", "id": "ascii-chart-sparkline-table", "preserves": ["ASCIISummaryBox behavior unchanged (sparkline table is multi-platform overview, summary box is single-run aggregate stats)", "All 9 existing chart types continue to render identically", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Sparkline Table chart (compact multi-metric leaderboard)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.sparkline_table import ASCIISparklineTable, SparklineColumn, SparklineTableData; cols = [SparklineColumn('Total(ms)', {'DuckDB': 1240, 'Polars': 1580}, False), SparklineColumn('Success', {'DuckDB': 100, 'Polars': 100}, True)]; data = SparklineTableData(['DuckDB', 'Polars'], cols); chart = ASCIISparklineTable(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered table with sparkline bars and values", "item_id": "ascii-chart-sparkline-table", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-sparkline-table", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create SparklineColumn, SparklineTableData dataclasses and ASCIISparklineTable class", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement per-column normalization with higher_is_better inversion and sparkline bar rendering", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement dynamic column width calculation with truncation for narrow terminals", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_normalized_results() factory that populates standard benchmark metrics", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'sparkline_table' case to render_ascii_chart() in exporters.py", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (multi-platform, mixed higher/lower-is-better, edge cases, narrow terminal)", "wid": "w6"}, {"evidence": null, "item_id": "ascii-chart-sparkline-table", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add sparkline_table to flagship template in templates.py", "wid": "w7"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "import or depend on specific platform adapters", "instead": "only use ExecutionPhases data", "why": "the chart should be platform-agnostic"}, {"dont": "render phases with zero duration as empty segments", "instead": "skip them entirely", "why": "they add visual noise"}], "approach": "Follow the pattern in benchbox/core/visualization/ascii/bar_chart.py for the\nhorizontal bar layout, extended with stacked segments:\n\n1. Create StackedBarSegment (phase_name, value, color) and StackedBarData (label,\n segments, total) dataclasses.\n2. Create ASCIIStackedBar(ASCIIChartBase) with a render() method.\n3. Each bar is composed of multiple segments with distinct fill characters or colors\n from DEFAULT_PALETTE. Use unicode block chars for consistent segment boundaries.\n4. Add a from_execution_phases() factory that extracts phase durations from\n BenchmarkResults or ExecutionPhases data.\n5. Show total time at the end of each bar.\n6. Include a phase-color legend below the chart.\n\nPhase ordering should be chronological: DataGen \u2192 Schema \u2192 Load \u2192 Warmup \u2192 Power \u2192\nThroughput \u2192 Maintenance. Phases with zero duration are omitted.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-15T00:00:00Z", "completed_pr": null, "created_at": "2026-02-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add a new ASCII chart type that shows how total benchmark time is distributed across\nexecution phases (data generation, schema creation, data loading, warmup, power test,\nthroughput test, maintenance) as a horizontal stacked bar for each platform.\n\nBenchBox already captures detailed phase timing in its ExecutionPhases model but has\nno dedicated visualization for this data. This chart exposes data that is already\ncollected but currently invisible.\n\nExample output:\n```\nPhase Breakdown by Platform\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nDuckDB \u2591\u2591\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588 12.4s\nPolars \u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 15.2s\nSQLite \u2591\u2591\u2591\u2591\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2592\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 28.7s\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2591 DataGen \u2592 Load \u2593 Power \u2588 Throughput\n```\n\nThis reveals insights like \"Platform X is 2x faster at queries but 5x slower at\ndata loading\" which are invisible in query-only charts. TPC benchmarks explicitly\nseparate phases because they measure different capabilities.", "id": "ascii-chart-stacked-phase-breakdown", "preserves": ["All 9 existing chart types continue to render identically", "ExecutionPhases model is read-only for this chart - no modifications to the model", "render_ascii_chart() in exporters.py handles all existing chart_type strings"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/ascii/"}], "state": "done", "title": "Add ASCII Stacked Phase Breakdown chart (datagen/load/power/throughput)", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.visualization.ascii.stacked_bar import ASCIIStackedBar, StackedBarData, StackedBarSegment; segs = [StackedBarSegment('Load', 2.0), StackedBarSegment('Power', 8.0), StackedBarSegment('Throughput', 4.0)]; data = [StackedBarData('DuckDB', segs, 14.0)]; chart = ASCIIStackedBar(data=data); print(chart.render())\"", "description": "New chart class renders without error", "expected": "Rendered chart with one platform row showing stacked segments", "item_id": "ascii-chart-stacked-phase-breakdown", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/visualization/ascii/test_ascii_charts.py -v", "description": "Existing chart tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ascii-chart-stacked-phase-breakdown", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create StackedBarSegment, StackedBarData dataclasses and ASCIIStackedBar class with render()", "wid": "w1"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement stacked segment rendering with per-phase colors/fill characters and total label", "wid": "w2"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add from_execution_phases() factory to extract phase durations from BenchmarkResults", "wid": "w3"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register chart in __init__.py and add 'stacked_phase' case to render_ascii_chart() in exporters.py", "wid": "w4"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests (single platform, multi-platform, empty phases, all-zero edge case)", "wid": "w5"}, {"evidence": null, "item_id": "ascii-chart-stacked-phase-breakdown", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add stacked_phase to default and flagship templates in templates.py", "wid": "w6"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-05T00:00:00Z", "completed_pr": null, "created_at": "2025-02-05T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement ASCII/text-based alternatives to all BenchBox chart visualizations,\nenabling rich visual output directly in terminal environments and MCP server\nresponses without requiring HTML files or external viewers.\n\nThe current visualization system uses Plotly to generate interactive HTML charts.\nThis TODO adds a parallel ASCII rendering capability that maintains the same\ndata models and interface while outputting terminal-friendly text graphics.", "id": "ascii-chart-visualizations", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "ASCII Chart Visualizations for CLI and MCP Server", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change the golden file format, comparison normalization (_normalize), or ChartOptions fixture values", "why": "out of scope for this hygiene fix."}, {"INSTEAD": "only the NO-flag path needs to stop writing.", "dont": "fold the missing-file case into the --update-golden branch's write path", "why": "--update-golden already handles 'file doesn't exist yet' correctly (it writes unconditionally)"}, {"INSTEAD": "it must be a hard fail with zero file writes.", "dont": "make the missing-golden case regenerate automatically even with a warning", "why": "that's the exact self-healing behavior being removed"}], "approach": "This is a small, surgical fix: remove the implicit write-then-fail\nbehavior from the missing-file branch so it degrades to a plain\npytest.fail with no side effect, matching how a content mismatch already\nbehaves (compare and fail, never auto-write). Optionally extract a small\nhelper (see w2) if it makes the new test cleaner, but a minimal patch of\nthe existing branch is acceptable if extraction adds more churn than\nvalue.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-04T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [], "deps": [], "description": "`tests/contracts/visualization/test_ascii_golden_output.py::test_golden_output`\n(parametrized over all 15 ASCII chart builders) is meant to catch\nunintended rendering changes by diffing against a stored golden file per\nchart, with regeneration gated behind an explicit `--update-golden` CLI\nflag (registered in `tests/contracts/visualization/conftest.py:9`).\n\nThe gate works correctly for the \"intentional change, want to update\"\npath (lines 342-345: `if update_golden: ... write ... pytest.skip(...)`).\nIt does NOT work for the \"golden file is simply missing\" path\n(lines 347-350):\n\n if not golden_path.exists():\n golden_path.parent.mkdir(parents=True, exist_ok=True)\n golden_path.write_text(normalized, encoding=\"utf-8\")\n pytest.fail(f\"Golden file did not exist - created {golden_path}. Re-run to verify parity.\")\n\nThis block writes the golden file unconditionally (without\n`--update-golden`) and then fails once with a message telling the human\nto re-run. On the very next run, the freshly-written file exists and the\ntest passes by construction - there was never an actual parity check.\n\nConcretely, this means: if a golden fixture file under\n`tests/fixtures/golden/ascii/` is accidentally deleted, renamed, or\ngitignored, or if a rebase/merge drops it, CI fails ONCE (as a\n`pytest.fail`, indistinguishable at a glance from a real mismatch), then\nquietly self-heals and goes green on the next CI run or re-run - with\nwhatever the current (possibly buggy) rendering happened to produce\nbaked in as the new \"golden\" answer. Nobody has to run `--update-golden`\nor review a diff for this to happen. This defeats the purpose of a golden\ntest, which is specifically to catch renders changing without human\nreview.", "id": "ascii-golden-selfheal-fix", "preserves": ["All 15 existing golden files continue to compare successfully with no changes to their content.", "Normal content-mismatch failures (line 353-355, existing golden differs from rendered output) are unchanged.", "The --update-golden path (lines 342-345) continues to regenerate and pytest.skip for BOTH new and changed golden files - do not require running the test twice for a brand-new chart type."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "do_not_modify", "path_glob": "tests/contracts/visualization/conftest.py"}, {"kind": "do_not_modify", "path_glob": "tests/fixtures/golden/ascii/"}, {"kind": "only_modify", "path_glob": "tests/contracts/visualization/test_ascii_golden_output.py"}], "state": "done", "title": "ASCII chart golden-output test: make a missing golden file a hard, non-self-healing failure", "verifications": [{"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v", "description": "All 15 existing golden-output tests still pass unchanged", "expected": "15 passed, 0 failed, 0 skipped", "item_id": "ascii-golden-selfheal-fix", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -q --update-golden", "description": "--update-golden still regenerates and skips cleanly", "expected": "15 skipped (each reporting 'Updated golden file: ...')", "item_id": "ascii-golden-selfheal-fix", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/contracts/visualization/ -k missing_golden -v", "description": "New test proves a missing golden file hard-fails without writing anything", "expected": "Test passes; asserts pytest.fail raised and no file created in the tmp golden dir", "item_id": "ascii-golden-selfheal-fix", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ascii-golden-selfheal-fix", "needs": [], "notes": "In `test_golden_output` (tests/contracts/visualization/test_ascii_golden_output.py:336-355),\nchange the `not golden_path.exists()` branch (lines 347-350) so it\ndoes NOT create the file or write anything - it should behave like a\nnormal comparison failure:\n\n if not golden_path.exists():\n pytest.fail(\n f\"Golden file missing: {golden_path}. If this chart is \"\n f\"new or the rendering intentionally changed, regenerate \"\n f\"with --update-golden and review the diff before \"\n f\"committing; do not let this test create it implicitly.\"\n )\n\nRegeneration for a genuinely-missing (e.g. brand-new chart type)\ngolden file should go through the SAME `--update-golden` path used\nfor intentional changes (lines 342-345), which already\nunconditionally writes+skips regardless of whether the file\npreviously existed - so no special-casing is needed there; only the\nno-flag path (347-350) needs to stop self-writing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make a missing golden file a hard failure that does not write anything", "wid": "w1"}, {"evidence": null, "item_id": "ascii-golden-selfheal-fix", "needs": ["w1"], "notes": "Add a test (in the same file, or a small standalone test module\nunder tests/contracts/visualization/) that:\n 1. Points GOLDEN_DIR (or a monkeypatched copy of it) at a tmp_path\n with no golden file present for a given chart name.\n 2. Runs the equivalent of `test_golden_output`'s body (or invokes\n it directly with `update_golden=False`) against that empty dir.\n 3. Asserts `pytest.fail` is raised (e.g. via `pytest.raises` around\n the extracted comparison logic, or via `pytester`/subprocess if\n testing the actual collected test) AND that no file was created\n in the tmp golden dir afterward.\nIf `test_golden_output`'s body isn't easily unit-testable in\nisolation (chart_name/golden_path/write logic is inlined), consider\nfactoring the compare-or-fail logic into a small helper function\n(e.g. `_compare_or_fail(normalized, golden_path, update_golden)`)\nthat both `test_golden_output` and the new missing-file test can\ncall directly - this also makes w1's change easier to unit test\nwithout spinning up a full chart builder.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a test covering the missing-golden-file hard-fail path", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the golden file format, comparison normalization (_normalize), or ChartOptions fixture values", "instead": "(not specified)", "why": "out of scope for this hygiene fix."}, {"dont": "fold the missing-file case into the --update-golden branch's write path", "instead": "only the NO-flag path needs to stop writing.", "why": "--update-golden already handles 'file doesn't exist yet' correctly (it writes unconditionally)"}, {"dont": "make the missing-golden case regenerate automatically even with a warning", "instead": "it must be a hard fail with zero file writes.", "why": "that's the exact self-healing behavior being removed"}], "approach": "This is a small, surgical fix: remove the implicit write-then-fail\nbehavior from the missing-file branch so it degrades to a plain\npytest.fail with no side effect, matching how a content mismatch already\nbehaves (compare and fail, never auto-write). Optionally extract a small\nhelper (see w2) if it makes the new test cleaner, but a minimal patch of\nthe existing branch is acceptable if extraction adds more churn than\nvalue.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-04T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [], "deps": [], "description": "`tests/contracts/visualization/test_ascii_golden_output.py::test_golden_output`\n(parametrized over all 15 ASCII chart builders) is meant to catch\nunintended rendering changes by diffing against a stored golden file per\nchart, with regeneration gated behind an explicit `--update-golden` CLI\nflag (registered in `tests/contracts/visualization/conftest.py:9`).\n\nThe gate works correctly for the \"intentional change, want to update\"\npath (lines 342-345: `if update_golden: ... write ... pytest.skip(...)`).\nIt does NOT work for the \"golden file is simply missing\" path\n(lines 347-350):\n\n if not golden_path.exists():\n golden_path.parent.mkdir(parents=True, exist_ok=True)\n golden_path.write_text(normalized, encoding=\"utf-8\")\n pytest.fail(f\"Golden file did not exist - created {golden_path}. Re-run to verify parity.\")\n\nThis block writes the golden file unconditionally (without\n`--update-golden`) and then fails once with a message telling the human\nto re-run. On the very next run, the freshly-written file exists and the\ntest passes by construction - there was never an actual parity check.\n\nConcretely, this means: if a golden fixture file under\n`tests/fixtures/golden/ascii/` is accidentally deleted, renamed, or\ngitignored, or if a rebase/merge drops it, CI fails ONCE (as a\n`pytest.fail`, indistinguishable at a glance from a real mismatch), then\nquietly self-heals and goes green on the next CI run or re-run - with\nwhatever the current (possibly buggy) rendering happened to produce\nbaked in as the new \"golden\" answer. Nobody has to run `--update-golden`\nor review a diff for this to happen. This defeats the purpose of a golden\ntest, which is specifically to catch renders changing without human\nreview.", "id": "ascii-golden-selfheal-fix", "preserves": ["All 15 existing golden files continue to compare successfully with no changes to their content.", "Normal content-mismatch failures (line 353-355, existing golden differs from rendered output) are unchanged.", "The --update-golden path (lines 342-345) continues to regenerate and pytest.skip for BOTH new and changed golden files - do not require running the test twice for a brand-new chart type."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "do_not_modify", "path_glob": "tests/contracts/visualization/conftest.py"}, {"kind": "do_not_modify", "path_glob": "tests/fixtures/golden/ascii/"}, {"kind": "only_modify", "path_glob": "tests/contracts/visualization/test_ascii_golden_output.py"}], "state": "done", "title": "ASCII chart golden-output test: make a missing golden file a hard, non-self-healing failure", "verifications": [{"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -v", "description": "All 15 existing golden-output tests still pass unchanged", "expected": "15 passed, 0 failed, 0 skipped", "item_id": "ascii-golden-selfheal-fix", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/contracts/visualization/test_ascii_golden_output.py -q --update-golden", "description": "--update-golden still regenerates and skips cleanly", "expected": "15 skipped (each reporting 'Updated golden file: ...')", "item_id": "ascii-golden-selfheal-fix", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/contracts/visualization/ -k missing_golden -v", "description": "New test proves a missing golden file hard-fails without writing anything", "expected": "Test passes; asserts pytest.fail raised and no file created in the tmp golden dir", "item_id": "ascii-golden-selfheal-fix", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ascii-golden-selfheal-fix", "needs": [], "notes": "In `test_golden_output` (tests/contracts/visualization/test_ascii_golden_output.py:336-355),\nchange the `not golden_path.exists()` branch (lines 347-350) so it\ndoes NOT create the file or write anything - it should behave like a\nnormal comparison failure:\n\n if not golden_path.exists():\n pytest.fail(\n f\"Golden file missing: {golden_path}. If this chart is \"\n f\"new or the rendering intentionally changed, regenerate \"\n f\"with --update-golden and review the diff before \"\n f\"committing; do not let this test create it implicitly.\"\n )\n\nRegeneration for a genuinely-missing (e.g. brand-new chart type)\ngolden file should go through the SAME `--update-golden` path used\nfor intentional changes (lines 342-345), which already\nunconditionally writes+skips regardless of whether the file\npreviously existed - so no special-casing is needed there; only the\nno-flag path (347-350) needs to stop self-writing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make a missing golden file a hard failure that does not write anything", "wid": "w1"}, {"evidence": null, "item_id": "ascii-golden-selfheal-fix", "needs": ["w1"], "notes": "Add a test (in the same file, or a small standalone test module\nunder tests/contracts/visualization/) that:\n 1. Points GOLDEN_DIR (or a monkeypatched copy of it) at a tmp_path\n with no golden file present for a given chart name.\n 2. Runs the equivalent of `test_golden_output`'s body (or invokes\n it directly with `update_golden=False`) against that empty dir.\n 3. Asserts `pytest.fail` is raised (e.g. via `pytest.raises` around\n the extracted comparison logic, or via `pytester`/subprocess if\n testing the actual collected test) AND that no file was created\n in the tmp golden dir afterward.\nIf `test_golden_output`'s body isn't easily unit-testable in\nisolation (chart_name/golden_path/write logic is inlined), consider\nfactoring the compare-or-fail logic into a small helper function\n(e.g. `_compare_or_fail(normalized, golden_path, update_golden)`)\nthat both `test_golden_output` and the new missing-file test can\ncall directly - this also makes w1's change easier to unit test\nwithout spinning up a full chart builder.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a test covering the missing-golden-file hard-fail path", "wid": "w2"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Polish", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-13T00:00:00Z", "deferrals": [], "deps": [], "description": "Improve Athena adapter configuration validation with actionable error messages.\n\nCurrent gaps:\n- No validation for required S3 bucket/output location\n- Generic errors when AWS credentials are missing\n- No workgroup existence check before connection\n- Unclear errors for permission issues\n\nGoal: When configuration is incomplete, users get specific guidance\non what's missing and how to fix it.", "id": "athena-configuration-validation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Athena Configuration Validation", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-21T00:00:00Z", "completed_pr": null, "created_at": "2025-12-21T00:00:00Z", "deferrals": [], "deps": [], "description": "The current Athena adapter uploads text files (pipe-delimited .tbl) to S3 and creates\nexternal tables pointing directly to them. This works but is suboptimal for query\nperformance - text file queries scan 100x more data than columnar formats.\n\nAWS documentation shows text table scans use 20GB vs Parquet using only 135MB for\nthe same data, with corresponding cost and performance improvements.\n\nThe recommended approach is:\n1. Upload text files to S3 staging location\n2. Create external staging table pointing to text files\n3. Use CTAS to convert to Parquet:\n CREATE TABLE {table} WITH (format='PARQUET', external_location='s3://.../')\n AS SELECT * FROM {table}_staging\n4. Query the optimized Parquet table\n5. Optionally clean up staging data\n\nThis should be the default behavior for Athena, with an option to use text-only\nfor debugging or when Parquet conversion is not desired.", "id": "athena-parquet-optimization", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Athena: Convert text tables to Parquet using CTAS for query performance", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:53Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #224 of platform-live-integration-tests.\nDeferred: Athena Spark, EMR Serverless, GCP Dataproc live tests\nReason deferred: These Spark-on-cloud platforms require significant cloud infrastructure (clusters, IAM roles, S3/GCS staging). Lower priority - the Spark execution path is partially covered by PySpark/LakeSail local tests.", "id": "athena-spark-emr-serverless-gcp-dataproc-live-tests", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Athena Spark, EMR Serverless, GCP Dataproc live tests", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-24T00:00:00Z", "deferrals": [], "deps": ["aws-glue-platform"], "description": "Add Amazon Athena for Apache Spark platform adapter. This is AWS's interactive\nnotebook-based Spark service with sub-second startup.\n\nUses notebook-style execution model (Jupyter kernel) rather than batch jobs.\nReuses 100% of S3 staging and AWS authentication patterns.", "id": "athena-spark-platform", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Amazon Athena for Apache Spark Platform", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:33:58Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #120 of close-pre-beta-release-blockers.\nDeferred: Audit blog posts and historical release summaries for outdated Alpha/install language\nReason deferred: Current-state docs and package artifacts are the immediate Beta blocker; historical content can follow.", "id": "audit-blog-posts-and-historical-release-summaries-for-outdat", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Audit blog posts and historical release summaries for outdated Alpha/install language", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "drive the backfill from `_project/scripts/audit_sha_backfill.py` so the derivation is auditable and idempotent.", "dont": "backfill develop_sha by hand", "why": "per-file judgment calls about which commit to use will drift over time"}, {"INSTEAD": "content-quality checks belong in a separate audit-review TODO.", "dont": "extend the validator to lint audit content (markdown, headings, sections)", "why": "the validator must stay narrow to remain trustworthy as a CI gate"}, {"INSTEAD": "sequence is w2 (validator) \u2192 w3 (backfill) \u2192 w5 (CI gate) in the same PR series, with w3's output landing before w5 enables enforcement.", "dont": "land the validator before the backfill", "why": "CI would fail on the existing tree from day one"}, {"INSTEAD": "use a path filter on `_project/audits/**` and `results-explorer/e2e/captures/**` so non-audit PRs see no overhead.", "dont": "make the gate run on every CI build", "why": "most PRs do not touch audits"}], "approach": "Build the validator before touching anything else: w2 makes the\ncontract executable. Then w3 backfills the entire existing tree so\nCI starts green and the validator can enforce uniformly without a\ndate cutoff. w4 wires the capture flow for new audits, w5 enforces\nvia CI, w6 documents. Keep the validator narrow (no markdown\nlinting, no audit-content checks) so it stays trustworthy as a CI\ngate. Reuse the existing explorer-job path filter pattern in w5\nrather than authoring a new workflow from scratch \u2014 that pattern\nis already proven and keeps CI cost flat for non-audit PRs.\n", "blocked_reason": null, "category": "Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-08T00:00:00Z", "completed_pr": null, "created_at": "2026-05-08T00:00:00Z", "deferrals": [], "deps": ["results-explorer-token-scan-ci-gate"], "description": "Release-readiness reports under `_project/audits/` describe a tree. If\nthe tree the report describes is not the tree that ships, the verdict\nis structurally unreliable \u2014 even when every individual check passes.\nPR #269 demonstrated this: a release-readiness report and capture\ngenerated against the PR branch were squash-merged onto a develop tip\nthat included PR #268's literal-styled `Result summary` panel and\n`Compare guardrails` section, and the report still claimed READY.\nPR #271 (commit `274a4a11e`) closed the immediate token regression\nmanually, but the framework gap remains.\n\nThis TODO adds a durable, audit-agnostic gate so the same class of\nregression is caught at PR time across every structured audit, not\nonly retheme work:\n\n1. Every `_project/audits/*.md` audit landing after the cutoff date\n must include a `develop_sha:` frontmatter field naming the develop\n SHA the report describes.\n2. The capture flow that authors release-final reports\n (`results-explorer/e2e/captures/release-final.spec.ts` and any\n sibling `*-final.spec.ts` drivers) records the develop SHA at\n run-start so the human author copies it into the audit\n frontmatter.\n3. A small validator (`_project/scripts/audit_sha_check.py`) and a\n `make audit-sha-check FILE=\u2026` target verify the field is present,\n resolves to a SHA reachable from `origin/develop`, and is not too\n far behind the merge target's tip.\n4. A path-aware CI hook on `_project/audits/**` and\n `results-explorer/e2e/captures/**` runs the validator on every PR\n and refuses to merge a release-readiness audit whose recorded SHA\n is missing or stale relative to the merge target.\n\nComplements (does not replace)\n`results-explorer-token-scan-ci-gate.yaml`: that gate prevents the\nliteral-color regression class from shipping; this gate makes every\nrelease-readiness audit's tree-claim auditable so future cross-PR\nraces on any surface (security, performance, docs migration) are\ncaught at the audit-author step rather than after merge.", "id": "audit-develop-sha-stamping-and-ci-gate", "preserves": ["Backfill must be idempotent \u2014 re-running the helper on already-stamped audits is a no-op.", "CI required-result aggregator path-filter behavior.", "Existing _project/audits/*.md content \u2014 backfill only adds the `develop_sha:` frontmatter field; body text and other frontmatter fields stay byte-identical.", "Existing pre-commit hooks and the `make lint` umbrella target."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/"}, {"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "CLAUDE.md"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/audits/"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-07-211858-release-readiness-against-pre-merge-tree.md"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/operations/"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/captures/"}], "state": "done", "title": "Stamp develop SHA into release-readiness audits and gate stale audits in CI", "verifications": [{"command": "bash -lc 'set -e; tmp=$(mktemp _project/audits/_gate_test_XXXXX.md); printf -- \"---\\nid: gate-test\\ndate: 2026-05-08\\n---\\n# Stub\\n\" > \"$tmp\"; trap \"rm -f $tmp\" EXIT; ! make audit-sha-check FILE=\"$tmp\"'", "description": "Validator fails on an audit missing develop_sha", "expected": "non-zero exit (validator caught the missing field)", "item_id": "audit-develop-sha-stamping-and-ci-gate", "last_result": null, "last_run": null, "seq": 1}, {"command": "bash -lc 'set -e; sha=$(git rev-parse origin/develop); tmp=$(mktemp _project/audits/_gate_test_XXXXX.md); printf -- \"---\\nid: gate-test\\ndate: 2026-05-08\\ndevelop_sha: $sha\\n---\\n# Stub\\n\" > \"$tmp\"; trap \"rm -f $tmp\" EXIT; make audit-sha-check FILE=\"$tmp\"'", "description": "Validator passes on an audit with a real develop_sha", "expected": "exit 0", "item_id": "audit-develop-sha-stamping-and-ci-gate", "last_result": null, "last_run": null, "seq": 2}, {"command": "echo 'Manual: open a no-op docs PR and confirm audit-sha-check job is skipped via path filter'", "description": "CI gate runs only on PRs touching _project/audits/** or captures", "expected": "Job is reported as skipped on the PR checks tab", "item_id": "audit-develop-sha-stamping-and-ci-gate", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": [], "notes": "Specify in `docs/operations/results-explorer-qa.md` (smallest\ndoc-change footprint; broaden later when other audit owners\nadopt):\n - Required field name: `develop_sha:` (full 40-char SHA).\n - Required on every audit under `_project/audits/*.md` \u2014\n historical audits are backfilled in w3 so the validator can\n enforce the requirement uniformly without grandfathering.\n - Allowed staleness: configurable via CLI flag, default 0 (any\n ancestor of `origin/develop` is acceptable; reviewers may\n require `--require-current` for top-of-tree audits).\n - Backfill rule (consumed by w3): for each existing audit,\n derive the SHA from the introducing commit via `git log\n --diff-filter=A --reverse --format=%H -- | head\n -1`, then use that commit's parent on develop. Document this\n rule so future agents can re-derive it deterministically.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the develop_sha frontmatter contract", "wid": "w1"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w1"], "notes": "New file: `_project/scripts/audit_sha_check.py`.\nBehavior:\n - Parse the audit's YAML frontmatter; require `develop_sha:`\n unconditionally (no date cutoff after backfill in w3 lands).\n - Run `git merge-base --is-ancestor origin/develop`\n and exit non-zero if the SHA is not reachable.\n - With `--require-current N`, also require that the SHA is\n within N commits of `origin/develop` tip.\nAdd `make audit-sha-check FILE=` that wraps the script\nunder the existing `audit-*` target convention, and add the\ntarget to the pre-approved-commands allowlist in CLAUDE.md so\nagents can invoke it without prompts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement audit_sha_check.py validator + Makefile target", "wid": "w2"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w2"], "notes": "For each `_project/audits/*.md`:\n sha=$(git log --diff-filter=A --reverse --format=%H -- \"$f\" | head -1)\n # Verify the introducing commit is reachable from origin/develop;\n # if not (e.g., authored on a feature branch and squashed), fall\n # back to the squash-merge commit that landed it on develop.\n # If still unreachable, fall back to `git rev-parse $sha^` (the\n # parent before the audit was added) and document the deviation\n # inline as a frontmatter comment.\nAdd a bulk-backfill helper script (`_project/scripts/audit_sha_backfill.py`)\nso the operation is repeatable and reviewers can audit the\nderivation per file. The script must:\n - Not modify audits whose `develop_sha:` is already present.\n - Print the derivation source per file (\"introducing commit\",\n \"merge commit\", \"parent fallback\") to stdout so reviewers\n can spot edge cases.\n - Be idempotent: running it twice changes nothing.\nRun the backfill, eyeball the per-file derivation log, and land\nthe backfilled audits in the same PR as the validator so CI\nstarts green on the existing tree.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Backfill develop_sha into every existing _project/audits/*.md", "wid": "w3"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w2"], "notes": "Modify `results-explorer/e2e/captures/release-final.spec.ts` (and\nany sibling `*-final.spec.ts` drivers) to:\n - Run `git fetch --no-tags origin develop` and `git rev-parse\n origin/develop` at spec setup.\n - Inject the resulting SHA into the screenshot manifest under a\n new `develop_sha:` key.\n - Print a copy-paste-ready frontmatter snippet to stdout at\n spec teardown so the audit author drops it into the report.\nOut of scope: rewriting the capture spec's screenshot logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the develop SHA into the release-final capture flow", "wid": "w4"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w3", "w4"], "notes": "Reuse the explorer test job's path filter pattern. New CI step:\n - Triggers on PRs touching `_project/audits/**` or\n `results-explorer/e2e/captures/**`.\n - Runs `make audit-sha-check FILE=` for every\n modified audit file (loop over `git diff --name-only`).\n - Refuses the PR if any audit fails the validator.\nJob must remain a no-op for PRs that don't touch the path filter.\nPre-merge sanity check: with the backfill from w3 landed, run the\nvalidator across the entire `_project/audits/` tree from a fresh\nclone and confirm exit 0 before declaring CI green.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add path-aware CI hook on _project/audits/** and capture specs", "wid": "w5"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w5"], "notes": "Update `docs/operations/results-explorer-qa.md` with:\n - The `develop_sha:` field requirement (uniform \u2014 no cutoff).\n - The `make audit-sha-check` invocation.\n - The backfill derivation rule from w1 so future sweeps can\n repeat it on any newly-orphaned audits.\n - A note that the contract applies to every report under\n `_project/audits/`, not only retheme audits.\nAppend a `commits:` reference back to this TODO from the source\nblind-spot file via a new triage-log entry once the gate ships.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the contract and link from the source blind-spot", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "maintain an explicit mapping", "dont": "assume package-name == import-name", "why": "several deps don't match (pyyaml/yaml, psycopg2-binary/psycopg2, google-cloud-*/google.cloud.*)"}, {"INSTEAD": "keep deliverables and follow-ups separate", "dont": "conflate this audit with the quarterly upper-bounds review", "why": "they answer different questions (keep-or-drop vs. cap-is-correct)"}, {"INSTEAD": "parse `pyproject.toml` directly", "dont": "rely on `uv pip list` as the source of truth for declared deps", "why": "it includes transitives and masks which extras group owns a package"}, {"INSTEAD": "surface as follow-up TODO instead", "dont": "remove any dep as part of this item", "why": "each removal has its own test-surface and risk profile"}, {"INSTEAD": "verify before flagging", "dont": "treat 'no imports found' as automatic grounds for removal", "why": "plugin-style deps (e.g. ray for modin, some driver backends) can be imported dynamically via entry points"}], "approach": "Research-only work - no behavior changes to the codebase. Produce an\nevidence-backed inventory and a set of follow-up implementation TODOs;\nmaintainer decides which removals to authorize.\n\nMethodology:\n 1. Source of truth for declared deps: parse `pyproject.toml` directly\n rather than relying on `uv pip list` (which includes transitive).\n Use `tomllib` in a small throwaway script under `_project/scratch/`\n if regex gets tangled by multi-line strings + env markers.\n 2. Source of truth for import sites: `Grep` across the tree with the\n import-name (not package-name) for each dep. Maintain an explicit\n pkg\u2192import-name map - there are several mismatches (pyyaml/yaml,\n psycopg2-binary/psycopg2, pillow/PIL, google-cloud-*/google.cloud.*,\n beautifulsoup4/bs4 if present). Record the map in the inventory so\n future audits reuse it.\n 3. Transitive-reach classification uses `uv tree` output as ground\n truth. A dep is \"transitively available\" only if it appears in\n `uv tree` for a group that a live consumer module depends on.\n 4. Ownership: tie every live import back to its owning benchbox\n module (e.g. `snowflake-connector-python` \u2192 `benchbox/platforms/snowflake/`).\n An extras group with zero live consumers is a FLAG-DEAD-EXTRA.\n 5. Report format mirrors `docs/development/dependency-compatibility.md`\n so a reader can cross-reference cap rationale against usage.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "audit-imported-dependencies-for-elimination", "id": 105, "reason": "Out of scope for this audit - each removal is its own reviewable change with its own test surface. Tracked via the follow-up TODOs created in w7.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: w7 follow-up dep-removal TODOs already completed (drop-lxml-dev-dep, drop-chdb-core, move-jsonschema-out, clean-up-unused-sphinx-dev-deps, add-dependency-audit-ci-guard) -- action already done.", "summary": "Actually remove any flagged deps from `pyproject.toml` / `uv.lock`"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "audit-imported-dependencies-for-elimination", "id": 106, "reason": "Covered by review-dependency-upper-bounds-quarterly.yaml.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason states this is already covered by the quarterly upper-bounds review, which is a currently open item. (tracked by review-dependency-upper-bounds-quarterly)", "summary": "Bump upper bounds on kept deps"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "audit-imported-dependencies-for-elimination", "id": 107, "reason": "Renames are breaking changes for existing installers; would require a deprecation cycle. Surface the duplication in the inventory but defer the rename work.", "resolution": "promoted", "resolved_item": "revisit-extras-naming-topology-e-g-dataframe-pandas-vs-panda", "resolved_reason": null, "summary": "Revisit extras naming / topology (e.g. `dataframe-pandas` vs `pandas` aliases)"}], "deps": [], "description": "Systematic, evidence-based review of every dependency declared in the\nproject with the goal of pruning anything that is not absolutely required.\n\nMotivation:\n - `pyproject.toml` declares ~15 core deps and ~50 optional-extra groups\n covering SQL platforms, cloud storage, DataFrame engines, benchmarks,\n docs, and MCP. Extras have accreted organically as platforms were\n added and it is no longer self-evident which imports are live, which\n are dead, and which are redundant with something already pulled in\n transitively.\n - Supply chain surface: every declared dep is an install target, a\n vulnerability surface, and a version-resolution constraint. Dropping\n unused deps shrinks install size, speeds CI, and reduces the matrix\n the release gate has to keep green.\n - Complements (does NOT replace) `review-dependency-upper-bounds-quarterly.yaml`.\n That item watches caps on *kept* deps for breakage; this item asks the\n prior question: does the dep belong in the manifest at all?\n\nScope:\n - All `[project] dependencies`, `[project.optional-dependencies]`,\n `[dependency-groups]` in `pyproject.toml`.\n - Any `uv.sources` overrides, git pins, or platform markers.\n - Any runtime imports in `benchbox/`, `scripts/`, `tests/`, `docs/`\n that name a third-party package not declared in the manifest\n (the inverse check - declared-but-unused AND used-but-undeclared).\n\nNon-goals:\n - Not a version-bump exercise. Upper-bounds review is separate.\n - Not a rewrite of extras topology. Consolidation proposals are fine,\n but execution of breaking extras renames is out of scope and would\n be split into a follow-up.\n - Does NOT touch `_binaries/` bundled C++ binaries or dsdgen/dsqgen\n toolchain - those are artifacts, not Python deps.\n\nDeliverable:\n - A committed markdown inventory at\n `docs/development/dependency-inventory.md` listing every declared\n dep, its extras group(s), every source file that imports it, and\n a one-line justification for keeping it. Deps with no live imports\n or with justification \"covered by \" are flagged as\n elimination candidates.\n - A short follow-up TODO (or set of TODOs) proposing concrete\n removals, gated on maintainer sign-off. No deps are deleted under\n this item - only inventoried and flagged.", "id": "audit-imported-dependencies-for-elimination", "preserves": ["Existing `review-dependency-upper-bounds-quarterly.yaml` scope stays intact - this audit explicitly complements, does not subsume", "No imports added, removed, or reordered in `benchbox/` source as part of the audit", "`pyproject.toml` is NOT modified by this audit - only inventoried", "`scripts/check_dependency_bounds.py` behavior and CLI unchanged", "`uv.lock` is NOT regenerated as part of this work"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "_binaries/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml"}, {"kind": "do_not_modify", "path_glob": "scripts/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "do_not_modify", "path_glob": "uv.lock"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-audit-raw.md"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}], "state": "done", "title": "Audit all imported dependencies and eliminate any not strictly required", "verifications": [{"command": "uv run -- python -c \"\nimport tomllib, pathlib, re\ndata = tomllib.loads(pathlib.Path('pyproject.toml').read_text())\ndef name(s): return re.split(r'[><=!;@\\[]', s.strip())[0].strip().lower()\ndeps = set()\nfor s in data['project'].get('dependencies', []): deps.add(name(s))\nfor g in data['project'].get('optional-dependencies', {}).values():\n for s in g:\n if isinstance(s, str): deps.add(name(s))\nfor g in data.get('dependency-groups', {}).values():\n for s in g:\n if isinstance(s, str): deps.add(name(s))\ndeps.discard('')\ninv = pathlib.Path('docs/development/dependency-inventory.md').read_text().lower()\nmissing = [p for p in sorted(deps) if not re.search(r'\\\\b' + re.escape(p) + r'\\\\b', inv)]\nprint('MISSING:', missing) if missing else print(f'OK: all {len(deps)} deps present in inventory')\n\"", "description": "Inventory document exists and lists every dep declared in pyproject.toml", "expected": "OK: all N deps present in inventory", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 1}, {"command": "git diff --name-only HEAD -- benchbox/ | wc -l", "description": "No source files were modified under benchbox/", "expected": "0", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 2}, {"command": "git diff --name-only HEAD -- pyproject.toml uv.lock | wc -l", "description": "pyproject.toml and uv.lock are untouched by this work", "expected": "0", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"\nimport pathlib, sys\ninv = pathlib.Path('docs/development/dependency-inventory.md').read_text()\nflag_count = inv.count('FLAG-')\ntodos = list(pathlib.Path('_project/TODO/main/planning').glob('*dep*'))\nif flag_count == 0: sys.exit('ERROR: no FLAG-* entries found in inventory')\nif not todos: sys.exit('ERROR: no follow-up dep TODOs found in planning/')\nprint(f'OK: {flag_count} FLAG-* entries, {len(todos)} follow-up TODO(s)')\n\"", "description": "Inventory contains FLAG-* entries and at least one follow-up dep TODO exists in planning/", "expected": "OK: N FLAG-* entries, M follow-up TODO(s)", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": [], "notes": "Parse `pyproject.toml` directly (tomllib) - do not use `uv pip list`\nbecause it surfaces transitives. Include env markers (python_version,\nsys_platform) in the captured row. Write raw table to\n`docs/development/dependency-audit-raw.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enumerate every declared dep from pyproject.toml (core, all extras, dependency-groups, tool.uv.sources). Capture name, version spec, owning group(s), inline rationale.", "wid": "w1"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w1"], "notes": "Handle package-name vs import-name mismatches: pyyaml\u2192yaml,\npsycopg2-binary\u2192psycopg2, google-cloud-bigquery\u2192google.cloud.bigquery,\nbeautifulsoup4\u2192bs4, pillow\u2192PIL, etc. Maintain the pkg\u2192import map in\nthe inventory so future audits reuse it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build import-site map for each declared dep: Grep `import ` / `from ` across benchbox/, scripts/, tests/, docs/. Record every file:line that imports each dep.", "wid": "w2"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w1"], "notes": "Ground truth for transitive reach is `uv tree`. Any \"transitive-reach\"\nfinding is fragile and becomes a candidate for promotion to an explicit\ndep or removal of the dependent code path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inverse check: find third-party imports in source that are NOT declared in pyproject.toml. Classify as (a) truly undeclared, (b) transitive-reach, or (c) stdlib misidentified.", "wid": "w3"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w2"], "notes": "Flag deps declared across multiple extras groups where one group\nalready transitively includes the other - consolidation candidates.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify each declared dep by purpose (Core / SQL-adapter / Cloud-storage / DataFrame / Benchmark / Dev-test / Docs / MCP) and identify the owning benchbox module(s).", "wid": "w4"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w3", "w4"], "notes": "Evidence = import-site count from w2, owner module from w4, overlap\nsource for REDUNDANT. Verify plugin-style deps (ray for modin, entry-\npoint-loaded backends) before flagging UNUSED.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Produce elimination candidate list per dep: KEEP, FLAG-UNUSED, FLAG-REDUNDANT, or FLAG-DEAD-EXTRA. Each FLAG entry gets a one-line justification + evidence.", "wid": "w5"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write docs/development/dependency-inventory.md - inventory table, elimination candidates with evidence, consolidation proposals, used-but-undeclared findings. Cross-link from AGENTS.md.", "wid": "w6"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create follow-up TODOs for each actionable finding - one per elimination/consolidation cluster. Each follow-up carries its own verification and must_preserve. This item does not perform removals.", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "backfill develop_sha by hand", "instead": "drive the backfill from `_project/scripts/audit_sha_backfill.py` so the derivation is auditable and idempotent.", "why": "per-file judgment calls about which commit to use will drift over time"}, {"dont": "extend the validator to lint audit content (markdown, headings, sections)", "instead": "content-quality checks belong in a separate audit-review TODO.", "why": "the validator must stay narrow to remain trustworthy as a CI gate"}, {"dont": "land the validator before the backfill", "instead": "sequence is w2 (validator) \u2192 w3 (backfill) \u2192 w5 (CI gate) in the same PR series, with w3's output landing before w5 enables enforcement.", "why": "CI would fail on the existing tree from day one"}, {"dont": "make the gate run on every CI build", "instead": "use a path filter on `_project/audits/**` and `results-explorer/e2e/captures/**` so non-audit PRs see no overhead.", "why": "most PRs do not touch audits"}], "approach": "Build the validator before touching anything else: w2 makes the\ncontract executable. Then w3 backfills the entire existing tree so\nCI starts green and the validator can enforce uniformly without a\ndate cutoff. w4 wires the capture flow for new audits, w5 enforces\nvia CI, w6 documents. Keep the validator narrow (no markdown\nlinting, no audit-content checks) so it stays trustworthy as a CI\ngate. Reuse the existing explorer-job path filter pattern in w5\nrather than authoring a new workflow from scratch \u2014 that pattern\nis already proven and keeps CI cost flat for non-audit PRs.\n", "blocked_reason": null, "category": "Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-08T00:00:00Z", "completed_pr": null, "created_at": "2026-05-08T00:00:00Z", "deferrals": [], "deps": ["results-explorer-token-scan-ci-gate"], "description": "Release-readiness reports under `_project/audits/` describe a tree. If\nthe tree the report describes is not the tree that ships, the verdict\nis structurally unreliable \u2014 even when every individual check passes.\nPR #269 demonstrated this: a release-readiness report and capture\ngenerated against the PR branch were squash-merged onto a develop tip\nthat included PR #268's literal-styled `Result summary` panel and\n`Compare guardrails` section, and the report still claimed READY.\nPR #271 (commit `274a4a11e`) closed the immediate token regression\nmanually, but the framework gap remains.\n\nThis TODO adds a durable, audit-agnostic gate so the same class of\nregression is caught at PR time across every structured audit, not\nonly retheme work:\n\n1. Every `_project/audits/*.md` audit landing after the cutoff date\n must include a `develop_sha:` frontmatter field naming the develop\n SHA the report describes.\n2. The capture flow that authors release-final reports\n (`results-explorer/e2e/captures/release-final.spec.ts` and any\n sibling `*-final.spec.ts` drivers) records the develop SHA at\n run-start so the human author copies it into the audit\n frontmatter.\n3. A small validator (`_project/scripts/audit_sha_check.py`) and a\n `make audit-sha-check FILE=\u2026` target verify the field is present,\n resolves to a SHA reachable from `origin/develop`, and is not too\n far behind the merge target's tip.\n4. A path-aware CI hook on `_project/audits/**` and\n `results-explorer/e2e/captures/**` runs the validator on every PR\n and refuses to merge a release-readiness audit whose recorded SHA\n is missing or stale relative to the merge target.\n\nComplements (does not replace)\n`results-explorer-token-scan-ci-gate.yaml`: that gate prevents the\nliteral-color regression class from shipping; this gate makes every\nrelease-readiness audit's tree-claim auditable so future cross-PR\nraces on any surface (security, performance, docs migration) are\ncaught at the audit-author step rather than after merge.", "id": "audit-develop-sha-stamping-and-ci-gate", "preserves": ["Backfill must be idempotent \u2014 re-running the helper on already-stamped audits is a no-op.", "CI required-result aggregator path-filter behavior.", "Existing _project/audits/*.md content \u2014 backfill only adds the `develop_sha:` frontmatter field; body text and other frontmatter fields stay byte-identical.", "Existing pre-commit hooks and the `make lint` umbrella target."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/"}, {"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "CLAUDE.md"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/audits/"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-07-211858-release-readiness-against-pre-merge-tree.md"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/operations/"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/captures/"}], "state": "done", "title": "Stamp develop SHA into release-readiness audits and gate stale audits in CI", "verifications": [{"command": "bash -lc 'set -e; tmp=$(mktemp _project/audits/_gate_test_XXXXX.md); printf -- \"---\\nid: gate-test\\ndate: 2026-05-08\\n---\\n# Stub\\n\" > \"$tmp\"; trap \"rm -f $tmp\" EXIT; ! make audit-sha-check FILE=\"$tmp\"'", "description": "Validator fails on an audit missing develop_sha", "expected": "non-zero exit (validator caught the missing field)", "item_id": "audit-develop-sha-stamping-and-ci-gate", "last_result": null, "last_run": null, "seq": 1}, {"command": "bash -lc 'set -e; sha=$(git rev-parse origin/develop); tmp=$(mktemp _project/audits/_gate_test_XXXXX.md); printf -- \"---\\nid: gate-test\\ndate: 2026-05-08\\ndevelop_sha: $sha\\n---\\n# Stub\\n\" > \"$tmp\"; trap \"rm -f $tmp\" EXIT; make audit-sha-check FILE=\"$tmp\"'", "description": "Validator passes on an audit with a real develop_sha", "expected": "exit 0", "item_id": "audit-develop-sha-stamping-and-ci-gate", "last_result": null, "last_run": null, "seq": 2}, {"command": "echo 'Manual: open a no-op docs PR and confirm audit-sha-check job is skipped via path filter'", "description": "CI gate runs only on PRs touching _project/audits/** or captures", "expected": "Job is reported as skipped on the PR checks tab", "item_id": "audit-develop-sha-stamping-and-ci-gate", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": [], "notes": "Specify in `docs/operations/results-explorer-qa.md` (smallest\ndoc-change footprint; broaden later when other audit owners\nadopt):\n - Required field name: `develop_sha:` (full 40-char SHA).\n - Required on every audit under `_project/audits/*.md` \u2014\n historical audits are backfilled in w3 so the validator can\n enforce the requirement uniformly without grandfathering.\n - Allowed staleness: configurable via CLI flag, default 0 (any\n ancestor of `origin/develop` is acceptable; reviewers may\n require `--require-current` for top-of-tree audits).\n - Backfill rule (consumed by w3): for each existing audit,\n derive the SHA from the introducing commit via `git log\n --diff-filter=A --reverse --format=%H -- | head\n -1`, then use that commit's parent on develop. Document this\n rule so future agents can re-derive it deterministically.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the develop_sha frontmatter contract", "wid": "w1"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w1"], "notes": "New file: `_project/scripts/audit_sha_check.py`.\nBehavior:\n - Parse the audit's YAML frontmatter; require `develop_sha:`\n unconditionally (no date cutoff after backfill in w3 lands).\n - Run `git merge-base --is-ancestor origin/develop`\n and exit non-zero if the SHA is not reachable.\n - With `--require-current N`, also require that the SHA is\n within N commits of `origin/develop` tip.\nAdd `make audit-sha-check FILE=` that wraps the script\nunder the existing `audit-*` target convention, and add the\ntarget to the pre-approved-commands allowlist in CLAUDE.md so\nagents can invoke it without prompts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement audit_sha_check.py validator + Makefile target", "wid": "w2"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w2"], "notes": "For each `_project/audits/*.md`:\n sha=$(git log --diff-filter=A --reverse --format=%H -- \"$f\" | head -1)\n # Verify the introducing commit is reachable from origin/develop;\n # if not (e.g., authored on a feature branch and squashed), fall\n # back to the squash-merge commit that landed it on develop.\n # If still unreachable, fall back to `git rev-parse $sha^` (the\n # parent before the audit was added) and document the deviation\n # inline as a frontmatter comment.\nAdd a bulk-backfill helper script (`_project/scripts/audit_sha_backfill.py`)\nso the operation is repeatable and reviewers can audit the\nderivation per file. The script must:\n - Not modify audits whose `develop_sha:` is already present.\n - Print the derivation source per file (\"introducing commit\",\n \"merge commit\", \"parent fallback\") to stdout so reviewers\n can spot edge cases.\n - Be idempotent: running it twice changes nothing.\nRun the backfill, eyeball the per-file derivation log, and land\nthe backfilled audits in the same PR as the validator so CI\nstarts green on the existing tree.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Backfill develop_sha into every existing _project/audits/*.md", "wid": "w3"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w2"], "notes": "Modify `results-explorer/e2e/captures/release-final.spec.ts` (and\nany sibling `*-final.spec.ts` drivers) to:\n - Run `git fetch --no-tags origin develop` and `git rev-parse\n origin/develop` at spec setup.\n - Inject the resulting SHA into the screenshot manifest under a\n new `develop_sha:` key.\n - Print a copy-paste-ready frontmatter snippet to stdout at\n spec teardown so the audit author drops it into the report.\nOut of scope: rewriting the capture spec's screenshot logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the develop SHA into the release-final capture flow", "wid": "w4"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w3", "w4"], "notes": "Reuse the explorer test job's path filter pattern. New CI step:\n - Triggers on PRs touching `_project/audits/**` or\n `results-explorer/e2e/captures/**`.\n - Runs `make audit-sha-check FILE=` for every\n modified audit file (loop over `git diff --name-only`).\n - Refuses the PR if any audit fails the validator.\nJob must remain a no-op for PRs that don't touch the path filter.\nPre-merge sanity check: with the backfill from w3 landed, run the\nvalidator across the entire `_project/audits/` tree from a fresh\nclone and confirm exit 0 before declaring CI green.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add path-aware CI hook on _project/audits/** and capture specs", "wid": "w5"}, {"evidence": null, "item_id": "audit-develop-sha-stamping-and-ci-gate", "needs": ["w5"], "notes": "Update `docs/operations/results-explorer-qa.md` with:\n - The `develop_sha:` field requirement (uniform \u2014 no cutoff).\n - The `make audit-sha-check` invocation.\n - The backfill derivation rule from w1 so future sweeps can\n repeat it on any newly-orphaned audits.\n - A note that the contract applies to every report under\n `_project/audits/`, not only retheme audits.\nAppend a `commits:` reference back to this TODO from the source\nblind-spot file via a new triage-log entry once the gate ships.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the contract and link from the source blind-spot", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "assume package-name == import-name", "instead": "maintain an explicit mapping", "why": "several deps don't match (pyyaml/yaml, psycopg2-binary/psycopg2, google-cloud-*/google.cloud.*)"}, {"dont": "conflate this audit with the quarterly upper-bounds review", "instead": "keep deliverables and follow-ups separate", "why": "they answer different questions (keep-or-drop vs. cap-is-correct)"}, {"dont": "rely on `uv pip list` as the source of truth for declared deps", "instead": "parse `pyproject.toml` directly", "why": "it includes transitives and masks which extras group owns a package"}, {"dont": "remove any dep as part of this item", "instead": "surface as follow-up TODO instead", "why": "each removal has its own test-surface and risk profile"}, {"dont": "treat 'no imports found' as automatic grounds for removal", "instead": "verify before flagging", "why": "plugin-style deps (e.g. ray for modin, some driver backends) can be imported dynamically via entry points"}], "approach": "Research-only work - no behavior changes to the codebase. Produce an\nevidence-backed inventory and a set of follow-up implementation TODOs;\nmaintainer decides which removals to authorize.\n\nMethodology:\n 1. Source of truth for declared deps: parse `pyproject.toml` directly\n rather than relying on `uv pip list` (which includes transitive).\n Use `tomllib` in a small throwaway script under `_project/scratch/`\n if regex gets tangled by multi-line strings + env markers.\n 2. Source of truth for import sites: `Grep` across the tree with the\n import-name (not package-name) for each dep. Maintain an explicit\n pkg\u2192import-name map - there are several mismatches (pyyaml/yaml,\n psycopg2-binary/psycopg2, pillow/PIL, google-cloud-*/google.cloud.*,\n beautifulsoup4/bs4 if present). Record the map in the inventory so\n future audits reuse it.\n 3. Transitive-reach classification uses `uv tree` output as ground\n truth. A dep is \"transitively available\" only if it appears in\n `uv tree` for a group that a live consumer module depends on.\n 4. Ownership: tie every live import back to its owning benchbox\n module (e.g. `snowflake-connector-python` \u2192 `benchbox/platforms/snowflake/`).\n An extras group with zero live consumers is a FLAG-DEAD-EXTRA.\n 5. Report format mirrors `docs/development/dependency-compatibility.md`\n so a reader can cross-reference cap rationale against usage.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "audit-imported-dependencies-for-elimination", "id": 105, "reason": "Out of scope for this audit - each removal is its own reviewable change with its own test surface. Tracked via the follow-up TODOs created in w7.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: w7 follow-up dep-removal TODOs already completed (drop-lxml-dev-dep, drop-chdb-core, move-jsonschema-out, clean-up-unused-sphinx-dev-deps, add-dependency-audit-ci-guard) -- action already done.", "summary": "Actually remove any flagged deps from `pyproject.toml` / `uv.lock`"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "audit-imported-dependencies-for-elimination", "id": 106, "reason": "Covered by review-dependency-upper-bounds-quarterly.yaml.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason states this is already covered by the quarterly upper-bounds review, which is a currently open item. (tracked by review-dependency-upper-bounds-quarterly)", "summary": "Bump upper bounds on kept deps"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "audit-imported-dependencies-for-elimination", "id": 107, "reason": "Renames are breaking changes for existing installers; would require a deprecation cycle. Surface the duplication in the inventory but defer the rename work.", "resolution": "promoted", "resolved_item": "revisit-extras-naming-topology-e-g-dataframe-pandas-vs-panda", "resolved_reason": null, "summary": "Revisit extras naming / topology (e.g. `dataframe-pandas` vs `pandas` aliases)"}], "deps": [], "description": "Systematic, evidence-based review of every dependency declared in the\nproject with the goal of pruning anything that is not absolutely required.\n\nMotivation:\n - `pyproject.toml` declares ~15 core deps and ~50 optional-extra groups\n covering SQL platforms, cloud storage, DataFrame engines, benchmarks,\n docs, and MCP. Extras have accreted organically as platforms were\n added and it is no longer self-evident which imports are live, which\n are dead, and which are redundant with something already pulled in\n transitively.\n - Supply chain surface: every declared dep is an install target, a\n vulnerability surface, and a version-resolution constraint. Dropping\n unused deps shrinks install size, speeds CI, and reduces the matrix\n the release gate has to keep green.\n - Complements (does NOT replace) `review-dependency-upper-bounds-quarterly.yaml`.\n That item watches caps on *kept* deps for breakage; this item asks the\n prior question: does the dep belong in the manifest at all?\n\nScope:\n - All `[project] dependencies`, `[project.optional-dependencies]`,\n `[dependency-groups]` in `pyproject.toml`.\n - Any `uv.sources` overrides, git pins, or platform markers.\n - Any runtime imports in `benchbox/`, `scripts/`, `tests/`, `docs/`\n that name a third-party package not declared in the manifest\n (the inverse check - declared-but-unused AND used-but-undeclared).\n\nNon-goals:\n - Not a version-bump exercise. Upper-bounds review is separate.\n - Not a rewrite of extras topology. Consolidation proposals are fine,\n but execution of breaking extras renames is out of scope and would\n be split into a follow-up.\n - Does NOT touch `_binaries/` bundled C++ binaries or dsdgen/dsqgen\n toolchain - those are artifacts, not Python deps.\n\nDeliverable:\n - A committed markdown inventory at\n `docs/development/dependency-inventory.md` listing every declared\n dep, its extras group(s), every source file that imports it, and\n a one-line justification for keeping it. Deps with no live imports\n or with justification \"covered by \" are flagged as\n elimination candidates.\n - A short follow-up TODO (or set of TODOs) proposing concrete\n removals, gated on maintainer sign-off. No deps are deleted under\n this item - only inventoried and flagged.", "id": "audit-imported-dependencies-for-elimination", "preserves": ["Existing `review-dependency-upper-bounds-quarterly.yaml` scope stays intact - this audit explicitly complements, does not subsume", "No imports added, removed, or reordered in `benchbox/` source as part of the audit", "`pyproject.toml` is NOT modified by this audit - only inventoried", "`scripts/check_dependency_bounds.py` behavior and CLI unchanged", "`uv.lock` is NOT regenerated as part of this work"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "_binaries/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml"}, {"kind": "do_not_modify", "path_glob": "scripts/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "do_not_modify", "path_glob": "uv.lock"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-audit-raw.md"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}], "state": "done", "title": "Audit all imported dependencies and eliminate any not strictly required", "verifications": [{"command": "uv run -- python -c \"\nimport tomllib, pathlib, re\ndata = tomllib.loads(pathlib.Path('pyproject.toml').read_text())\ndef name(s): return re.split(r'[><=!;@\\[]', s.strip())[0].strip().lower()\ndeps = set()\nfor s in data['project'].get('dependencies', []): deps.add(name(s))\nfor g in data['project'].get('optional-dependencies', {}).values():\n for s in g:\n if isinstance(s, str): deps.add(name(s))\nfor g in data.get('dependency-groups', {}).values():\n for s in g:\n if isinstance(s, str): deps.add(name(s))\ndeps.discard('')\ninv = pathlib.Path('docs/development/dependency-inventory.md').read_text().lower()\nmissing = [p for p in sorted(deps) if not re.search(r'\\\\b' + re.escape(p) + r'\\\\b', inv)]\nprint('MISSING:', missing) if missing else print(f'OK: all {len(deps)} deps present in inventory')\n\"", "description": "Inventory document exists and lists every dep declared in pyproject.toml", "expected": "OK: all N deps present in inventory", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 1}, {"command": "git diff --name-only HEAD -- benchbox/ | wc -l", "description": "No source files were modified under benchbox/", "expected": "0", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 2}, {"command": "git diff --name-only HEAD -- pyproject.toml uv.lock | wc -l", "description": "pyproject.toml and uv.lock are untouched by this work", "expected": "0", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"\nimport pathlib, sys\ninv = pathlib.Path('docs/development/dependency-inventory.md').read_text()\nflag_count = inv.count('FLAG-')\ntodos = list(pathlib.Path('_project/TODO/main/planning').glob('*dep*'))\nif flag_count == 0: sys.exit('ERROR: no FLAG-* entries found in inventory')\nif not todos: sys.exit('ERROR: no follow-up dep TODOs found in planning/')\nprint(f'OK: {flag_count} FLAG-* entries, {len(todos)} follow-up TODO(s)')\n\"", "description": "Inventory contains FLAG-* entries and at least one follow-up dep TODO exists in planning/", "expected": "OK: N FLAG-* entries, M follow-up TODO(s)", "item_id": "audit-imported-dependencies-for-elimination", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": [], "notes": "Parse `pyproject.toml` directly (tomllib) - do not use `uv pip list`\nbecause it surfaces transitives. Include env markers (python_version,\nsys_platform) in the captured row. Write raw table to\n`docs/development/dependency-audit-raw.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enumerate every declared dep from pyproject.toml (core, all extras, dependency-groups, tool.uv.sources). Capture name, version spec, owning group(s), inline rationale.", "wid": "w1"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w1"], "notes": "Handle package-name vs import-name mismatches: pyyaml\u2192yaml,\npsycopg2-binary\u2192psycopg2, google-cloud-bigquery\u2192google.cloud.bigquery,\nbeautifulsoup4\u2192bs4, pillow\u2192PIL, etc. Maintain the pkg\u2192import map in\nthe inventory so future audits reuse it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build import-site map for each declared dep: Grep `import ` / `from ` across benchbox/, scripts/, tests/, docs/. Record every file:line that imports each dep.", "wid": "w2"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w1"], "notes": "Ground truth for transitive reach is `uv tree`. Any \"transitive-reach\"\nfinding is fragile and becomes a candidate for promotion to an explicit\ndep or removal of the dependent code path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inverse check: find third-party imports in source that are NOT declared in pyproject.toml. Classify as (a) truly undeclared, (b) transitive-reach, or (c) stdlib misidentified.", "wid": "w3"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w2"], "notes": "Flag deps declared across multiple extras groups where one group\nalready transitively includes the other - consolidation candidates.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify each declared dep by purpose (Core / SQL-adapter / Cloud-storage / DataFrame / Benchmark / Dev-test / Docs / MCP) and identify the owning benchbox module(s).", "wid": "w4"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w3", "w4"], "notes": "Evidence = import-site count from w2, owner module from w4, overlap\nsource for REDUNDANT. Verify plugin-style deps (ray for modin, entry-\npoint-loaded backends) before flagging UNUSED.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Produce elimination candidate list per dep: KEEP, FLAG-UNUSED, FLAG-REDUNDANT, or FLAG-DEAD-EXTRA. Each FLAG entry gets a one-line justification + evidence.", "wid": "w5"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write docs/development/dependency-inventory.md - inventory table, elimination candidates with evidence, consolidation proposals, used-but-undeclared findings. Cross-link from AGENTS.md.", "wid": "w6"}, {"evidence": null, "item_id": "audit-imported-dependencies-for-elimination", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create follow-up TODOs for each actionable finding - one per elimination/consolidation cluster. Each follow-up carries its own verification and must_preserve. This item does not perform removals.", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: Completed\"\n- \"**Description**: Automatically compile TPC-H (`qgen`, `dbgen`) and TPC-DS (`dsdgen`, `dsqgen`) binaries when source code is present but compiled files are missing\"\n- \"**Tasks**:\"\n- \"**Implementation Summary**:\"\n- \"**Files Created/Modified**:\"\n- \"**Impact**: Successfully reduces setup friction for users with TPC source code, eliminates need for manual compilation steps, improves developer experience with automatic tool preparation while maintaining backward compatibility\"", "id": "auto-compile-tpc-binary-tools-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Auto-Compile TPC Binary Tools \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:41Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #509 of uat-resume-shard-config-sprawl.\nDeferred: Auto-expire or auto-archive generated rerun shards\nReason deferred: Tooling automation is a follow-up; the lifecycle policy + relocation is the fix.", "id": "auto-expire-or-auto-archive-generated-rerun-shards", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Auto-expire or auto-archive generated rerun shards", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:43Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #511 of uat-single-connection-registry.\nDeferred: Auto-generate compose port mappings from adapter defaults\nReason deferred: Generating compose from adapters is a larger infra change; deriving the read path is the fix.", "id": "auto-generate-compose-port-mappings-from-adapter-defaults", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Auto-generate compose port mappings from adapter defaults", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "fork auto_merge_soundness_paths.py's logic into the workflow YAML as inline shell/python", "why": "keep executing the actual script file (from the base ref) so there is exactly one implementation to maintain."}, {"INSTEAD": "(not specified)", "dont": "skip the base-branch fetch step's existing --no-tags behavior or change its shallow/full-history semantics; only add the base-ref file materialization after it.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "treat this as a substitute for the ruleset-layer review requirement", "why": "this is documented as defense-in-depth in the description; don't let this item's success_metrics imply it replaces auto-merge-review-gate-admin-enforcement."}], "approach": null, "blocked_reason": null, "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": ["soundness-surface-widening"], "description": "`.github/workflows/auto-merge-on-open.yml` (lines 31-48: the checkout, base-branch\nfetch, and \"Check soundness-critical paths\" step) checks out the PR at its own\nHEAD and then runs `_project/scripts/auto_merge_soundness_paths.py` *from that\nsame checkout*. For a same-repo (non-fork) PR \u2014 the normal case for this\nsingle-maintainer-owned repo where contributors push branches directly \u2014 the\nscript executing the predicate is whatever version the PR itself introduces. A\nPR that edits `auto_merge_soundness_paths.py` (e.g. narrows `SOUNDNESS_PREFIXES`,\nor the workflow file itself) evaluates its own diff against its own modified\nrules, so it can neutralize its review gate in the same commit that would\notherwise have triggered it.\n\nNote honestly: with the repo-layer ruleset review requirement from\nauto-merge-review-gate-admin-enforcement enforced (once its deferred admin\naction lands), this workflow's withholding behavior is defense-in-depth, not\nthe primary control \u2014 the ruleset's `pull_request` rule would block a\nzero-approval squash-merge regardless of what this workflow decides. This item\nis still worth doing because (a) the ruleset enforcement is not yet live, and\n(b) even once it is, a same-commit self-neutralization that quietly avoids\never showing \"review required\" is a bad signal for reviewers relying on the\nauto-merge state as a triage cue.", "id": "auto-merge-predicate-base-ref-execution", "preserves": ["The synchronize-event re-evaluation behavior (a later push that newly touches a soundness path flips auto-merge back off) continues to work.", "The workflow's output contract (steps.soundness.outputs.soundness_path, consumed by the existing enable/disable auto-merge steps) is unchanged.", "make pr-open's local invocation of the predicate is unaffected by this item (base-ref execution is CI-workflow-specific; a local developer already has full repo access)."], "prior_art": [{"concept": ".github/workflows/auto-merge-on-open.yml:31-48 \u2014 the checkout + soundness-check step being hardened; keep its step names and output contract (steps.soundness.outputs.soundness_path) stable so the enable/disable steps below it don't need changes.", "decision": "reuse", "path": ".github/workflows/auto-merge-on-open.yml:31-48"}, {"concept": "Makefile:pr-open (~line 1163) \u2014 also invokes the predicate via `uv run --project _project/scripts`; out of scope for this item's local-CLI flow (a developer running make pr-open locally already has arbitrary code execution), but keep the two invocations' semantics compatible.", "decision": "reuse", "path": "Makefile:pr-open"}, {"concept": "_project/scripts/auto_merge_soundness_paths.py \u2014 the predicate being protected; do not fork or duplicate its logic, only change which copy of the file gets executed.", "decision": "reuse", "path": "_project/scripts/auto_merge_soundness_paths.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "_project/scripts/auto_merge_soundness_paths.py"}, {"kind": "only_modify", "path_glob": ".github/workflows/auto-merge-on-open.yml"}, {"kind": "only_modify", "path_glob": "tests/unit/test_auto_merge_soundness_paths.py"}], "state": "done", "title": "Run the auto-merge soundness predicate from the base ref, not the PR's own checkout", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_auto_merge_soundness_paths.py -v", "description": "Backstop workflow tests still pass with the added base-ref and self-touch assertions", "expected": "all tests pass, including the extended test_backstop_workflow_uses_shared_predicate_and_skips_auto_merge", "item_id": "auto-merge-predicate-base-ref-execution", "last_result": null, "last_run": null, "seq": 1}, {"command": "gh run view --log | grep -A3 'Check soundness-critical paths'", "description": "A workflow_dispatch or manual trace confirms /tmp/predicate.py is materialized from origin/develop, not the PR's HEAD copy, before the predicate runs", "expected": "log shows the git show origin/${{ github.base_ref }} step succeeding before the predicate invocation", "item_id": "auto-merge-predicate-base-ref-execution", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "auto-merge-predicate-base-ref-execution", "needs": [], "notes": "In the \"Check soundness-critical paths\" step, before running the predicate,\nmaterialize the base-ref version: `git show\n\"origin/${{ github.base_ref }}:_project/scripts/auto_merge_soundness_paths.py\"\n> /tmp/predicate.py` (base branch is already fetched by the preceding\n\"Fetch base branch\" step), then invoke `python /tmp/predicate.py --stdin\n--format github-output` instead of the checked-out repo path. Confirm the\nscript has no relative imports or sibling-file dependencies that would\nbreak when run standalone from /tmp (check the top of\nauto_merge_soundness_paths.py \u2014 it currently has none, keep it that way or\ncopy siblings too if that changes).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fetch and execute the base-ref copy of the predicate script instead of the PR's own checkout copy", "wid": "w1"}, {"evidence": null, "item_id": "auto-merge-predicate-base-ref-execution", "needs": ["w1"], "notes": "Add an explicit check: if the PR's diff touches\n_project/scripts/auto_merge_soundness_paths.py or\n.github/workflows/auto-merge-on-open.yml, force soundness_path=true\nregardless of what the base-ref predicate returns for the rest of the\ndiff. This closes the residual gap where a PR could still edit the\npredicate to make future PRs unsafe, without itself needing owner review\nunder the base-ref-only check (base-ref execution alone stops the\npredicate from judging its own diff leniently, but does not by itself\nflag \"this diff touches the predicate\" as inherently review-worthy).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Treat modification of the predicate or the workflow itself as soundness=true", "wid": "w2"}, {"evidence": null, "item_id": "auto-merge-predicate-base-ref-execution", "needs": ["w1", "w2"], "notes": "Extend tests/unit/test_auto_merge_soundness_paths.py's\ntest_backstop_workflow_uses_shared_predicate_and_skips_auto_merge (which\ncurrently string-matches the workflow YAML) with assertions for the new\n`git show origin/.../auto_merge_soundness_paths.py` line and the\npredicate/workflow self-touch check, so a future edit that removes the\nbase-ref execution regresses a fast unit test rather than only being\ncaught by manual review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update or add a workflow-level test exercising the base-ref execution path", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "duplicate the soundness path list; the meta-check must read it from auto_merge_soundness_paths.py.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "mark this Completed while the ruleset still shows review_count 0 / code-owner review false", "why": "that repeats the overstated DONE metric."}], "approach": null, "blocked_reason": null, "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "auto-merge-review-gate-admin-enforcement", "id": 343, "reason": "Requires a repo admin (gh api PUT on the ruleset); cannot be done by an agent PR. Tracked as the gating human action for w3.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: CLAUDE.md states this exact require_code_owner_review ruleset target was retired 2026-07-18; maintainer-manual-merge path replaces it.", "summary": "Apply the develop ruleset change: require_code_owner_review=true."}], "deps": [], "description": "Follow-up to auto-merge-review-gate-soundness-paths (PR #912, already DONE). The\nadversarial holistic-review found the gate is only half-real in production. The\ncode side is correct and sound: pr-open (Makefile:1161) and the\nauto-merge-on-open.yml workflow share one predicate\n(_project/scripts/auto_merge_soundness_paths.py, invoked with --no-renames) and\ncorrectly WITHHOLD auto-merge enablement for PRs touching comparator/plan-parser\nsoundness paths. But nothing ENFORCES a review: the develop ruleset\n`develop-squash-only` has required_approving_review_count: 0 and\nrequire_code_owner_review: false (verified via `gh api repos/.../rules/branches/develop`;\nlegacy branch protection returns 404 \"Branch not protected\"). So a soundness-path\nPR can still be squash-merged with zero approvals \u2014 manually or by re-enabling\nauto-merge. The DONE item's success_metric (\"PRs touching the comparator or plan\nparsers cannot squash-auto-merge without a review\") is NOT met at the repo level.\n\nTwo parts: (1) a CI meta-check that makes the gap LOUD and self-healing so it\ncannot silently regress once set; (2) the actual ruleset change, which requires\na repo admin and is tracked here as the gating human action.", "id": "auto-merge-review-gate-admin-enforcement", "preserves": ["Non-soundness-path PRs keep squash auto-merge on CI-pass; this item only hardens the soundness exception.", "The code-side withholding behavior (pr-open + auto-merge-on-open.yml shared predicate, --no-renames) is unchanged."], "prior_art": [{"concept": "{'ref': '.github/workflows/pr.yml:270', 'note': \"Existing 'Oracle coverage map drift check' / drift-check step pattern in the lint job \u2014 model the ruleset meta-check on these gh-api-backed drift steps.\"}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'CODEOWNERS', 'note': 'Owns the soundness globs (parsers/**, equivalence/**, **/validation.py); require_code_owner_review only triggers if an owner team/user is actually assigned to those paths \u2014 verify, do not assume.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '_project/scripts/auto_merge_soundness_paths.py', 'note': 'The shared soundness-path predicate; the meta-check should reuse its SOUNDNESS_PREFIXES so the CI assertion and the auto-merge withholding agree on one source of truth.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/pr.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/auto-merge-review-gate-soundness-paths.yaml"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/unit/release/"}], "state": "done", "title": "Make the soundness-path review gate enforceable at the repo layer, not just code", "verifications": [{"command": "uv run -- python -m pytest tests/unit/release/ -k ruleset_review_enforcement -q", "description": "Meta-check fails on an unenforced ruleset and passes on an enforced one", "expected": "asserts the check fails when review_count<1 or code-owner review false; passes otherwise", "item_id": "auto-merge-review-gate-admin-enforcement", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": [], "notes": "Capture `gh api repos/joeharris76/BenchBox/rulesets` and the develop ruleset\ndetail; record required_approving_review_count and require_code_owner_review.\nConfirm CODEOWNERS assigns a real owner to parsers/**, equivalence/**,\n**/validation.py. Summarize (values + glob->owner) in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm live ruleset state and CODEOWNERS ownership of soundness globs", "wid": "w0"}, {"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": ["w0"], "notes": "Add a step in an ALWAYS-required job (e.g. pr.yml lint / code-lint), never\ncontent-guard: content-guard is gated by content-guard-needed and skipped for\ncode-only PRs (ci-required-result ignores it unless content paths changed), so\na check placed there would let a code-only PR pass while review enforcement is\nabsent - defeating this item's stated self-healing guarantee. The step queries\nthe live develop ruleset via gh api and FAILS if require_code_owner_review !=\ntrue. required_approving_review_count is deliberately NOT part of this check:\nit is branch-wide, not CODEOWNERS-scoped, so asserting it here would gate\nevery develop PR rather than just soundness-path ones (see\nruleset_review_enforcement.py's module docstring). Reuse\nauto_merge_soundness_paths.py SOUNDNESS_PREFIXES for the path set it narrates.\nIf a ruleset-read token is unavailable in CI, fall back to a documented manual\nrelease-runbook checklist item and record that decision.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a CI meta-check that fails when the develop ruleset lacks review enforcement", "wid": "w1"}, {"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": ["w0"], "notes": "Amend _project/DONE/main/active/auto-merge-review-gate-soundness-paths.yaml\nsuccess_metrics to state the code side is complete and the hard review\nprecondition is enforced by the ruleset change tracked in THIS item (link by\nid). Add the exact ruleset PUT to the release/admin runbook (AGENTS.md or\ndocs) so the human action is discoverable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct the DONE success_metric and document the admin runbook step", "wid": "w2"}, {"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": ["w1", "w2"], "notes": "CANCELLED by the 2026-07-18 retirement decision (see resolution_notes):\nthe admin action this waited on was retired without being applied, so\nthere is no enforcement to verify. Closed as retired, not as verified.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify enforcement end-to-end once the admin applies the ruleset change", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fork auto_merge_soundness_paths.py's logic into the workflow YAML as inline shell/python", "instead": "(not specified)", "why": "keep executing the actual script file (from the base ref) so there is exactly one implementation to maintain."}, {"dont": "skip the base-branch fetch step's existing --no-tags behavior or change its shallow/full-history semantics; only add the base-ref file materialization after it.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "treat this as a substitute for the ruleset-layer review requirement", "instead": "(not specified)", "why": "this is documented as defense-in-depth in the description; don't let this item's success_metrics imply it replaces auto-merge-review-gate-admin-enforcement."}], "approach": null, "blocked_reason": null, "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": ["soundness-surface-widening"], "description": "`.github/workflows/auto-merge-on-open.yml` (lines 31-48: the checkout, base-branch\nfetch, and \"Check soundness-critical paths\" step) checks out the PR at its own\nHEAD and then runs `_project/scripts/auto_merge_soundness_paths.py` *from that\nsame checkout*. For a same-repo (non-fork) PR \u2014 the normal case for this\nsingle-maintainer-owned repo where contributors push branches directly \u2014 the\nscript executing the predicate is whatever version the PR itself introduces. A\nPR that edits `auto_merge_soundness_paths.py` (e.g. narrows `SOUNDNESS_PREFIXES`,\nor the workflow file itself) evaluates its own diff against its own modified\nrules, so it can neutralize its review gate in the same commit that would\notherwise have triggered it.\n\nNote honestly: with the repo-layer ruleset review requirement from\nauto-merge-review-gate-admin-enforcement enforced (once its deferred admin\naction lands), this workflow's withholding behavior is defense-in-depth, not\nthe primary control \u2014 the ruleset's `pull_request` rule would block a\nzero-approval squash-merge regardless of what this workflow decides. This item\nis still worth doing because (a) the ruleset enforcement is not yet live, and\n(b) even once it is, a same-commit self-neutralization that quietly avoids\never showing \"review required\" is a bad signal for reviewers relying on the\nauto-merge state as a triage cue.", "id": "auto-merge-predicate-base-ref-execution", "preserves": ["The synchronize-event re-evaluation behavior (a later push that newly touches a soundness path flips auto-merge back off) continues to work.", "The workflow's output contract (steps.soundness.outputs.soundness_path, consumed by the existing enable/disable auto-merge steps) is unchanged.", "make pr-open's local invocation of the predicate is unaffected by this item (base-ref execution is CI-workflow-specific; a local developer already has full repo access)."], "prior_art": [{"concept": ".github/workflows/auto-merge-on-open.yml:31-48 \u2014 the checkout + soundness-check step being hardened; keep its step names and output contract (steps.soundness.outputs.soundness_path) stable so the enable/disable steps below it don't need changes.", "decision": "reuse", "path": ".github/workflows/auto-merge-on-open.yml:31-48"}, {"concept": "Makefile:pr-open (~line 1163) \u2014 also invokes the predicate via `uv run --project _project/scripts`; out of scope for this item's local-CLI flow (a developer running make pr-open locally already has arbitrary code execution), but keep the two invocations' semantics compatible.", "decision": "reuse", "path": "Makefile:pr-open"}, {"concept": "_project/scripts/auto_merge_soundness_paths.py \u2014 the predicate being protected; do not fork or duplicate its logic, only change which copy of the file gets executed.", "decision": "reuse", "path": "_project/scripts/auto_merge_soundness_paths.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "_project/scripts/auto_merge_soundness_paths.py"}, {"kind": "only_modify", "path_glob": ".github/workflows/auto-merge-on-open.yml"}, {"kind": "only_modify", "path_glob": "tests/unit/test_auto_merge_soundness_paths.py"}], "state": "done", "title": "Run the auto-merge soundness predicate from the base ref, not the PR's own checkout", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_auto_merge_soundness_paths.py -v", "description": "Backstop workflow tests still pass with the added base-ref and self-touch assertions", "expected": "all tests pass, including the extended test_backstop_workflow_uses_shared_predicate_and_skips_auto_merge", "item_id": "auto-merge-predicate-base-ref-execution", "last_result": null, "last_run": null, "seq": 1}, {"command": "gh run view --log | grep -A3 'Check soundness-critical paths'", "description": "A workflow_dispatch or manual trace confirms /tmp/predicate.py is materialized from origin/develop, not the PR's HEAD copy, before the predicate runs", "expected": "log shows the git show origin/${{ github.base_ref }} step succeeding before the predicate invocation", "item_id": "auto-merge-predicate-base-ref-execution", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "auto-merge-predicate-base-ref-execution", "needs": [], "notes": "In the \"Check soundness-critical paths\" step, before running the predicate,\nmaterialize the base-ref version: `git show\n\"origin/${{ github.base_ref }}:_project/scripts/auto_merge_soundness_paths.py\"\n> /tmp/predicate.py` (base branch is already fetched by the preceding\n\"Fetch base branch\" step), then invoke `python /tmp/predicate.py --stdin\n--format github-output` instead of the checked-out repo path. Confirm the\nscript has no relative imports or sibling-file dependencies that would\nbreak when run standalone from /tmp (check the top of\nauto_merge_soundness_paths.py \u2014 it currently has none, keep it that way or\ncopy siblings too if that changes).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fetch and execute the base-ref copy of the predicate script instead of the PR's own checkout copy", "wid": "w1"}, {"evidence": null, "item_id": "auto-merge-predicate-base-ref-execution", "needs": ["w1"], "notes": "Add an explicit check: if the PR's diff touches\n_project/scripts/auto_merge_soundness_paths.py or\n.github/workflows/auto-merge-on-open.yml, force soundness_path=true\nregardless of what the base-ref predicate returns for the rest of the\ndiff. This closes the residual gap where a PR could still edit the\npredicate to make future PRs unsafe, without itself needing owner review\nunder the base-ref-only check (base-ref execution alone stops the\npredicate from judging its own diff leniently, but does not by itself\nflag \"this diff touches the predicate\" as inherently review-worthy).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Treat modification of the predicate or the workflow itself as soundness=true", "wid": "w2"}, {"evidence": null, "item_id": "auto-merge-predicate-base-ref-execution", "needs": ["w1", "w2"], "notes": "Extend tests/unit/test_auto_merge_soundness_paths.py's\ntest_backstop_workflow_uses_shared_predicate_and_skips_auto_merge (which\ncurrently string-matches the workflow YAML) with assertions for the new\n`git show origin/.../auto_merge_soundness_paths.py` line and the\npredicate/workflow self-touch check, so a future edit that removes the\nbase-ref execution regresses a fast unit test rather than only being\ncaught by manual review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update or add a workflow-level test exercising the base-ref execution path", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "duplicate the soundness path list; the meta-check must read it from auto_merge_soundness_paths.py.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "mark this Completed while the ruleset still shows review_count 0 / code-owner review false", "instead": "(not specified)", "why": "that repeats the overstated DONE metric."}], "approach": null, "blocked_reason": null, "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "auto-merge-review-gate-admin-enforcement", "id": 343, "reason": "Requires a repo admin (gh api PUT on the ruleset); cannot be done by an agent PR. Tracked as the gating human action for w3.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: CLAUDE.md states this exact require_code_owner_review ruleset target was retired 2026-07-18; maintainer-manual-merge path replaces it.", "summary": "Apply the develop ruleset change: require_code_owner_review=true."}], "deps": [], "description": "Follow-up to auto-merge-review-gate-soundness-paths (PR #912, already DONE). The\nadversarial holistic-review found the gate is only half-real in production. The\ncode side is correct and sound: pr-open (Makefile:1161) and the\nauto-merge-on-open.yml workflow share one predicate\n(_project/scripts/auto_merge_soundness_paths.py, invoked with --no-renames) and\ncorrectly WITHHOLD auto-merge enablement for PRs touching comparator/plan-parser\nsoundness paths. But nothing ENFORCES a review: the develop ruleset\n`develop-squash-only` has required_approving_review_count: 0 and\nrequire_code_owner_review: false (verified via `gh api repos/.../rules/branches/develop`;\nlegacy branch protection returns 404 \"Branch not protected\"). So a soundness-path\nPR can still be squash-merged with zero approvals \u2014 manually or by re-enabling\nauto-merge. The DONE item's success_metric (\"PRs touching the comparator or plan\nparsers cannot squash-auto-merge without a review\") is NOT met at the repo level.\n\nTwo parts: (1) a CI meta-check that makes the gap LOUD and self-healing so it\ncannot silently regress once set; (2) the actual ruleset change, which requires\na repo admin and is tracked here as the gating human action.", "id": "auto-merge-review-gate-admin-enforcement", "preserves": ["Non-soundness-path PRs keep squash auto-merge on CI-pass; this item only hardens the soundness exception.", "The code-side withholding behavior (pr-open + auto-merge-on-open.yml shared predicate, --no-renames) is unchanged."], "prior_art": [{"concept": "{'ref': '.github/workflows/pr.yml:270', 'note': \"Existing 'Oracle coverage map drift check' / drift-check step pattern in the lint job \u2014 model the ruleset meta-check on these gh-api-backed drift steps.\"}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'CODEOWNERS', 'note': 'Owns the soundness globs (parsers/**, equivalence/**, **/validation.py); require_code_owner_review only triggers if an owner team/user is actually assigned to those paths \u2014 verify, do not assume.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '_project/scripts/auto_merge_soundness_paths.py', 'note': 'The shared soundness-path predicate; the meta-check should reuse its SOUNDNESS_PREFIXES so the CI assertion and the auto-merge withholding agree on one source of truth.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/pr.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/auto-merge-review-gate-soundness-paths.yaml"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/unit/release/"}], "state": "done", "title": "Make the soundness-path review gate enforceable at the repo layer, not just code", "verifications": [{"command": "uv run -- python -m pytest tests/unit/release/ -k ruleset_review_enforcement -q", "description": "Meta-check fails on an unenforced ruleset and passes on an enforced one", "expected": "asserts the check fails when review_count<1 or code-owner review false; passes otherwise", "item_id": "auto-merge-review-gate-admin-enforcement", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": [], "notes": "Capture `gh api repos/joeharris76/BenchBox/rulesets` and the develop ruleset\ndetail; record required_approving_review_count and require_code_owner_review.\nConfirm CODEOWNERS assigns a real owner to parsers/**, equivalence/**,\n**/validation.py. Summarize (values + glob->owner) in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm live ruleset state and CODEOWNERS ownership of soundness globs", "wid": "w0"}, {"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": ["w0"], "notes": "Add a step in an ALWAYS-required job (e.g. pr.yml lint / code-lint), never\ncontent-guard: content-guard is gated by content-guard-needed and skipped for\ncode-only PRs (ci-required-result ignores it unless content paths changed), so\na check placed there would let a code-only PR pass while review enforcement is\nabsent - defeating this item's stated self-healing guarantee. The step queries\nthe live develop ruleset via gh api and FAILS if require_code_owner_review !=\ntrue. required_approving_review_count is deliberately NOT part of this check:\nit is branch-wide, not CODEOWNERS-scoped, so asserting it here would gate\nevery develop PR rather than just soundness-path ones (see\nruleset_review_enforcement.py's module docstring). Reuse\nauto_merge_soundness_paths.py SOUNDNESS_PREFIXES for the path set it narrates.\nIf a ruleset-read token is unavailable in CI, fall back to a documented manual\nrelease-runbook checklist item and record that decision.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a CI meta-check that fails when the develop ruleset lacks review enforcement", "wid": "w1"}, {"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": ["w0"], "notes": "Amend _project/DONE/main/active/auto-merge-review-gate-soundness-paths.yaml\nsuccess_metrics to state the code side is complete and the hard review\nprecondition is enforced by the ruleset change tracked in THIS item (link by\nid). Add the exact ruleset PUT to the release/admin runbook (AGENTS.md or\ndocs) so the human action is discoverable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct the DONE success_metric and document the admin runbook step", "wid": "w2"}, {"evidence": null, "item_id": "auto-merge-review-gate-admin-enforcement", "needs": ["w1", "w2"], "notes": "CANCELLED by the 2026-07-18 retirement decision (see resolution_notes):\nthe admin action this waited on was retired without being applied, so\nthere is no enforcement to verify. Closed as retired, not as verified.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify enforcement end-to-end once the admin applies the ruleset change", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Process and Workflow", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "Nine numbered \"PR review follow-up sweep\" PRs landed this month fixing REAL\ndefects on already-merged PRs - including comparator-soundness holes in a\ncorrectness gate (e.g. #862: the tie-aware comparator used a constant/literal\ncolumn as the boundary-tie key, letting an arbitrary row-swap pass; #872/#878:\nunconditional multiset acceptance in the final ORDER BY group; #894: a Spark\nparser accepting an EXPLAIN-failure sentinel as a one-node plan). These landed up\nto ~3 weeks after merge because squash-auto-merge-on-CI-green makes human review a\npost-merge activity, not a merge precondition.\n\nEscaped defects cluster in a small set of files: the result comparator and the\nquery-plan parsers. Require one approving review for PRs touching those paths\nbefore auto-merge, while leaving the cheap-merge default intact for everything\nelse.", "id": "auto-merge-review-gate-soundness-paths", "preserves": ["No change to CI checks themselves; this adds a human-review precondition for a narrow path set.", "The cheap squash-auto-merge default is unchanged for all non-soundness paths.", "The local pr-open path and the GitHub auto-merge-on-open backstop use the same soundness-path predicate."], "prior_art": [{"concept": "{'ref': \"Makefile (pr-open target) / CLAUDE.md 'squash auto-merge is the default'\", 'note': 'make pr-open enables auto-merge automatically; it must NOT enable auto-merge when the diff touches the soundness paths.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '.github/CODEOWNERS (if present) / GitHub branch protection on develop', 'note': 'The mechanism to require review on specific paths; if no CODEOWNERS exists, add one scoped to the soundness paths.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '.github/workflows/auto-merge-on-open.yml:3', 'note': 'Backstop workflow auto-enables squash auto-merge for PRs opened outside make pr-open; it must honor the same soundness-path exception or it bypasses the local guard.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py, benchbox/core/equivalence/, benchbox/core/query_plans/parsers/', 'note': 'The escaped-defect-dense paths to gate.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".github/"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "CLAUDE.md"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "tests/unit/test_auto_merge_soundness_paths.py"}], "state": "done", "title": "Require a review pass before squash-auto-merge for soundness-critical paths (comparator + plan parsers)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_auto_merge_soundness_paths.py -q", "description": "Shared soundness-path predicate controls both auto-merge paths", "expected": "pr-open helper and auto-merge-on-open workflow both skip soundness-path diffs and still enable auto-merge for non-soundness diffs", "item_id": "auto-merge-review-gate-soundness-paths", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "auto-merge-review-gate-soundness-paths", "needs": [], "notes": "Add a CODEOWNERS entry (and/or document the branch-protection rule) requiring\none approving review for PRs touching benchbox/core/**/validation.py,\nbenchbox/core/equivalence/**, and benchbox/core/query_plans/parsers/**. Keep\nall other paths on the existing auto-merge default.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add CODEOWNERS / branch-protection requiring one review on the soundness paths", "wid": "w1"}, {"evidence": null, "item_id": "auto-merge-review-gate-soundness-paths", "needs": ["w1"], "notes": "Adjust the pr-open flow so it does NOT auto-enable squash auto-merge when the\nchanged files intersect the soundness paths (diff --name-only against the path\nset). For those PRs, open without auto-merge so the required review gates the\nmerge. Document the behavior in AGENTS.md/CLAUDE.md in the terse house style.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make pr-open skip auto-merge when the diff touches a soundness path", "wid": "w2"}, {"evidence": null, "item_id": "auto-merge-review-gate-soundness-paths", "needs": ["w1"], "notes": ".github/workflows/auto-merge-on-open.yml currently runs `gh pr merge --auto\n--squash` UNCONDITIONALLY for every non-draft PR opened against develop. That is\nthe GitHub-side backstop for PRs opened via the UI, `gh pr create`, or any path\nthat skips `make pr-open`. While it stays unconditional it RE-ENABLES auto-merge\non soundness-path PRs even when w2 opened them without it, defeating the\nreview-before-auto-merge goal.\n\nGate this workflow on the SAME soundness-path predicate w2 uses: detect whether\nthe PR diff intersects the soundness paths (e.g. a paths/paths-ignore filter or\na diff --name-only check in the job) and SKIP the `gh pr merge --auto --squash`\nstep for those PRs. Keep the cheap auto-merge default for every non-soundness\ndiff. The predicate MUST be the single shared source the regression test pins,\nso the local pr-open guard and this backstop never diverge.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate the auto-merge-on-open backstop workflow on the same soundness-path predicate", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:28Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #487 of retire-sqlglot-duckdb-all-workaround.\nDeferred: Auto-retirement workflow (run repros on every sqlglot upgrade)\nReason deferred: Worth considering as a separate process improvement. Out of scope\nfor this cleanup.\n", "id": "auto-retirement-workflow-run-repros-on-every-sqlglot-upgrade", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Auto-retirement workflow (run repros on every sqlglot upgrade)", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add third-party dependencies for the signature script (stdlib only, like the existing token-scan gates)", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "implement a green->red-transition-only guard (misses new failures on an already-red develop)", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "parse human-oriented pytest terminal output; use junit XML", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "silently skip reverting when signature comparison errors out", "why": "fail open to a revert PR, and say so in its body"}], "approach": "Keep the workflow as orchestration; put comparison logic in a tested\nstdlib-only script following the path_filter_decision.py precedent. Fail\nopen (current always-revert behavior) whenever the previous run's signature\nis unavailable. Coordination: medium-tier-pre-merge-lane also touches\ndevelop-post-merge.yml and deps.needs on this TODO - land this one first.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-11T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "auto-revert-signature-attribution", "id": 50, "reason": "Handled operationally by fix-session-isolation-placeholder-rows w3.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: One-time historical PR relabeling; reason states it's handled operationally elsewhere and source item that would need it is stale/gone.", "summary": "Retroactive relabeling/closing of pre-existing bogus revert PRs"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "auto-revert-signature-attribution", "id": 51, "reason": "This TODO covers the four develop-post-merge gates; new lanes adopt the same helper when added.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on a future new PR-lane job needing signature capture; no such lane is currently pending. Convention (reuse helper) already established, not a TODO.", "summary": "Signature capture for jobs added later (e.g. a future PR-lane medium job)"}], "deps": [], "description": "The auto-revert-on-failure job in .github/workflows/develop-post-merge.yml\nfires on ANY gate-job failure and reverts `github.sha` unconditionally. On\n2026-07-10 two real breakages (#1093 test-side, #1100 code-side) produced\n15 consecutive red runs and ~14 auto-revert PRs, of which only 2 (#1099,\n#1112) blamed the guilty commit. TODO-only and docs-only merges (#1118,\n#1123) got revert PRs. When the pile of revert PRs was bulk-closed at\n19:22, the one correct revert (#1112) was discarded with the noise.\n\nFix: attribute by failure signature, not by newest SHA.\n\n- Each gate job (lint, fast-test, explorer-tokens, medium-test) exposes a\n machine-readable failure signature (for pytest lanes: the set of failed\n test node IDs; for lint/token lanes: the failing job name + step is\n enough) uploaded as a per-run artifact.\n- auto-revert-on-failure downloads the signature artifact of the PREVIOUS\n completed develop-post-merge run and diffs:\n * previous run green, or new failure IDs appeared -> this merge\n introduced them: open the revert PR (current behavior), listing the\n new failure IDs in the body.\n * previous run red with an identical-or-superset signature and no new\n IDs -> do NOT open a revert PR; upsert a single open\n \"develop red\" incident issue (label incident:develop-red) appending\n the run link, so persistent red is one issue, not N PRs.\n- A plain green->red transition guard is NOT sufficient: #1100 added a\n second failure to an already-red develop and a transition guard would\n have skipped the one revert PR that was actually correct. Signature\n diffing handles both cases.\n\nAlso in scope (same file):\n- When all gate jobs succeed, auto-close any open auto-revert/* PRs and\n open incident:develop-red issues with a comment naming the green run\n (extension alongside close-orphaned-prs).\n- Fix the stale metrics-job comment (lines ~448-451) claiming\n MEDIUM_TEST_RESULT is inert under continue-on-error; medium-test was\n promoted to blocking on 2026-07-06.", "id": "auto-revert-signature-attribution", "preserves": ["A genuinely new failure on an already-red develop STILL opens a revert PR (the #1100 case)", "Existing conflict fallback: git revert conflict still opens the incident:develop-red-revert-conflict issue with SHA-in-body dedup", "Idempotency: re-runs for the same SHA must not open duplicate revert PRs or issues", "Metrics artifact stays a superset of the current JSON schema (existing keys unchanged)", "close-orphaned-prs semantics untouched"], "prior_art": [{"concept": ".github/workflows/develop-post-merge.yml:112-229 close-orphaned-prs - same close-PRs-with-comment shape to reuse for green-run cleanup", "decision": "reuse", "path": ".github/workflows/develop-post-merge.yml:112-229"}, {"concept": ".github/workflows/develop-post-merge.yml:231-430 auto-revert-on-failure - extend, keep conflict-issue fallback and idempotency guards", "decision": "extend", "path": ".github/workflows/develop-post-merge.yml:231-430"}, {"concept": ".github/workflows/develop-post-merge.yml:325-366 open_manual_action_issue - existing issue-dedup-by-search pattern to generalize for the incident issue upsert", "decision": "reuse", "path": ".github/workflows/develop-post-merge.yml:325-366"}, {"concept": ".github/workflows/develop-post-merge.yml:432-552 metrics job - stale MEDIUM_TEST_RESULT comment lives here; metrics JSON schema must stay backward-compatible", "decision": "reuse", "path": ".github/workflows/develop-post-merge.yml:432-552"}, {"concept": "scripts/path_filter_decision.py - precedent for workflow logic extracted to a stdlib-only tested script instead of inline bash (pr.yml ci-paths job)", "decision": "reuse", "path": "scripts/path_filter_decision.py"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/develop-post-merge.yml"}, {"kind": "only_modify", "path_glob": "scripts/post_merge_signature.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_post_merge_signature.py"}], "state": "done", "title": "Make auto-revert-on-failure signature-aware so persistent red does not blame every subsequent merge", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_post_merge_signature.py -q", "description": "New signature build/diff unit tests are green.", "expected": "all pass", "item_id": "auto-revert-signature-attribution", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"import yaml; yaml.safe_load(open('.github/workflows/develop-post-merge.yml'))\"", "description": "The edited workflow still parses as YAML.", "expected": "exit 0", "item_id": "auto-revert-signature-attribution", "last_result": null, "last_run": null, "seq": 2}, {"command": "gh run list --workflow=develop-post-merge.yml --limit 1 (or GitHub MCP equivalent) after merge", "description": "Post-merge observation: one develop-post-merge run opens no bogus revert PR on green and its metrics artifact still parses.", "expected": "green run; no new auto-revert/* PR; metrics JSON parses", "item_id": "auto-revert-signature-attribution", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": [], "notes": "The CI run links (29097093548 etc.) expire; the durable pins are the\nrevert PRs themselves (#1099, #1104, #1107-#1112, #1115, #1120-#1122,\n#1125-#1127 - only #1099/#1112 blamed guilty commits) and the\nif-condition at develop-post-merge.yml:239. Record the re-check (SHA +\ncondition text) in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the incident evidence: confirm the auto-revert job's if-condition still fires on any gate failure at develop tip, and pin the 2026-07-10 facts.", "wid": "w0"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w0"], "notes": "Set PYTEST_ADDOPTS=\"--junitxml=\" as a step env in the workflow so\n`make ci-test` / `make test-medium` pick it up without changing local\ndeveloper behavior. Extract failed node IDs with a stdlib-only script\n(see w2); upload with actions/upload-artifact, if: always().\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit failure signatures: junit XML via PYTEST_ADDOPTS on the pytest gates plus an always-run signature-artifact upload step per gate job.", "wid": "w1"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add stdlib-only scripts/post_merge_signature.py: build a signature JSON from junit XML / job results, and diff two signatures returning the set of NEW failure IDs; unit-test both paths.", "wid": "w2"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w2"], "notes": "Previous-run lookup: gh api /actions/workflows/develop-post-merge.yml/runs\n?branch=develop&status=completed&per_page=1&created<... or walk runs and\npick the one whose run_number is highest below the current run. Missing\nartifact (expired, first run after this change) must fail open into\ntoday's behavior, never crash the job.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate auto-revert on the signature diff vs the previous run: revert PR only when new failure IDs exist, else upsert the incident issue.", "wid": "w3"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": [], "notes": "Covers both incident:develop-red and incident:develop-red-revert-conflict\nissues. Reuse the close-orphaned-prs close-with-comment shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Green-run cleanup job: when all four gates succeed, close open auto-revert/* PRs and open incident issues, commenting the green run and SHA.", "wid": "w4"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the stale metrics-job MEDIUM_TEST_RESULT comment; extend the metrics JSON with the signature outcome (new_failure_ids, revert_suppressed) as additive fields only.", "wid": "w5"}], "worktree": "auto-revert-incident"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "auto-regenerate during the pipeline build", "why": "PRs need to fail loudly with \"run the script\" guidance, not silently regenerate and hide drift."}, {"INSTEAD": "(not specified)", "dont": "include `generated_at` in `--check` mode", "why": "timestamp drift would defeat the purpose; the deterministic comparison must be byte-stable."}, {"INSTEAD": "(not specified)", "dont": "parse the submission-manifest.json contents for trust resolution", "why": "the established contract (PR"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["implement-pr-based-result-submission-workflow"], "description": "`results-data/corpus-inventory.json` is currently maintained by hand. It\nenumerates every bundle in the seed corpus with metadata (benchmark,\nplatform, scale, trust label, hash). Once Phase 2 community PRs start\nlanding, hand-maintenance will drift: contributors will forget to update\nthe inventory, maintainers will forget to nudge them, and the explorer\nstatic-build pipeline will silently see a stale view of the corpus.\n\nThis item replaces the hand-maintained file with a generated one. The\ngenerator scans `results-data/bundles/` recursively, extracts metadata\nfrom each bundle's schema-v2 header (and sidecar manifest where present),\nand writes a deterministic, sorted `corpus-inventory.json`. A pre-commit\nhook and a CI step prevent drift.", "id": "automate-corpus-inventory-generation", "preserves": ["Existing fields/shape consumed by any downstream tool referencing `corpus-inventory.json`. Audit consumers in w5 before changing the shape.", "Existing pipeline at `benchbox/core/explorer_pipeline/` - it builds its own manifest; the inventory is a separate, lighter index.", "The schema-v2 bundle format and the trust-label sidecar contract - generator reads, never mutates."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/validate-submission.yml"}, {"kind": "only_modify", "path_glob": ".pre-commit-config.yaml"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json"}, {"kind": "only_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/test_generate_corpus_inventory.py"}], "state": "done", "title": "Automate corpus-inventory.json generation from results-data/bundles/", "verifications": [{"command": "uv run -- python scripts/generate_corpus_inventory.py --write && uv run -- python scripts/generate_corpus_inventory.py --check", "description": "Generator runs cleanly against current corpus, --check passes after --write", "expected": "exit 0 from both runs", "item_id": "automate-corpus-inventory-generation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/scripts/test_generate_corpus_inventory.py -q", "description": "Unit tests pass", "expected": "all tests pass", "item_id": "automate-corpus-inventory-generation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/generate_corpus_inventory.py --write && cp results-data/corpus-inventory.json /tmp/inv1.json && uv run -- python scripts/generate_corpus_inventory.py --write && diff /tmp/inv1.json results-data/corpus-inventory.json", "description": "Generated file is byte-identical when run twice (deterministic)", "expected": "no diff (modulo generated_at if not stripped)", "item_id": "automate-corpus-inventory-generation", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": [], "notes": "Inputs: scan `results-data/bundles/` for `*.json` (excluding sidecar\nfiles matching `submission-manifest.json` and any `.plans.json` /\n`.tuning.json` companions).\n\nFor each bundle:\n - Parse schema-v2 header: benchmark, platform, scale_factor,\n timestamp, query_count, query_set\n - Detect sibling `submission-manifest.json` \u2192 trust_label =\n community-submission, else default maintainer-run\n - Compute SHA256 of the bundle file\n - Capture relative path\n\nOutput: `results-data/corpus-inventory.json` with\n - `generated_at` (UTC ISO timestamp, omit if --check)\n - `bundles[]` sorted by (benchmark, platform, scale_factor, timestamp)\n - `cohorts` summary: per-(benchmark, scale) lists of platforms\n - `summary`: total_bundles, by_benchmark, by_platform, by_trust_label\n\nFlags:\n - `--check` \u2192 exit 1 if generated content differs from on-disk file\n (no write, deterministic - no `generated_at`)\n - `--write` \u2192 overwrite `corpus-inventory.json`\nDefault: `--write`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write `scripts/generate_corpus_inventory.py`", "wid": "w1"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w1"], "notes": "`tests/unit/scripts/test_generate_corpus_inventory.py`:\n - Empty bundles dir \u2192 empty inventory with consistent shape\n - Single bundle without sidecar \u2192 maintainer-run trust label\n - Single bundle with sidecar \u2192 community-submission\n - Multiple bundles \u2192 sorted deterministically\n - Cohort summary: 3 platforms at same (benchmark, scale) groups\n correctly\n - Hash matches sha256 of bundle file\n - `--check` returns 0 on match, 1 on drift, prints diff\n - Companion files (.plans.json, .tuning.json) excluded from bundle\n enumeration\n - Malformed bundle JSON \u2192 fail with clear message naming the file\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for the generator", "wid": "w2"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w2"], "notes": "Update `.github/workflows/validate-submission.yml` to add a step after\nbundle validation:\n `uv run python scripts/generate_corpus_inventory.py --check`\nA failing check posts a PR comment instructing the contributor to run\n`uv run python scripts/generate_corpus_inventory.py --write` locally\nand amend the PR. Keep the message friendly - most contributors will\nnot have read the contributor guide before this point.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the generator into validate-submission CI workflow", "wid": "w3"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w2"], "notes": "Add to `.pre-commit-config.yaml` (or create one if absent):\n - id: corpus-inventory\n name: corpus-inventory drift check\n entry: uv run python scripts/generate_corpus_inventory.py --check\n files: ^results-data/bundles/\n language: system\n pass_filenames: false\nDocument hook installation in `docs/contributing-results.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add pre-commit hook entry for the generator", "wid": "w4"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w1"], "notes": "1. Run `--write` against the current corpus.\n2. Diff against the existing hand-maintained file. Any diff is suspect:\n investigate whether the hand-maintained file is right (then fix the\n generator) or the generator is right (then commit the regenerated\n file with a chore commit).\n3. The pipeline at `benchbox/core/explorer_pipeline/` already derives\n its own manifest, so confirm no consumer depends on the\n hand-curated shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate corpus-inventory.json and reconcile differences", "wid": "w5"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w3", "w5"], "notes": "Update `docs/contributing-results.md`:\n - Replace any \"edit corpus-inventory.json\" instruction with\n \"run `uv run python scripts/generate_corpus_inventory.py --write`\".\n - Note that CI will reject PRs whose inventory drifts.\nAdd a one-liner to the Phase 2 ops runbook (TODO\n`write-results-platform-phase-2-operations-runbook`) referencing the\nsame.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the generator in contributor guide and ops runbook", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "only generate the report", "dont": "auto-promote the Phase 3 design TODOs from the workflow", "why": "the human-judgment paragraph per metric is the safeguard against false-positive promotion"}, {"INSTEAD": "quarterly + manual dispatch is correct", "dont": "run the workflow on every push", "why": "the cadence is what gives the metrics meaning"}, {"INSTEAD": "use the 1-7 day cron plus Monday guard.", "dont": "schedule the workflow on the first calendar day while documenting first Monday", "why": "the cadence must be machine-checkable"}], "approach": "Build w1 (JSON output) first since w2 and w3 both depend on it. w3\n(the workflow) is the load-bearing piece \u2014 w2 and w4 are conveniences\nthat the workflow can adopt incrementally.\n", "blocked_reason": null, "category": "Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-27T00:00:00Z", "deferrals": [], "deps": ["define-phase-3-promotion-metrics"], "description": "`scripts/phase2_metrics.py` (landed 2026-04-27) computes the six\nPhase 2 \u2192 Phase 3 promotion metrics on demand and writes a markdown\nhandoff. The script is sound, but the **cadence depends on a human\nremembering to run it quarterly** \u2014 the same failure mode the metrics\nwere introduced to prevent.\n\nThis TODO closes the loop: machine-readable output, automatic diff\nagainst the previous review, and a scheduled CI hook so the report is\nproduced on the first Monday of each quarter without intervention.\n\nSurfaced by the post-merge review of PR #33 (commit d6daac8c5).", "id": "automate-phase2-metrics-quarterly-cadence", "preserves": ["Markdown output remains the default (the existing handoff file format)", "Read-only: no GitHub writes from the metrics script itself; the workflow opens a PR but does not auto-merge", "The decision rule (two thresholds, two consecutive quarters) stays in `_project/analysis/phase-3-promotion-metrics.md`, not the script"], "prior_art": [{"concept": ".github/workflows/nightly.yml and .github/workflows/release-canary.yml \u2014 existing scheduled workflow idioms; REUSE cron plus workflow_dispatch structure", "decision": "reuse", "path": ".github/workflows/nightly.yml"}, {"concept": ".github/workflows/sync-results-data-to-published.yml and .github/workflows/seed-corpus.yml \u2014 existing GitHub Actions PR-creation flows; REUSE scoped write permissions and gh-cli PR creation rather than adding a bespoke GitHub client", "decision": "reuse", "path": ".github/workflows/sync-results-data-to-published.yml"}, {"concept": "_project/handoffs/phase-3-review-baseline-2026-04-27.md \u2014 current markdown report shape; EXTEND with JSON sidecar/diff data without breaking markdown default output", "decision": "extend", "path": "_project/handoffs/phase-3-review-baseline-2026-04-27.md"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/"}], "state": "done", "title": "Automate the Phase 3 promotion-metrics quarterly cadence", "verifications": [{"command": "bash -o pipefail -c 'uv run -- python scripts/phase2_metrics.py --format json | uv run -- python -c \"import json,sys; json.load(sys.stdin)\"'", "description": "JSON output validates as JSON", "expected": "", "item_id": "automate-phase2-metrics-quarterly-cadence", "last_result": null, "last_run": null, "seq": 1}, {"command": "rg -q '0 9 1-7 1,4,7,10 \\*' .github/workflows/phase3-promotion-review.yml && rg -q 'date -u \\+%u' .github/workflows/phase3-promotion-review.yml", "description": "Workflow file exists and implements first-Monday quarterly cadence", "expected": "cron and Monday guard are both present", "item_id": "automate-phase2-metrics-quarterly-cadence", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/phase2_metrics.py | grep -c 'Trend vs'", "description": "Diff section appears when at least two reviews exist", "expected": ">= 1", "item_id": "automate-phase2-metrics-quarterly-cadence", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": [], "notes": "Today the script always emits markdown. Add `--format {markdown,json}`\nand emit a JSON object with: generated_at, repo, base_branch,\nresults: [{name, value, breached, threshold, note}],\nextraction_triggers: [...]. Markdown remains the default. The JSON\nshape is what the diff-vs-last-review step (w2) and the scheduled\ncheck (w3) consume.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `--format json` to phase2_metrics for automation consumers", "wid": "w1"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w1"], "notes": "Walk `_project/handoffs/phase-3-review-*.md` (or the JSON sidecars\nfrom w1), pick the most recent before today, and append a \"Trend\nvs \" section to the report \u2014 per metric, value-then vs\nvalue-now, plus a status-change column (ok\u2192BREACHED, etc.).\nThe \"two consecutive quarters\" hysteresis rule in\n`_project/analysis/phase-3-promotion-metrics.md` is the primary\nconsumer of this signal \u2014 surface it explicitly so the reviewer\ndoesn't have to diff manually.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Compute diff vs the most recent prior review", "wid": "w2"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w1"], "notes": "`.github/workflows/phase3-promotion-review.yml` runs on a cron of\n`0 9 1-7 1,4,7,10 *` plus a shell guard that exits unless\n`date -u +%u` is `1`. That implements \"first Monday of each\nquarter\" without relying on day-of-month/day-of-week OR semantics.\n`workflow_dispatch` remains available for manual recovery.\nStep 1: `uv run -- python scripts/phase2_metrics.py --output\n_project/handoffs/phase-3-review-$(date +%Y-%m-%d).md`.\nStep 2: open a PR titled `chore(phase-3): quarterly promotion review\n` with the handoff committed. The maintainer reviews the PR\nand writes the one-paragraph interpretation per metric per the\nprocedure in `phase-3-promotion-metrics.md`, then merges or rejects.\nNo auto-merge \u2014 the human-judgment paragraph is the point.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a GitHub Actions workflow that runs the script quarterly", "wid": "w3"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w1"], "notes": "Optional flag for users who want to wire this into a non-quarterly\ncheck (e.g., a weekly cron during a high-throughput period). Off by\ndefault \u2014 the canonical use is quarterly review where the report is\nalways generated regardless of breach state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `--exit-non-zero-on-breach` for ad-hoc CI gating", "wid": "w4"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w3"], "notes": "Update `_project/analysis/phase-3-promotion-metrics.md` \"Review\nCadence and Procedure\" section: replace step 2 (\"Run the metrics\nscript\") with \"Review the auto-generated PR titled \"\nand document the manual fallback (`workflow_dispatch` if the cron\nmisses).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the automation in phase-3-promotion-metrics.md", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add third-party dependencies for the signature script (stdlib only, like the existing token-scan gates)", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "implement a green->red-transition-only guard (misses new failures on an already-red develop)", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "parse human-oriented pytest terminal output; use junit XML", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "silently skip reverting when signature comparison errors out", "instead": "(not specified)", "why": "fail open to a revert PR, and say so in its body"}], "approach": "Keep the workflow as orchestration; put comparison logic in a tested\nstdlib-only script following the path_filter_decision.py precedent. Fail\nopen (current always-revert behavior) whenever the previous run's signature\nis unavailable. Coordination: medium-tier-pre-merge-lane also touches\ndevelop-post-merge.yml and deps.needs on this TODO - land this one first.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-11T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "auto-revert-signature-attribution", "id": 50, "reason": "Handled operationally by fix-session-isolation-placeholder-rows w3.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: One-time historical PR relabeling; reason states it's handled operationally elsewhere and source item that would need it is stale/gone.", "summary": "Retroactive relabeling/closing of pre-existing bogus revert PRs"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "auto-revert-signature-attribution", "id": 51, "reason": "This TODO covers the four develop-post-merge gates; new lanes adopt the same helper when added.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on a future new PR-lane job needing signature capture; no such lane is currently pending. Convention (reuse helper) already established, not a TODO.", "summary": "Signature capture for jobs added later (e.g. a future PR-lane medium job)"}], "deps": [], "description": "The auto-revert-on-failure job in .github/workflows/develop-post-merge.yml\nfires on ANY gate-job failure and reverts `github.sha` unconditionally. On\n2026-07-10 two real breakages (#1093 test-side, #1100 code-side) produced\n15 consecutive red runs and ~14 auto-revert PRs, of which only 2 (#1099,\n#1112) blamed the guilty commit. TODO-only and docs-only merges (#1118,\n#1123) got revert PRs. When the pile of revert PRs was bulk-closed at\n19:22, the one correct revert (#1112) was discarded with the noise.\n\nFix: attribute by failure signature, not by newest SHA.\n\n- Each gate job (lint, fast-test, explorer-tokens, medium-test) exposes a\n machine-readable failure signature (for pytest lanes: the set of failed\n test node IDs; for lint/token lanes: the failing job name + step is\n enough) uploaded as a per-run artifact.\n- auto-revert-on-failure downloads the signature artifact of the PREVIOUS\n completed develop-post-merge run and diffs:\n * previous run green, or new failure IDs appeared -> this merge\n introduced them: open the revert PR (current behavior), listing the\n new failure IDs in the body.\n * previous run red with an identical-or-superset signature and no new\n IDs -> do NOT open a revert PR; upsert a single open\n \"develop red\" incident issue (label incident:develop-red) appending\n the run link, so persistent red is one issue, not N PRs.\n- A plain green->red transition guard is NOT sufficient: #1100 added a\n second failure to an already-red develop and a transition guard would\n have skipped the one revert PR that was actually correct. Signature\n diffing handles both cases.\n\nAlso in scope (same file):\n- When all gate jobs succeed, auto-close any open auto-revert/* PRs and\n open incident:develop-red issues with a comment naming the green run\n (extension alongside close-orphaned-prs).\n- Fix the stale metrics-job comment (lines ~448-451) claiming\n MEDIUM_TEST_RESULT is inert under continue-on-error; medium-test was\n promoted to blocking on 2026-07-06.", "id": "auto-revert-signature-attribution", "preserves": ["A genuinely new failure on an already-red develop STILL opens a revert PR (the #1100 case)", "Existing conflict fallback: git revert conflict still opens the incident:develop-red-revert-conflict issue with SHA-in-body dedup", "Idempotency: re-runs for the same SHA must not open duplicate revert PRs or issues", "Metrics artifact stays a superset of the current JSON schema (existing keys unchanged)", "close-orphaned-prs semantics untouched"], "prior_art": [{"concept": ".github/workflows/develop-post-merge.yml:112-229 close-orphaned-prs - same close-PRs-with-comment shape to reuse for green-run cleanup", "decision": "reuse", "path": ".github/workflows/develop-post-merge.yml:112-229"}, {"concept": ".github/workflows/develop-post-merge.yml:231-430 auto-revert-on-failure - extend, keep conflict-issue fallback and idempotency guards", "decision": "extend", "path": ".github/workflows/develop-post-merge.yml:231-430"}, {"concept": ".github/workflows/develop-post-merge.yml:325-366 open_manual_action_issue - existing issue-dedup-by-search pattern to generalize for the incident issue upsert", "decision": "reuse", "path": ".github/workflows/develop-post-merge.yml:325-366"}, {"concept": ".github/workflows/develop-post-merge.yml:432-552 metrics job - stale MEDIUM_TEST_RESULT comment lives here; metrics JSON schema must stay backward-compatible", "decision": "reuse", "path": ".github/workflows/develop-post-merge.yml:432-552"}, {"concept": "scripts/path_filter_decision.py - precedent for workflow logic extracted to a stdlib-only tested script instead of inline bash (pr.yml ci-paths job)", "decision": "reuse", "path": "scripts/path_filter_decision.py"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/develop-post-merge.yml"}, {"kind": "only_modify", "path_glob": "scripts/post_merge_signature.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_post_merge_signature.py"}], "state": "done", "title": "Make auto-revert-on-failure signature-aware so persistent red does not blame every subsequent merge", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_post_merge_signature.py -q", "description": "New signature build/diff unit tests are green.", "expected": "all pass", "item_id": "auto-revert-signature-attribution", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"import yaml; yaml.safe_load(open('.github/workflows/develop-post-merge.yml'))\"", "description": "The edited workflow still parses as YAML.", "expected": "exit 0", "item_id": "auto-revert-signature-attribution", "last_result": null, "last_run": null, "seq": 2}, {"command": "gh run list --workflow=develop-post-merge.yml --limit 1 (or GitHub MCP equivalent) after merge", "description": "Post-merge observation: one develop-post-merge run opens no bogus revert PR on green and its metrics artifact still parses.", "expected": "green run; no new auto-revert/* PR; metrics JSON parses", "item_id": "auto-revert-signature-attribution", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": [], "notes": "The CI run links (29097093548 etc.) expire; the durable pins are the\nrevert PRs themselves (#1099, #1104, #1107-#1112, #1115, #1120-#1122,\n#1125-#1127 - only #1099/#1112 blamed guilty commits) and the\nif-condition at develop-post-merge.yml:239. Record the re-check (SHA +\ncondition text) in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the incident evidence: confirm the auto-revert job's if-condition still fires on any gate failure at develop tip, and pin the 2026-07-10 facts.", "wid": "w0"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w0"], "notes": "Set PYTEST_ADDOPTS=\"--junitxml=\" as a step env in the workflow so\n`make ci-test` / `make test-medium` pick it up without changing local\ndeveloper behavior. Extract failed node IDs with a stdlib-only script\n(see w2); upload with actions/upload-artifact, if: always().\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit failure signatures: junit XML via PYTEST_ADDOPTS on the pytest gates plus an always-run signature-artifact upload step per gate job.", "wid": "w1"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add stdlib-only scripts/post_merge_signature.py: build a signature JSON from junit XML / job results, and diff two signatures returning the set of NEW failure IDs; unit-test both paths.", "wid": "w2"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w2"], "notes": "Previous-run lookup: gh api /actions/workflows/develop-post-merge.yml/runs\n?branch=develop&status=completed&per_page=1&created<... or walk runs and\npick the one whose run_number is highest below the current run. Missing\nartifact (expired, first run after this change) must fail open into\ntoday's behavior, never crash the job.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate auto-revert on the signature diff vs the previous run: revert PR only when new failure IDs exist, else upsert the incident issue.", "wid": "w3"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": [], "notes": "Covers both incident:develop-red and incident:develop-red-revert-conflict\nissues. Reuse the close-orphaned-prs close-with-comment shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Green-run cleanup job: when all four gates succeed, close open auto-revert/* PRs and open incident issues, commenting the green run and SHA.", "wid": "w4"}, {"evidence": null, "item_id": "auto-revert-signature-attribution", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the stale metrics-job MEDIUM_TEST_RESULT comment; extend the metrics JSON with the signature outcome (new_failure_ids, revert_suppressed) as additive fields only.", "wid": "w5"}], "worktree": "auto-revert-incident"} +{"anti_patterns": [{"dont": "auto-regenerate during the pipeline build", "instead": "(not specified)", "why": "PRs need to fail loudly with \"run the script\" guidance, not silently regenerate and hide drift."}, {"dont": "include `generated_at` in `--check` mode", "instead": "(not specified)", "why": "timestamp drift would defeat the purpose; the deterministic comparison must be byte-stable."}, {"dont": "parse the submission-manifest.json contents for trust resolution", "instead": "(not specified)", "why": "the established contract (PR"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["implement-pr-based-result-submission-workflow"], "description": "`results-data/corpus-inventory.json` is currently maintained by hand. It\nenumerates every bundle in the seed corpus with metadata (benchmark,\nplatform, scale, trust label, hash). Once Phase 2 community PRs start\nlanding, hand-maintenance will drift: contributors will forget to update\nthe inventory, maintainers will forget to nudge them, and the explorer\nstatic-build pipeline will silently see a stale view of the corpus.\n\nThis item replaces the hand-maintained file with a generated one. The\ngenerator scans `results-data/bundles/` recursively, extracts metadata\nfrom each bundle's schema-v2 header (and sidecar manifest where present),\nand writes a deterministic, sorted `corpus-inventory.json`. A pre-commit\nhook and a CI step prevent drift.", "id": "automate-corpus-inventory-generation", "preserves": ["Existing fields/shape consumed by any downstream tool referencing `corpus-inventory.json`. Audit consumers in w5 before changing the shape.", "Existing pipeline at `benchbox/core/explorer_pipeline/` - it builds its own manifest; the inventory is a separate, lighter index.", "The schema-v2 bundle format and the trust-label sidecar contract - generator reads, never mutates."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/validate-submission.yml"}, {"kind": "only_modify", "path_glob": ".pre-commit-config.yaml"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json"}, {"kind": "only_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/test_generate_corpus_inventory.py"}], "state": "done", "title": "Automate corpus-inventory.json generation from results-data/bundles/", "verifications": [{"command": "uv run -- python scripts/generate_corpus_inventory.py --write && uv run -- python scripts/generate_corpus_inventory.py --check", "description": "Generator runs cleanly against current corpus, --check passes after --write", "expected": "exit 0 from both runs", "item_id": "automate-corpus-inventory-generation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/scripts/test_generate_corpus_inventory.py -q", "description": "Unit tests pass", "expected": "all tests pass", "item_id": "automate-corpus-inventory-generation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/generate_corpus_inventory.py --write && cp results-data/corpus-inventory.json /tmp/inv1.json && uv run -- python scripts/generate_corpus_inventory.py --write && diff /tmp/inv1.json results-data/corpus-inventory.json", "description": "Generated file is byte-identical when run twice (deterministic)", "expected": "no diff (modulo generated_at if not stripped)", "item_id": "automate-corpus-inventory-generation", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": [], "notes": "Inputs: scan `results-data/bundles/` for `*.json` (excluding sidecar\nfiles matching `submission-manifest.json` and any `.plans.json` /\n`.tuning.json` companions).\n\nFor each bundle:\n - Parse schema-v2 header: benchmark, platform, scale_factor,\n timestamp, query_count, query_set\n - Detect sibling `submission-manifest.json` \u2192 trust_label =\n community-submission, else default maintainer-run\n - Compute SHA256 of the bundle file\n - Capture relative path\n\nOutput: `results-data/corpus-inventory.json` with\n - `generated_at` (UTC ISO timestamp, omit if --check)\n - `bundles[]` sorted by (benchmark, platform, scale_factor, timestamp)\n - `cohorts` summary: per-(benchmark, scale) lists of platforms\n - `summary`: total_bundles, by_benchmark, by_platform, by_trust_label\n\nFlags:\n - `--check` \u2192 exit 1 if generated content differs from on-disk file\n (no write, deterministic - no `generated_at`)\n - `--write` \u2192 overwrite `corpus-inventory.json`\nDefault: `--write`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write `scripts/generate_corpus_inventory.py`", "wid": "w1"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w1"], "notes": "`tests/unit/scripts/test_generate_corpus_inventory.py`:\n - Empty bundles dir \u2192 empty inventory with consistent shape\n - Single bundle without sidecar \u2192 maintainer-run trust label\n - Single bundle with sidecar \u2192 community-submission\n - Multiple bundles \u2192 sorted deterministically\n - Cohort summary: 3 platforms at same (benchmark, scale) groups\n correctly\n - Hash matches sha256 of bundle file\n - `--check` returns 0 on match, 1 on drift, prints diff\n - Companion files (.plans.json, .tuning.json) excluded from bundle\n enumeration\n - Malformed bundle JSON \u2192 fail with clear message naming the file\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for the generator", "wid": "w2"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w2"], "notes": "Update `.github/workflows/validate-submission.yml` to add a step after\nbundle validation:\n `uv run python scripts/generate_corpus_inventory.py --check`\nA failing check posts a PR comment instructing the contributor to run\n`uv run python scripts/generate_corpus_inventory.py --write` locally\nand amend the PR. Keep the message friendly - most contributors will\nnot have read the contributor guide before this point.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the generator into validate-submission CI workflow", "wid": "w3"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w2"], "notes": "Add to `.pre-commit-config.yaml` (or create one if absent):\n - id: corpus-inventory\n name: corpus-inventory drift check\n entry: uv run python scripts/generate_corpus_inventory.py --check\n files: ^results-data/bundles/\n language: system\n pass_filenames: false\nDocument hook installation in `docs/contributing-results.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add pre-commit hook entry for the generator", "wid": "w4"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w1"], "notes": "1. Run `--write` against the current corpus.\n2. Diff against the existing hand-maintained file. Any diff is suspect:\n investigate whether the hand-maintained file is right (then fix the\n generator) or the generator is right (then commit the regenerated\n file with a chore commit).\n3. The pipeline at `benchbox/core/explorer_pipeline/` already derives\n its own manifest, so confirm no consumer depends on the\n hand-curated shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate corpus-inventory.json and reconcile differences", "wid": "w5"}, {"evidence": null, "item_id": "automate-corpus-inventory-generation", "needs": ["w3", "w5"], "notes": "Update `docs/contributing-results.md`:\n - Replace any \"edit corpus-inventory.json\" instruction with\n \"run `uv run python scripts/generate_corpus_inventory.py --write`\".\n - Note that CI will reject PRs whose inventory drifts.\nAdd a one-liner to the Phase 2 ops runbook (TODO\n`write-results-platform-phase-2-operations-runbook`) referencing the\nsame.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the generator in contributor guide and ops runbook", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "auto-promote the Phase 3 design TODOs from the workflow", "instead": "only generate the report", "why": "the human-judgment paragraph per metric is the safeguard against false-positive promotion"}, {"dont": "run the workflow on every push", "instead": "quarterly + manual dispatch is correct", "why": "the cadence is what gives the metrics meaning"}, {"dont": "schedule the workflow on the first calendar day while documenting first Monday", "instead": "use the 1-7 day cron plus Monday guard.", "why": "the cadence must be machine-checkable"}], "approach": "Build w1 (JSON output) first since w2 and w3 both depend on it. w3\n(the workflow) is the load-bearing piece \u2014 w2 and w4 are conveniences\nthat the workflow can adopt incrementally.\n", "blocked_reason": null, "category": "Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-27T00:00:00Z", "deferrals": [], "deps": ["define-phase-3-promotion-metrics"], "description": "`scripts/phase2_metrics.py` (landed 2026-04-27) computes the six\nPhase 2 \u2192 Phase 3 promotion metrics on demand and writes a markdown\nhandoff. The script is sound, but the **cadence depends on a human\nremembering to run it quarterly** \u2014 the same failure mode the metrics\nwere introduced to prevent.\n\nThis TODO closes the loop: machine-readable output, automatic diff\nagainst the previous review, and a scheduled CI hook so the report is\nproduced on the first Monday of each quarter without intervention.\n\nSurfaced by the post-merge review of PR #33 (commit d6daac8c5).", "id": "automate-phase2-metrics-quarterly-cadence", "preserves": ["Markdown output remains the default (the existing handoff file format)", "Read-only: no GitHub writes from the metrics script itself; the workflow opens a PR but does not auto-merge", "The decision rule (two thresholds, two consecutive quarters) stays in `_project/analysis/phase-3-promotion-metrics.md`, not the script"], "prior_art": [{"concept": ".github/workflows/nightly.yml and .github/workflows/release-canary.yml \u2014 existing scheduled workflow idioms; REUSE cron plus workflow_dispatch structure", "decision": "reuse", "path": ".github/workflows/nightly.yml"}, {"concept": ".github/workflows/sync-results-data-to-published.yml and .github/workflows/seed-corpus.yml \u2014 existing GitHub Actions PR-creation flows; REUSE scoped write permissions and gh-cli PR creation rather than adding a bespoke GitHub client", "decision": "reuse", "path": ".github/workflows/sync-results-data-to-published.yml"}, {"concept": "_project/handoffs/phase-3-review-baseline-2026-04-27.md \u2014 current markdown report shape; EXTEND with JSON sidecar/diff data without breaking markdown default output", "decision": "extend", "path": "_project/handoffs/phase-3-review-baseline-2026-04-27.md"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/"}], "state": "done", "title": "Automate the Phase 3 promotion-metrics quarterly cadence", "verifications": [{"command": "bash -o pipefail -c 'uv run -- python scripts/phase2_metrics.py --format json | uv run -- python -c \"import json,sys; json.load(sys.stdin)\"'", "description": "JSON output validates as JSON", "expected": "", "item_id": "automate-phase2-metrics-quarterly-cadence", "last_result": null, "last_run": null, "seq": 1}, {"command": "rg -q '0 9 1-7 1,4,7,10 \\*' .github/workflows/phase3-promotion-review.yml && rg -q 'date -u \\+%u' .github/workflows/phase3-promotion-review.yml", "description": "Workflow file exists and implements first-Monday quarterly cadence", "expected": "cron and Monday guard are both present", "item_id": "automate-phase2-metrics-quarterly-cadence", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/phase2_metrics.py | grep -c 'Trend vs'", "description": "Diff section appears when at least two reviews exist", "expected": ">= 1", "item_id": "automate-phase2-metrics-quarterly-cadence", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": [], "notes": "Today the script always emits markdown. Add `--format {markdown,json}`\nand emit a JSON object with: generated_at, repo, base_branch,\nresults: [{name, value, breached, threshold, note}],\nextraction_triggers: [...]. Markdown remains the default. The JSON\nshape is what the diff-vs-last-review step (w2) and the scheduled\ncheck (w3) consume.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `--format json` to phase2_metrics for automation consumers", "wid": "w1"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w1"], "notes": "Walk `_project/handoffs/phase-3-review-*.md` (or the JSON sidecars\nfrom w1), pick the most recent before today, and append a \"Trend\nvs \" section to the report \u2014 per metric, value-then vs\nvalue-now, plus a status-change column (ok\u2192BREACHED, etc.).\nThe \"two consecutive quarters\" hysteresis rule in\n`_project/analysis/phase-3-promotion-metrics.md` is the primary\nconsumer of this signal \u2014 surface it explicitly so the reviewer\ndoesn't have to diff manually.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Compute diff vs the most recent prior review", "wid": "w2"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w1"], "notes": "`.github/workflows/phase3-promotion-review.yml` runs on a cron of\n`0 9 1-7 1,4,7,10 *` plus a shell guard that exits unless\n`date -u +%u` is `1`. That implements \"first Monday of each\nquarter\" without relying on day-of-month/day-of-week OR semantics.\n`workflow_dispatch` remains available for manual recovery.\nStep 1: `uv run -- python scripts/phase2_metrics.py --output\n_project/handoffs/phase-3-review-$(date +%Y-%m-%d).md`.\nStep 2: open a PR titled `chore(phase-3): quarterly promotion review\n` with the handoff committed. The maintainer reviews the PR\nand writes the one-paragraph interpretation per metric per the\nprocedure in `phase-3-promotion-metrics.md`, then merges or rejects.\nNo auto-merge \u2014 the human-judgment paragraph is the point.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a GitHub Actions workflow that runs the script quarterly", "wid": "w3"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w1"], "notes": "Optional flag for users who want to wire this into a non-quarterly\ncheck (e.g., a weekly cron during a high-throughput period). Off by\ndefault \u2014 the canonical use is quarterly review where the report is\nalways generated regardless of breach state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `--exit-non-zero-on-breach` for ad-hoc CI gating", "wid": "w4"}, {"evidence": null, "item_id": "automate-phase2-metrics-quarterly-cadence", "needs": ["w3"], "notes": "Update `_project/analysis/phase-3-promotion-metrics.md` \"Review\nCadence and Procedure\" section: replace step 2 (\"Run the metrics\nscript\") with \"Review the auto-generated PR titled \"\nand document the manual fallback (`workflow_dispatch` if the cron\nmisses).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the automation in phase-3-promotion-metrics.md", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:33:54Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #116 of cleanup-project-scratch-directory.\nDeferred: Automate stale-reference detection via CI lint\nReason deferred: A grep-based CI step could catch future cases where docs are created inside\n_project/ and linked from outside. Deferred because it requires writing a\ncheck_project_references.py script and wiring it into the Makefile - small\nenough to add as a follow-up TODO once this cleanup is done.\n", "id": "automate-stale-reference-detection-via-ci-lint", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Automate stale-reference detection via CI lint", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:35:34Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #332 of landing-prompts-launch-gates.\nDeferred: Automated a11y tooling (pa11y/axe)\nReason deferred: Filed as follow-up unless added during w3; manual checklist covers MVP.", "id": "automated-a11y-tooling-pa11y-axe", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Automated a11y tooling (pa11y/axe)", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Supporting Infrastructure", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Automated pipeline to publish results to cloud storage, generate permalinks,\nand manage benchmark artifacts.", "id": "automated-data-publishing-pipeline", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Automated Data Publishing Pipeline", "verifications": [], "work": [], "worktree": "infrastructure-publishing"} @@ -85,96 +85,96 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-24T00:00:00Z", "completed_pr": null, "created_at": "2025-12-24T00:00:00Z", "deferrals": [], "deps": ["cloud-spark-shared-infrastructure"], "description": "Add AWS Glue managed Spark platform adapter supporting both SQL and DataFrame modes.\nAWS Glue has ~60% market share for AWS Spark workloads. Implementation reuses\ncloud-spark shared infrastructure for staging, session management, and config optimization.\n\nUses AWS Glue Jobs API for job submission, Glue Data Catalog for metadata,\nand S3 for data staging.", "id": "aws-glue-platform", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "AWS Glue Managed Spark Platform", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation and User Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-13T00:00:00Z", "deferrals": [], "deps": [], "description": "Create platform documentation for Microsoft Azure data platforms.\n\nTwo platforms need documentation:\n- Azure Synapse Analytics (dedicated + serverless SQL pools)\n- Microsoft Fabric Warehouse (Power BI integration)\n\nBoth platforms share Azure AD authentication patterns but have\ndistinct features and use cases.", "id": "azure-fabric-platform-documentation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Azure Synapse and Fabric Warehouse Documentation", "verifications": [], "work": [], "worktree": "docs-experience"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:37:01Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #582 of first-class-clickhouse-local-server-cloud-platforms.\nDeferred: Backfill or rewrite historical result artifacts so old `clickhouse` labels are normalized to new platform names in archived datasets\nReason deferred: Runtime read compatibility matters immediately; bulk artifact migration can follow once the live platform contract is stable", "id": "backfill-or-rewrite-historical-result-artifacts-so-old-click", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Backfill or rewrite historical result artifacts so old `clickhouse` labels are normalized to new platform names in archived datasets", "verifications": [], "work": [], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "support object storage URIs (s3://, gs://)", "dont": "assume local filesystem access for data in distributed mode", "why": "executors read from shared storage"}, {"INSTEAD": "import and reuse it directly", "dont": "fork or modify DataFusionQueryTransformer", "why": "Ballista uses the same DataFusion SQL dialect"}, {"INSTEAD": "make host:port configurable with sensible defaults", "dont": "hardcode scheduler addresses", "why": "deployment varies"}, {"INSTEAD": "create a new ballista.py file", "dont": "modify the existing DataFusion adapter", "why": "Ballista is a separate platform"}, {"INSTEAD": "support both standalone and remote deployment modes", "dont": "skip standalone mode support", "why": "it enables CI testing without a cluster"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-03-24T00:00:00Z", "deferrals": [], "deps": [], "description": "Add support for Apache DataFusion Ballista, a distributed SQL query engine built on\nApache DataFusion and Apache Arrow. Ballista extends DataFusion's single-node execution\nto a distributed cluster architecture with a scheduler and multiple executor nodes.\n\nBallista is notable for:\n- Distributed SQL execution across a cluster (scheduler + executors)\n- Built on DataFusion - same SQL dialect, same query planning, same Arrow-native execution\n- Python bindings built from source (BallistaBuilder and BallistaSessionContext APIs)\n- Supports Parquet, CSV, and other Arrow-compatible formats on shared storage\n- Kubernetes-native deployment with Helm charts\n- Standalone mode for single-machine testing\n- Columnar execution via Apache Arrow with query vectorization\n- Claims 2.9x speedup over Spark on TPC-H at 100GB scale factor\n\nSince Ballista uses DataFusion internally, the existing DataFusion adapter provides\na strong foundation. The key differences are: (1) connection via BallistaBuilder to a\nremote scheduler instead of a local SessionContext, (2) data must reside on shared\nstorage accessible to all executors (S3, GCS, HDFS, or shared filesystem), and\n(3) distributed execution semantics (partitioning, shuffles, task scheduling).\n\nIMPORTANT: The `ballista` name on PyPI is occupied by an UNRELATED package (a PyTorch\nhelper). The Apache Ballista Python bindings are NOT published to PyPI as prebuilt\nwheels. Users must build from source using maturin from the main Ballista repo\n(apache/datafusion-ballista/python/). This significantly affects installation UX\nand CI integration.\n\nReference: https://github.com/apache/datafusion-ballista\nPython bindings: https://github.com/apache/datafusion-ballista/tree/main/python\n", "id": "ballista-distributed-adapter", "preserves": ["All existing DataFusion adapter tests pass: uv run -- python -m pytest tests/unit/platforms/test_datafusion_adapter.py -v", "All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "DataFusion query transformer behavior unchanged (reused, not forked)", "DataFusion write transformer behavior unchanged (reused, not forked)", "Platform registry autodiscovery continues working for all existing platforms"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/query_plans/parsers/datafusion.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/datafusion_query_transformer.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/datafusion_write_transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/ballista.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/ballista.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_ballista_adapter.py"}], "state": "planning", "title": "Add Apache DataFusion Ballista Distributed Query Platform Adapter", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.ballista import BallistaAdapter; print('import OK')\"", "description": "Ballista adapter imports correctly", "expected": "import OK", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_ballista_adapter.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.platform_registry import PlatformRegistry; r = PlatformRegistry(); print('ballista' in r.get_all_platforms())\"", "description": "Platform registry lists ballista", "expected": "True", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -c \"from ballista import BallistaBuilder; ctx = BallistaBuilder().standalone().build(); df = ctx.sql('SELECT 1 AS x'); print(df.collect())\"", "description": "Standalone mode smoke test (if ballista installed)", "expected": "Arrow batch with value 1", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "ballista-distributed-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Build Ballista Python bindings from source and document setup", "wid": "w1"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Create BallistaAdapter class with standalone and remote connection modes", "wid": "w2"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Implement data loading for standalone mode (local filesystem)", "wid": "w3"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Implement data loading for distributed mode (shared storage)", "wid": "w4"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire up query transformer and plan parser reuse", "wid": "w5"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Register Ballista in platform registry and lazy loading", "wid": "w6"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add comprehensive unit tests", "wid": "w7"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w3", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Run TPC-H validation in standalone mode at SF 0.01", "wid": "w8"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Create platform documentation", "wid": "w9"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a toggle that only changes the header or page background", "why": "charts, tables, badges, and controls must also switch."}, {"INSTEAD": "(not specified)", "dont": "hard-code SVG axis/grid/text colors in chart components after this work", "why": "those are exactly where dark mode breaks."}, {"INSTEAD": "(not specified)", "dont": "keep the old mixed dark-shell/light-data contract as the only Results Explorer theme", "why": "users need an actual dark and light choice."}, {"INSTEAD": "(not specified)", "dont": "leave docs/blog with both Furo and BenchBox theme controls showing contradictory state.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "rely only on unit tests for a visual theme change", "why": "contrast, flash, chart legibility, and responsive behavior require browser verification."}, {"INSTEAD": "(not specified)", "dont": "solve dark mode by reducing information density or hiding table/chart details.", "why": "(unstated)"}], "approach": "Treat theming as a shared product contract, not a Results-only retheme.\nFirst inventory current assumptions, then define semantic tokens and theme\npersistence, then migrate each surface to the shared tokens. Move chart and\nSVG colors behind a theme-aware API before claiming dark-mode support. Finish\nby strengthening token gates and visual/a11y coverage so future PRs cannot\nregress one theme while testing the other.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-05-13T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-light-dark-theme-system", "id": 344, "reason": "Not required for the product contract; light/dark plus reduced/high-contrast states are sufficient for this phase.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicit product-scope decision: light/dark plus reduced/high-contrast states deemed sufficient; custom palettes not required for this phase.", "summary": "User-customizable color palettes beyond light/dark"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-light-dark-theme-system", "id": 345, "reason": "Useful later, but current need is a durable implementation and regression gates.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'public design-system documentation site' idea with no scope; theming item itself is already done.", "summary": "A public design-system documentation site"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-light-dark-theme-system", "id": 346, "reason": "Docs already rely on Furo; integrate with its theme mechanics instead of creating a new docs frontend.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Rejected alternative (full Furo replacement); the decision made was to integrate with Furo instead. Not deferred work.", "summary": "Replacing Furo entirely"}], "deps": ["benchbox-site-header-unification"], "description": "BenchBox currently has incompatible theme models across public surfaces.\nLanding is effectively dark-only through `landing/style.css`; docs/blog use\nFuro's light/dark support but force the custom BenchBox nav to stay dark;\nResults Explorer uses a custom token system and has tests that intentionally\npin a mixed dark-shell/light-data workspace. Users should be able to choose\nlight or dark theme consistently across landing, docs/blog, and Results\nExplorer, with the choice persisted and respected by every page.\n\nThis phase supersedes the previous Results Explorer decision that hard-coded\n\"dark BenchBox shell + light analytical data panels\" as the theme contract.\nThat visual treatment can remain the light-theme default if it still works,\nbut the product contract must become a real two-theme system: light and dark\ntokens for shell, data panels, controls, tables, charts, badges, focus rings,\nempty/error/loading states, code blocks, and sticky/scrollable surfaces.\n\nThis TODO depends on the shared header phase so the theme toggle and header\ntokens are implemented once instead of being patched separately into three\nheaders.", "id": "benchbox-light-dark-theme-system", "preserves": ["Charts, heatmaps, badges, and status states remain distinguishable in both themes and in reduced/high-contrast modes.", "Docs/blog Furo functionality remains intact.", "Header remains identical across surfaces in both light and dark themes.", "No Results data/query semantics, compare cohort rules, or result-link behavior changes as part of theming.", "Results Explorer dense analytical tables remain scannable and information-dense in both themes.", "System preference is respected for first-time visitors unless the user chooses an explicit theme.", "Theme choice persists across reloads and navigation.", "Users can choose light or dark theme and the choice applies consistently across landing, docs/blog, and Results Explorer."], "prior_art": [{"concept": "_project/DONE/main/planning/results-explorer-retheme-theme-system-foundation.yaml:1 \u2014 prior Results-only retheme foundation to supersede", "decision": "supersede", "path": "_project/DONE/main/planning/results-explorer-retheme-theme-system-foundation.yaml:1"}, {"concept": "_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml:1 \u2014 prior mixed-theme contract to replace with a real theme choice", "decision": "reuse", "path": "_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml:1"}, {"concept": "_project/analysis/results-explorer-theme-inventory-2026-05-07.md:106 \u2014 prior dark-shell/light-data decision that this TODO must supersede", "decision": "supersede", "path": "_project/analysis/results-explorer-theme-inventory-2026-05-07.md:106"}, {"concept": "benchbox-site-header-unification \u2014 dependency that establishes the shared global header before adding a theme toggle", "decision": "reuse", "path": "benchbox-site-header-unification"}, {"concept": "docs/_static/custom.css:53 \u2014 current forced-dark BenchBox nav override that must be removed or superseded", "decision": "reuse", "path": "docs/_static/custom.css:53"}, {"concept": "docs/conf.py:169 \u2014 Furo theme configuration and existing light/dark variable hooks", "decision": "reuse", "path": "docs/conf.py:169"}, {"concept": "docs/operations/results-explorer-token-scan.md:1 \u2014 existing Results token gate and documented arbitrary-value/SVG gaps", "decision": "reuse", "path": "docs/operations/results-explorer-token-scan.md:1"}, {"concept": "landing/script.js:133 \u2014 current JS-injected mobile nav styles that must become theme-aware", "decision": "reuse", "path": "landing/script.js:133"}, {"concept": "landing/style.css:3 \u2014 current dark-only landing CSS variables", "decision": "reuse", "path": "landing/style.css:3"}, {"concept": "results-explorer/src/index.css:6 \u2014 current Results Explorer `--bb-*` token definitions and mixed surface contract", "decision": "reuse", "path": "results-explorer/src/index.css:6"}, {"concept": "results-explorer/src/lib/chartTheme.ts:1 \u2014 current chart color source, currently fixed hex values", "decision": "reuse", "path": "results-explorer/src/lib/chartTheme.ts:1"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "_project/scripts/scan_explorer_tokens.py"}, {"kind": "only_modify", "path_glob": "docs/_static/"}, {"kind": "only_modify", "path_glob": "docs/_templates/"}, {"kind": "only_modify", "path_glob": "docs/conf.py"}, {"kind": "only_modify", "path_glob": "docs/operations/"}, {"kind": "only_modify", "path_glob": "docs/operations/results-explorer-token-scan.md"}, {"kind": "only_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/"}, {"kind": "only_modify", "path_glob": "results-explorer/package-lock.json"}, {"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/test_scan_explorer_tokens.py"}], "state": "done", "title": "Implement comprehensive BenchBox light/dark theming across public site and Results Explorer", "verifications": [{"command": "test -f _project/analysis/benchbox-theme-contract.md && rg -n 'landing|docs|blog|Results|light|dark|data-theme|localStorage|chart' _project/analysis/benchbox-theme-contract.md", "description": "Shared theme contract is documented and references all public surfaces", "expected": "contract exists and covers shell, static pages, docs/blog, Results, persistence, and charts", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm test", "description": "Results Explorer unit and component tests pass", "expected": "all Vitest tests pass", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run build", "description": "Results Explorer builds", "expected": "Vite build succeeds", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- sphinx-build -b html docs docs/_build/html", "description": "Docs build succeeds with synchronized theme assets", "expected": "Sphinx build succeeds", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 4}, {"command": "make lint-explorer-tokens && make lint-site-theme-tokens", "description": "Token/literal gates pass", "expected": "Results and site-wide theme token scans pass with only documented allowlisted palette definitions; if the implementation intentionally replaces these targets, update this verification command in the same PR", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 5}, {"command": "Run the light/dark browser verification added by this TODO across landing, docs/blog, and Results routes at desktop and mobile widths", "description": "Cross-surface light/dark browser verification passes", "expected": "theme toggle, persistence, contrast, charts, tables, and header parity pass in both themes", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": [], "notes": "Produce a compact inventory covering:\n- landing `--bg-*`, `--text-*`, `--accent-*`, hard-coded colors, gradients,\n badges, code blocks, and mobile menu JS-injected styles;\n- docs/blog Furo `light_css_variables`, `dark_css_variables`, custom CSS,\n nav overrides, code blocks, tables, admonitions, tags, and blog widgets;\n- Results Explorer `--bb-*` tokens, `data-surface` usage, direct\n arbitrary-value classes, raw SVG/chart hex colors, badge tones,\n skeleton colors, sticky table colors, high/reduced-color modes, and\n tests pinning mixed-theme behavior.\n\nDecision gate: do not start retheming until the inventory identifies\nwhich tokens are site-wide, which are Results-data-specific, and which\nare docs/Furo adapter variables.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory theme assumptions and hard-coded colors across public surfaces", "wid": "w0"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w0"], "notes": "Define:\n- theme values: `light`, `dark`, and optional `system`;\n- where the selected value is stored (`localStorage` key and fallback);\n- how early the theme is applied to avoid flash of wrong theme;\n- whether the toggle lives in the shared global header or in a menu;\n- accessible label, keyboard behavior, and announced state;\n- interaction with Furo's existing theme controls on docs/blog;\n- default behavior for first-time visitors using `prefers-color-scheme`.\n\nDecision gate: choose whether BenchBox controls Furo's theme directly or\nmaps a BenchBox site preference onto Furo's supported `data-theme`\ncontract. Do not ship two unsynchronized theme toggles on docs/blog.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the site-wide theme contract, persistence, and toggle behavior", "wid": "w1"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w1"], "notes": "Define token names and values for both light and dark modes:\n- global shell: header, footer, nav, CTA, active state;\n- app/page background;\n- data panel, muted panel, elevated panel, sticky cell/header;\n- primary/muted/subtle/inverse text;\n- border, strong border, divider, shadow/elevation;\n- focus-visible rings on light and dark surfaces;\n- status tones: info, success, warning, danger, neutral;\n- trust/validation/tuning/computed/visibility badges;\n- code block and inline code;\n- chart axis, grid, label, annotation, tooltip, categorical palette,\n sequential palette, diverging palette, faster/slower fills, missing\n state, and reduced/high-contrast variants.\n\nThe token contract must make dark mode a first-class mode for dense\nanalytical tables and charts, not a dark shell around light islands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create shared semantic tokens for both themes", "wid": "w2"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2"], "notes": "Update landing and shared header assets so theme selection is applied\nbefore first paint as much as static deployment allows. The landing page\nmust support light and dark for:\n- header and mobile menu;\n- hero and background treatment;\n- cards, badges, links, code blocks, copy buttons, and footer;\n- responsive states and hover/focus states.\n\nAvoid duplicating logic in every page. Keep the implementation compatible\nwith static hosting and no frontend build step for `landing/`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement shared theme initialization and toggle on static public pages", "wid": "w3"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2", "w3"], "notes": "Update `docs/conf.py`, `docs/_templates/page.html`, and\n`docs/_static/custom.css` so docs/blog use the same visible theme choice\nas the rest of BenchBox. Remove the custom nav forced-dark overrides and\nalign Furo `light_css_variables` and `dark_css_variables` with the shared\nBenchBox tokens.\n\nEnsure Furo search, sidebars, API pages, tables, code blocks, admonitions,\ntags, blog archives, and blog sidebars remain readable in both themes.\nIf Furo's native toggle cannot be synchronized cleanly, document the\ndecision and hide/replace one control so users do not see conflicting\ntheme state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate the shared theme contract with docs/blog and Furo", "wid": "w4"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2", "w3"], "notes": "Replace the old mixed-theme contract with theme-aware tokens in\n`results-explorer/src/index.css` and Results components. Cover:\n- `body`, app shell, global header, secondary Results nav, footer;\n- Home hero/filter bands and leaderboard surfaces;\n- Benchmark and platform detail controls, filter panels, compare bars,\n sticky headers, sticky columns, and large tables;\n- Compare builder, decision summary, guardrails, receipts, query diff,\n result cards, and warning callouts;\n- Query Workbench facets, visible columns, exports, result table,\n Advanced SQL, starter queries, and SQL result table;\n- loading, empty, error, warning, disabled, selected, hover, focus, and\n reduced-color states.\n\nRemove tests that assert the old hard-coded mixed dark/light background\nvalues and replace them with tests that verify both themes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retheme Results Explorer app shell, pages, forms, panels, tables, and states", "wid": "w5"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2", "w5"], "notes": "Refactor chart color access so components do not embed light-theme hex\nvalues for axes, grid lines, labels, annotations, legends, missing states,\nor fills. Cover at least:\n- `ChartPanel.tsx`;\n- `CDFChart.tsx`;\n- `CostScatter.tsx`;\n- `DistributionBox.tsx`;\n- `DivergingBarChart.tsx`;\n- `NormalizedSpeedupChart.tsx`;\n- `PercentileLadder.tsx`;\n- `PowerBar.tsx`;\n- `QueryHistogram.tsx`;\n- `QueryTimingChart.tsx`;\n- `StackedPhase.tsx`;\n- `TimeSeries.tsx`;\n- `results-explorer/src/lib/chartTheme.ts`.\n\nUse a theme-aware chart theme API or CSS-variable reads with SSR/test\nfallbacks. Keep platform categorical colors stable enough that users can\ncompare charts across tabs, but adjust axis/grid/text colors by theme.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Results Explorer charts and SVGs theme-aware", "wid": "w6"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w5", "w6"], "notes": "Update the Results Explorer token scan and documentation so it catches\ntheme bypasses that are currently known gaps:\n- arbitrary-value Tailwind classes such as `text-[#374151]`;\n- inline SVG `fill=\\\"#...\\\"` and `stroke=\\\"#...\\\"` literals outside the\n approved chart theme source;\n- raw `rgb()`/`rgba()` colors in public surfaces;\n- landing/docs header literal colors that should be tokenized.\n\nKeep a narrow allowlist for intentional palette definitions and document\neach allowlist reason. The gate should prevent new spot fixes from\nsilently making one theme correct and the other broken.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Upgrade token/literal gates to protect both themes", "wid": "w7"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w5", "w6", "w7"], "notes": "Update or supersede artifacts that currently encode the old Results-only\nmixed theme contract, including:\n- `_project/analysis/results-explorer-theme-inventory-2026-05-07.md`;\n- `_project/analysis/results-explorer-retheme-audit-remediation-plan-2026-05-07.md`;\n- `_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml`;\n- `results-explorer/e2e/routes/home.spec.ts`;\n- any release-gate reports or operations docs that describe dark shell\n plus light analytical panels as the final product contract.\n\nDo not rewrite history in DONE items. Add clear supersession notes where\nappropriate, and ensure active tests assert the new user-selectable\nlight/dark contract instead of treating mixed surfaces as mandatory.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire stale mixed-theme documentation, tests, and release evidence", "wid": "w8"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w3", "w4", "w5", "w6", "w7", "w8"], "notes": "Add or extend browser checks for both light and dark themes at desktop\nand mobile widths across:\n- landing `/`;\n- docs `/docs/`;\n- blog `/blog/` if locally buildable;\n- Results `/results/`;\n- Results benchmark detail;\n- Results platform detail;\n- Results compare builder and completed compare;\n- Results query workbench.\n\nVerify:\n- theme toggle changes visible colors on every surface;\n- persisted choice survives reload and route changes;\n- first paint does not visibly flash the opposite theme beyond the\n accepted technical limit;\n- WCAG contrast passes for text, controls, badges, heatmaps, chart axes,\n disabled controls, selected rows, and warning/error states;\n- no chart or table becomes unreadable in either theme;\n- header remains identical across surfaces in both themes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cross-surface visual, accessibility, and persistence verification", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "assume local filesystem access for data in distributed mode", "instead": "support object storage URIs (s3://, gs://)", "why": "executors read from shared storage"}, {"dont": "fork or modify DataFusionQueryTransformer", "instead": "import and reuse it directly", "why": "Ballista uses the same DataFusion SQL dialect"}, {"dont": "hardcode scheduler addresses", "instead": "make host:port configurable with sensible defaults", "why": "deployment varies"}, {"dont": "modify the existing DataFusion adapter", "instead": "create a new ballista.py file", "why": "Ballista is a separate platform"}, {"dont": "skip standalone mode support", "instead": "support both standalone and remote deployment modes", "why": "it enables CI testing without a cluster"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-03-24T00:00:00Z", "deferrals": [], "deps": [], "description": "Add support for Apache DataFusion Ballista, a distributed SQL query engine built on\nApache DataFusion and Apache Arrow. Ballista extends DataFusion's single-node execution\nto a distributed cluster architecture with a scheduler and multiple executor nodes.\n\nBallista is notable for:\n- Distributed SQL execution across a cluster (scheduler + executors)\n- Built on DataFusion - same SQL dialect, same query planning, same Arrow-native execution\n- Python bindings built from source (BallistaBuilder and BallistaSessionContext APIs)\n- Supports Parquet, CSV, and other Arrow-compatible formats on shared storage\n- Kubernetes-native deployment with Helm charts\n- Standalone mode for single-machine testing\n- Columnar execution via Apache Arrow with query vectorization\n- Claims 2.9x speedup over Spark on TPC-H at 100GB scale factor\n\nSince Ballista uses DataFusion internally, the existing DataFusion adapter provides\na strong foundation. The key differences are: (1) connection via BallistaBuilder to a\nremote scheduler instead of a local SessionContext, (2) data must reside on shared\nstorage accessible to all executors (S3, GCS, HDFS, or shared filesystem), and\n(3) distributed execution semantics (partitioning, shuffles, task scheduling).\n\nIMPORTANT: The `ballista` name on PyPI is occupied by an UNRELATED package (a PyTorch\nhelper). The Apache Ballista Python bindings are NOT published to PyPI as prebuilt\nwheels. Users must build from source using maturin from the main Ballista repo\n(apache/datafusion-ballista/python/). This significantly affects installation UX\nand CI integration.\n\nReference: https://github.com/apache/datafusion-ballista\nPython bindings: https://github.com/apache/datafusion-ballista/tree/main/python\n", "id": "ballista-distributed-adapter", "preserves": ["All existing DataFusion adapter tests pass: uv run -- python -m pytest tests/unit/platforms/test_datafusion_adapter.py -v", "All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "DataFusion query transformer behavior unchanged (reused, not forked)", "DataFusion write transformer behavior unchanged (reused, not forked)", "Platform registry autodiscovery continues working for all existing platforms"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/query_plans/parsers/datafusion.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/datafusion_query_transformer.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/datafusion_write_transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/ballista.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/ballista.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_ballista_adapter.py"}], "state": "planning", "title": "Add Apache DataFusion Ballista Distributed Query Platform Adapter", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.ballista import BallistaAdapter; print('import OK')\"", "description": "Ballista adapter imports correctly", "expected": "import OK", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_ballista_adapter.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.platform_registry import PlatformRegistry; r = PlatformRegistry(); print('ballista' in r.get_all_platforms())\"", "description": "Platform registry lists ballista", "expected": "True", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -c \"from ballista import BallistaBuilder; ctx = BallistaBuilder().standalone().build(); df = ctx.sql('SELECT 1 AS x'); print(df.collect())\"", "description": "Standalone mode smoke test (if ballista installed)", "expected": "Arrow batch with value 1", "item_id": "ballista-distributed-adapter", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "ballista-distributed-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Build Ballista Python bindings from source and document setup", "wid": "w1"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Create BallistaAdapter class with standalone and remote connection modes", "wid": "w2"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Implement data loading for standalone mode (local filesystem)", "wid": "w3"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Implement data loading for distributed mode (shared storage)", "wid": "w4"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire up query transformer and plan parser reuse", "wid": "w5"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Register Ballista in platform registry and lazy loading", "wid": "w6"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add comprehensive unit tests", "wid": "w7"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w3", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Run TPC-H validation in standalone mode at SF 0.01", "wid": "w8"}, {"evidence": null, "item_id": "ballista-distributed-adapter", "needs": ["w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Create platform documentation", "wid": "w9"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "add a toggle that only changes the header or page background", "instead": "(not specified)", "why": "charts, tables, badges, and controls must also switch."}, {"dont": "hard-code SVG axis/grid/text colors in chart components after this work", "instead": "(not specified)", "why": "those are exactly where dark mode breaks."}, {"dont": "keep the old mixed dark-shell/light-data contract as the only Results Explorer theme", "instead": "(not specified)", "why": "users need an actual dark and light choice."}, {"dont": "leave docs/blog with both Furo and BenchBox theme controls showing contradictory state.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "rely only on unit tests for a visual theme change", "instead": "(not specified)", "why": "contrast, flash, chart legibility, and responsive behavior require browser verification."}, {"dont": "solve dark mode by reducing information density or hiding table/chart details.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Treat theming as a shared product contract, not a Results-only retheme.\nFirst inventory current assumptions, then define semantic tokens and theme\npersistence, then migrate each surface to the shared tokens. Move chart and\nSVG colors behind a theme-aware API before claiming dark-mode support. Finish\nby strengthening token gates and visual/a11y coverage so future PRs cannot\nregress one theme while testing the other.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-05-13T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-light-dark-theme-system", "id": 344, "reason": "Not required for the product contract; light/dark plus reduced/high-contrast states are sufficient for this phase.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicit product-scope decision: light/dark plus reduced/high-contrast states deemed sufficient; custom palettes not required for this phase.", "summary": "User-customizable color palettes beyond light/dark"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-light-dark-theme-system", "id": 345, "reason": "Useful later, but current need is a durable implementation and regression gates.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'public design-system documentation site' idea with no scope; theming item itself is already done.", "summary": "A public design-system documentation site"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-light-dark-theme-system", "id": 346, "reason": "Docs already rely on Furo; integrate with its theme mechanics instead of creating a new docs frontend.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Rejected alternative (full Furo replacement); the decision made was to integrate with Furo instead. Not deferred work.", "summary": "Replacing Furo entirely"}], "deps": ["benchbox-site-header-unification"], "description": "BenchBox currently has incompatible theme models across public surfaces.\nLanding is effectively dark-only through `landing/style.css`; docs/blog use\nFuro's light/dark support but force the custom BenchBox nav to stay dark;\nResults Explorer uses a custom token system and has tests that intentionally\npin a mixed dark-shell/light-data workspace. Users should be able to choose\nlight or dark theme consistently across landing, docs/blog, and Results\nExplorer, with the choice persisted and respected by every page.\n\nThis phase supersedes the previous Results Explorer decision that hard-coded\n\"dark BenchBox shell + light analytical data panels\" as the theme contract.\nThat visual treatment can remain the light-theme default if it still works,\nbut the product contract must become a real two-theme system: light and dark\ntokens for shell, data panels, controls, tables, charts, badges, focus rings,\nempty/error/loading states, code blocks, and sticky/scrollable surfaces.\n\nThis TODO depends on the shared header phase so the theme toggle and header\ntokens are implemented once instead of being patched separately into three\nheaders.", "id": "benchbox-light-dark-theme-system", "preserves": ["Charts, heatmaps, badges, and status states remain distinguishable in both themes and in reduced/high-contrast modes.", "Docs/blog Furo functionality remains intact.", "Header remains identical across surfaces in both light and dark themes.", "No Results data/query semantics, compare cohort rules, or result-link behavior changes as part of theming.", "Results Explorer dense analytical tables remain scannable and information-dense in both themes.", "System preference is respected for first-time visitors unless the user chooses an explicit theme.", "Theme choice persists across reloads and navigation.", "Users can choose light or dark theme and the choice applies consistently across landing, docs/blog, and Results Explorer."], "prior_art": [{"concept": "_project/DONE/main/planning/results-explorer-retheme-theme-system-foundation.yaml:1 \u2014 prior Results-only retheme foundation to supersede", "decision": "supersede", "path": "_project/DONE/main/planning/results-explorer-retheme-theme-system-foundation.yaml:1"}, {"concept": "_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml:1 \u2014 prior mixed-theme contract to replace with a real theme choice", "decision": "reuse", "path": "_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml:1"}, {"concept": "_project/analysis/results-explorer-theme-inventory-2026-05-07.md:106 \u2014 prior dark-shell/light-data decision that this TODO must supersede", "decision": "supersede", "path": "_project/analysis/results-explorer-theme-inventory-2026-05-07.md:106"}, {"concept": "benchbox-site-header-unification \u2014 dependency that establishes the shared global header before adding a theme toggle", "decision": "reuse", "path": "benchbox-site-header-unification"}, {"concept": "docs/_static/custom.css:53 \u2014 current forced-dark BenchBox nav override that must be removed or superseded", "decision": "reuse", "path": "docs/_static/custom.css:53"}, {"concept": "docs/conf.py:169 \u2014 Furo theme configuration and existing light/dark variable hooks", "decision": "reuse", "path": "docs/conf.py:169"}, {"concept": "docs/operations/results-explorer-token-scan.md:1 \u2014 existing Results token gate and documented arbitrary-value/SVG gaps", "decision": "reuse", "path": "docs/operations/results-explorer-token-scan.md:1"}, {"concept": "landing/script.js:133 \u2014 current JS-injected mobile nav styles that must become theme-aware", "decision": "reuse", "path": "landing/script.js:133"}, {"concept": "landing/style.css:3 \u2014 current dark-only landing CSS variables", "decision": "reuse", "path": "landing/style.css:3"}, {"concept": "results-explorer/src/index.css:6 \u2014 current Results Explorer `--bb-*` token definitions and mixed surface contract", "decision": "reuse", "path": "results-explorer/src/index.css:6"}, {"concept": "results-explorer/src/lib/chartTheme.ts:1 \u2014 current chart color source, currently fixed hex values", "decision": "reuse", "path": "results-explorer/src/lib/chartTheme.ts:1"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "_project/scripts/scan_explorer_tokens.py"}, {"kind": "only_modify", "path_glob": "docs/_static/"}, {"kind": "only_modify", "path_glob": "docs/_templates/"}, {"kind": "only_modify", "path_glob": "docs/conf.py"}, {"kind": "only_modify", "path_glob": "docs/operations/"}, {"kind": "only_modify", "path_glob": "docs/operations/results-explorer-token-scan.md"}, {"kind": "only_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/"}, {"kind": "only_modify", "path_glob": "results-explorer/package-lock.json"}, {"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/test_scan_explorer_tokens.py"}], "state": "done", "title": "Implement comprehensive BenchBox light/dark theming across public site and Results Explorer", "verifications": [{"command": "test -f _project/analysis/benchbox-theme-contract.md && rg -n 'landing|docs|blog|Results|light|dark|data-theme|localStorage|chart' _project/analysis/benchbox-theme-contract.md", "description": "Shared theme contract is documented and references all public surfaces", "expected": "contract exists and covers shell, static pages, docs/blog, Results, persistence, and charts", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm test", "description": "Results Explorer unit and component tests pass", "expected": "all Vitest tests pass", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run build", "description": "Results Explorer builds", "expected": "Vite build succeeds", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- sphinx-build -b html docs docs/_build/html", "description": "Docs build succeeds with synchronized theme assets", "expected": "Sphinx build succeeds", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 4}, {"command": "make lint-explorer-tokens && make lint-site-theme-tokens", "description": "Token/literal gates pass", "expected": "Results and site-wide theme token scans pass with only documented allowlisted palette definitions; if the implementation intentionally replaces these targets, update this verification command in the same PR", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 5}, {"command": "Run the light/dark browser verification added by this TODO across landing, docs/blog, and Results routes at desktop and mobile widths", "description": "Cross-surface light/dark browser verification passes", "expected": "theme toggle, persistence, contrast, charts, tables, and header parity pass in both themes", "item_id": "benchbox-light-dark-theme-system", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": [], "notes": "Produce a compact inventory covering:\n- landing `--bg-*`, `--text-*`, `--accent-*`, hard-coded colors, gradients,\n badges, code blocks, and mobile menu JS-injected styles;\n- docs/blog Furo `light_css_variables`, `dark_css_variables`, custom CSS,\n nav overrides, code blocks, tables, admonitions, tags, and blog widgets;\n- Results Explorer `--bb-*` tokens, `data-surface` usage, direct\n arbitrary-value classes, raw SVG/chart hex colors, badge tones,\n skeleton colors, sticky table colors, high/reduced-color modes, and\n tests pinning mixed-theme behavior.\n\nDecision gate: do not start retheming until the inventory identifies\nwhich tokens are site-wide, which are Results-data-specific, and which\nare docs/Furo adapter variables.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory theme assumptions and hard-coded colors across public surfaces", "wid": "w0"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w0"], "notes": "Define:\n- theme values: `light`, `dark`, and optional `system`;\n- where the selected value is stored (`localStorage` key and fallback);\n- how early the theme is applied to avoid flash of wrong theme;\n- whether the toggle lives in the shared global header or in a menu;\n- accessible label, keyboard behavior, and announced state;\n- interaction with Furo's existing theme controls on docs/blog;\n- default behavior for first-time visitors using `prefers-color-scheme`.\n\nDecision gate: choose whether BenchBox controls Furo's theme directly or\nmaps a BenchBox site preference onto Furo's supported `data-theme`\ncontract. Do not ship two unsynchronized theme toggles on docs/blog.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the site-wide theme contract, persistence, and toggle behavior", "wid": "w1"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w1"], "notes": "Define token names and values for both light and dark modes:\n- global shell: header, footer, nav, CTA, active state;\n- app/page background;\n- data panel, muted panel, elevated panel, sticky cell/header;\n- primary/muted/subtle/inverse text;\n- border, strong border, divider, shadow/elevation;\n- focus-visible rings on light and dark surfaces;\n- status tones: info, success, warning, danger, neutral;\n- trust/validation/tuning/computed/visibility badges;\n- code block and inline code;\n- chart axis, grid, label, annotation, tooltip, categorical palette,\n sequential palette, diverging palette, faster/slower fills, missing\n state, and reduced/high-contrast variants.\n\nThe token contract must make dark mode a first-class mode for dense\nanalytical tables and charts, not a dark shell around light islands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create shared semantic tokens for both themes", "wid": "w2"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2"], "notes": "Update landing and shared header assets so theme selection is applied\nbefore first paint as much as static deployment allows. The landing page\nmust support light and dark for:\n- header and mobile menu;\n- hero and background treatment;\n- cards, badges, links, code blocks, copy buttons, and footer;\n- responsive states and hover/focus states.\n\nAvoid duplicating logic in every page. Keep the implementation compatible\nwith static hosting and no frontend build step for `landing/`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement shared theme initialization and toggle on static public pages", "wid": "w3"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2", "w3"], "notes": "Update `docs/conf.py`, `docs/_templates/page.html`, and\n`docs/_static/custom.css` so docs/blog use the same visible theme choice\nas the rest of BenchBox. Remove the custom nav forced-dark overrides and\nalign Furo `light_css_variables` and `dark_css_variables` with the shared\nBenchBox tokens.\n\nEnsure Furo search, sidebars, API pages, tables, code blocks, admonitions,\ntags, blog archives, and blog sidebars remain readable in both themes.\nIf Furo's native toggle cannot be synchronized cleanly, document the\ndecision and hide/replace one control so users do not see conflicting\ntheme state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate the shared theme contract with docs/blog and Furo", "wid": "w4"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2", "w3"], "notes": "Replace the old mixed-theme contract with theme-aware tokens in\n`results-explorer/src/index.css` and Results components. Cover:\n- `body`, app shell, global header, secondary Results nav, footer;\n- Home hero/filter bands and leaderboard surfaces;\n- Benchmark and platform detail controls, filter panels, compare bars,\n sticky headers, sticky columns, and large tables;\n- Compare builder, decision summary, guardrails, receipts, query diff,\n result cards, and warning callouts;\n- Query Workbench facets, visible columns, exports, result table,\n Advanced SQL, starter queries, and SQL result table;\n- loading, empty, error, warning, disabled, selected, hover, focus, and\n reduced-color states.\n\nRemove tests that assert the old hard-coded mixed dark/light background\nvalues and replace them with tests that verify both themes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retheme Results Explorer app shell, pages, forms, panels, tables, and states", "wid": "w5"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w2", "w5"], "notes": "Refactor chart color access so components do not embed light-theme hex\nvalues for axes, grid lines, labels, annotations, legends, missing states,\nor fills. Cover at least:\n- `ChartPanel.tsx`;\n- `CDFChart.tsx`;\n- `CostScatter.tsx`;\n- `DistributionBox.tsx`;\n- `DivergingBarChart.tsx`;\n- `NormalizedSpeedupChart.tsx`;\n- `PercentileLadder.tsx`;\n- `PowerBar.tsx`;\n- `QueryHistogram.tsx`;\n- `QueryTimingChart.tsx`;\n- `StackedPhase.tsx`;\n- `TimeSeries.tsx`;\n- `results-explorer/src/lib/chartTheme.ts`.\n\nUse a theme-aware chart theme API or CSS-variable reads with SSR/test\nfallbacks. Keep platform categorical colors stable enough that users can\ncompare charts across tabs, but adjust axis/grid/text colors by theme.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Results Explorer charts and SVGs theme-aware", "wid": "w6"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w5", "w6"], "notes": "Update the Results Explorer token scan and documentation so it catches\ntheme bypasses that are currently known gaps:\n- arbitrary-value Tailwind classes such as `text-[#374151]`;\n- inline SVG `fill=\\\"#...\\\"` and `stroke=\\\"#...\\\"` literals outside the\n approved chart theme source;\n- raw `rgb()`/`rgba()` colors in public surfaces;\n- landing/docs header literal colors that should be tokenized.\n\nKeep a narrow allowlist for intentional palette definitions and document\neach allowlist reason. The gate should prevent new spot fixes from\nsilently making one theme correct and the other broken.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Upgrade token/literal gates to protect both themes", "wid": "w7"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w5", "w6", "w7"], "notes": "Update or supersede artifacts that currently encode the old Results-only\nmixed theme contract, including:\n- `_project/analysis/results-explorer-theme-inventory-2026-05-07.md`;\n- `_project/analysis/results-explorer-retheme-audit-remediation-plan-2026-05-07.md`;\n- `_project/DONE/results-explorer-contract-followup/planning/results-explorer-home-theme-and-responsive-density-contract.yaml`;\n- `results-explorer/e2e/routes/home.spec.ts`;\n- any release-gate reports or operations docs that describe dark shell\n plus light analytical panels as the final product contract.\n\nDo not rewrite history in DONE items. Add clear supersession notes where\nappropriate, and ensure active tests assert the new user-selectable\nlight/dark contract instead of treating mixed surfaces as mandatory.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire stale mixed-theme documentation, tests, and release evidence", "wid": "w8"}, {"evidence": null, "item_id": "benchbox-light-dark-theme-system", "needs": ["w3", "w4", "w5", "w6", "w7", "w8"], "notes": "Add or extend browser checks for both light and dark themes at desktop\nand mobile widths across:\n- landing `/`;\n- docs `/docs/`;\n- blog `/blog/` if locally buildable;\n- Results `/results/`;\n- Results benchmark detail;\n- Results platform detail;\n- Results compare builder and completed compare;\n- Results query workbench.\n\nVerify:\n- theme toggle changes visible colors on every surface;\n- persisted choice survives reload and route changes;\n- first paint does not visibly flash the opposite theme beyond the\n accepted technical limit;\n- WCAG contrast passes for text, controls, badges, heatmaps, chart axes,\n disabled controls, selected rows, and warning/error states;\n- no chart or table becomes unreadable in either theme;\n- header remains identical across surfaces in both themes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cross-surface visual, accessibility, and persistence verification", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Supporting Infrastructure", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-03T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement a Model Context Protocol (MCP) server that enables AI agents like Claude Code\nto run benchmarks, analyze results, and interact with BenchBox programmatically.\n\nMCP (Model Context Protocol) is Anthropic's open standard for connecting AI systems to\nexternal tools and data sources. By implementing an MCP server, BenchBox becomes\naccessible to any MCP-compatible AI agent, enabling natural language benchmark\norchestration.\n\nThe MCP server will expose BenchBox functionality through three primitives:\n- **Tools**: Executable actions (run benchmarks, export results)\n- **Resources**: Read-only data (benchmark metadata, historical results, system info)\n- **Prompts**: Reusable templates (analysis patterns, comparison queries)\n\nThis enables use cases like:\n- \"Run TPC-H at scale factor 1 on DuckDB and show me the results\"\n- \"Compare my last 3 benchmark runs and identify regressions\"\n- \"Which platforms support the TPC-DS benchmark?\"\n- \"Analyze query 5 performance across all my runs\"", "id": "benchbox-mcp-server-implementation", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "BenchBox MCP Server for AI Agent Integration", "verifications": [], "work": [], "worktree": "mcp-server-integration"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "copy-paste three visually similar headers and call them unified", "why": "the current defect is independent implementations drifting."}, {"INSTEAD": "(not specified)", "dont": "make the Results global header almost identical except for spacing or CTA behavior", "why": "the user requirement is identical."}, {"INSTEAD": "(not specified)", "dont": "remove Furo's theme controls or search while harmonizing docs/blog", "why": "docs usability is outside the header defect."}, {"INSTEAD": "(not specified)", "dont": "start the full dark/light retheme in this phase", "why": "phase 1 needs a stable shell contract first."}, {"INSTEAD": "(not specified)", "dont": "use the Results secondary nav to justify changing the global header", "why": "the secondary nav is a separate product navigation layer."}], "approach": "Start by deciding the canonical global header from real rendered surfaces,\nthen make every surface consume or test-enforce that contract. Treat Results\nExplorer's secondary nav as a product-specific child nav, not as part of the\nglobal shell. Keep CSS token names compatible with the follow-up light/dark\ntheme work so phase 2 can swap values without rewriting header markup again.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-05-13T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-site-header-unification", "id": 347, "reason": "Owned by dependent TODO `benchbox-light-dark-theme-system`; this phase must avoid blocking it but should not implement it opportunistically.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Dependent TODO benchbox-light-dark-theme-system is Completed (2026-05-14); full light/dark theme switching already shipped.", "summary": "Full light/dark theme switching"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-site-header-unification", "id": 348, "reason": "The request is header parity, not new navigation architecture. Keep the Results secondary nav unless a concrete usability defect requires a separate TODO.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Deferred only if a concrete usability defect surfaces requiring a new Explorer IA TODO; none identified.", "summary": "Redesign the Results Explorer information architecture"}], "deps": [], "description": "The public BenchBox site currently ships multiple independent header\nimplementations. The landing page uses `landing/index.html` and\n`landing/style.css`; docs/blog inject a Furo-adjacent header from\n`docs/_templates/page.html` and `docs/_static/custom.css`; Results Explorer\nrenders a separate Preact header in `results-explorer/src/components/Layout.tsx`.\nThose headers differ in link set, CTA presence, max width, fixed/sticky/static\npositioning, spacing, mobile behavior, active state, and theme assumptions.\n\nThis is a product-shell defect: the user should experience one BenchBox\nglobal header everywhere, including `/`, `/docs/`, `/blog/`, and `/results/`.\nResults Explorer may keep a secondary Results-specific navigation row below\nthe global header, but the global header itself must be identical in content,\nlayout, spacing, colors, active-link behavior, and responsive behavior.\n\nThis phase intentionally solves only the shared header contract. It should\nnot attempt the full light/dark retheme; it must, however, avoid introducing\nnew header styling that will block the follow-up theme phase.", "id": "benchbox-site-header-unification", "preserves": ["Docs/blog Furo search, sidebar, edit/view buttons, keyboard navigation, and built-in theme controls remain functional.", "Existing public URLs continue to work: `/`, `/docs/`, `/blog/`, `/results/`, GitHub, and installation/run-benchmark destination.", "Header changes do not alter Results Explorer data queries, result routes, compare flow, or DuckDB loading.", "Landing smooth-scroll/copy interactions and mobile navigation remain functional.", "One global BenchBox header contract applies to landing, docs/blog, and Results Explorer.", "Results Explorer keeps a clearly separate secondary Results navigation row unless a separate IA decision changes it."], "prior_art": [{"concept": "docs/_static/custom.css:10 \u2014 current docs nav CSS, including forced dark overrides", "decision": "reuse", "path": "docs/_static/custom.css:10"}, {"concept": "docs/_templates/page.html:14 \u2014 current Sphinx/Furo injected site navigation", "decision": "reuse", "path": "docs/_templates/page.html:14"}, {"concept": "landing/index.html:20 \u2014 current public landing header markup; candidate canonical global link set", "decision": "reuse", "path": "landing/index.html:20"}, {"concept": "landing/script.js:3 \u2014 current landing smooth-scroll and mobile menu behavior", "decision": "reuse", "path": "landing/script.js:3"}, {"concept": "landing/shared/ \u2014 preferred new home for shared static shell CSS/JS if this TODO creates reusable assets", "decision": "reuse", "path": "landing/shared/"}, {"concept": "landing/style.css:55 \u2014 current landing nav spacing, fixed positioning, backdrop, and responsive rules", "decision": "reuse", "path": "landing/style.css:55"}, {"concept": "results-explorer/e2e/routes/home.spec.ts:28 \u2014 existing mixed-theme assertion that phase 2 must supersede", "decision": "supersede", "path": "results-explorer/e2e/routes/home.spec.ts:28"}, {"concept": "results-explorer/src/components/Layout.tsx:18 \u2014 current Results Explorer global header and secondary nav", "decision": "reuse", "path": "results-explorer/src/components/Layout.tsx:18"}, {"concept": "results-explorer/src/components/__tests__/Layout.test.tsx:30 \u2014 current Results header/link/active-state unit coverage", "decision": "reuse", "path": "results-explorer/src/components/__tests__/Layout.test.tsx:30"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "docs/_static/"}, {"kind": "only_modify", "path_glob": "docs/_templates/"}, {"kind": "only_modify", "path_glob": "docs/conf.py"}, {"kind": "only_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "landing/shared/"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/Layout.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/Layout.test.tsx"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Unify BenchBox site header across landing, docs/blog, and Results Explorer", "verifications": [{"command": "cd results-explorer && npm test -- src/components/__tests__/Layout.test.tsx", "description": "Results Explorer layout tests pass after header refactor", "expected": "Layout tests pass and assert global-header parity plus Results subnav separation", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 1}, {"command": "rg -n 'benchbox-site-header|site-header|nav-logo|Run benchmark|Results' landing docs/_templates docs/_static results-explorer/src/components/Layout.tsx", "description": "Landing header source uses the shared header contract", "expected": "review output shows one shared/global header contract, not divergent local implementations", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- sphinx-build -b html docs docs/_build/html", "description": "Docs build still succeeds", "expected": "Sphinx build succeeds without template or static asset errors", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run build", "description": "Results Explorer builds", "expected": "Vite build succeeds", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 4}, {"command": "Run the header parity browser check added by this TODO for landing, docs/blog, and Results at desktop and mobile widths", "description": "Cross-surface visual parity check passes", "expected": "global header labels, spacing, active state, CTA, logo, and mobile behavior match across surfaces", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "benchbox-site-header-unification", "needs": [], "notes": "Inspect the current landing, docs/blog, and Results Explorer header\nimplementations and capture a compact comparison matrix covering:\n- link order and labels;\n- external URL casing and target behavior;\n- CTA presence and destination;\n- max width, height, padding, gap, typography, border, backdrop, and z-index;\n- fixed vs sticky vs static positioning;\n- active-link semantics for Home, Docs, Blog, Results, and GitHub;\n- mobile breakpoint behavior and keyboard/focus behavior.\n\nDecision gate: choose the canonical header contract before editing code.\nThe default should be the public landing header shape unless a specific\naccessibility or routing issue requires a documented adjustment. Record\nthe chosen contract in a short analysis note or in the implementation PR\ndescription; do not let each surface make local design choices.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture current header variants and define the exact global-header contract", "wid": "w0"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w0"], "notes": "Extract the canonical header markup and CSS into reusable site-shell\nassets that landing and docs/blog can consume without duplicating class\ndefinitions. Prefer a small shared CSS file and template/include strategy\nthat works with the existing static landing deployment and Sphinx/Furo\ndocs build.\n\nPreferred durable home: keep static shared shell assets under\n`landing/shared/` and copy/reference them from docs as part of the normal\ndocs static build, or document why that path is not viable before choosing\na different location. The implementation must leave a clear source of\ntruth so the next header change has one obvious file to edit.\n\nThe shared contract must include:\n- the exact global link set and order;\n- active-state attributes/classes that can be applied by page context;\n- mobile menu behavior if the canonical header needs it;\n- no hard dependency on the Results Explorer Vite build;\n- no duplicated magic numbers between landing and docs custom CSS.\n\nDecision gate: if true shared includes are impractical because of the\ndeployment pipeline, document the constraint and keep one generated source\nof truth or a test-enforced mirror. Do not manually fork the header again.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create a shared static header asset contract for non-React pages", "wid": "w1"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w1"], "notes": "Replace the landing page's local header markup/styles with the shared\nglobal-header contract while preserving landing-only section links only\nif the contract explicitly allows an in-page secondary section nav. If\nsection links remain, they must be visually distinct from the global\nheader and must not appear as global site navigation on other surfaces.\n\nPreserve existing copy buttons, smooth-scroll behavior, mobile menu\nbehavior, and first-viewport hero layout. Any JS moved from\n`landing/script.js` must remain static-file friendly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the shared header to the landing page", "wid": "w2"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w1"], "notes": "Replace the docs/blog custom nav in `docs/_templates/page.html` and\n`docs/_static/custom.css` with the shared global-header contract. Keep\nFuro's sidebars, search, edit/view buttons, keyboard navigation, and\nbuilt-in theme controls working.\n\nRemove or revise the existing docs CSS that forces the BenchBox nav to\nstay dark under `[data-theme=\\\"light\\\"]`, because phase 2 will make the\nheader theme-aware. For this phase, preserve the current visual result\nunless the chosen canonical contract says otherwise.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the shared header to docs and blog without breaking Furo controls", "wid": "w3"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w1"], "notes": "Refactor `results-explorer/src/components/Layout.tsx` so the global\nheader matches the canonical shared header. Keep the Results Explorer\nsecondary nav (`Leaderboards`, `Benchmarks`, `Platforms`, `Compare`,\n`Query`) below the global header as a separate product nav with its own\naria-label and active-state tests.\n\nThe Results global header must not invent different spacing, link order,\nCTA text, max width, border treatment, or mobile behavior. It may use a\nPreact component wrapper only to integrate routing and active state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the shared global header to Results Explorer and preserve the Results subnav", "wid": "w4"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w2", "w3", "w4"], "notes": "Add verification that compares the global header across:\n- landing `/`;\n- docs `/docs/`;\n- blog `/blog/` if locally buildable;\n- Results `/results/`.\n\nAt minimum, assert identical visible global nav labels/order, CTA label\nand URL, logo URL, active-link semantics, and mobile behavior. Add\ndesktop and mobile visual screenshots or DOM snapshots sufficient to\ncatch future drift. Results-specific secondary nav must be asserted as\nseparate from the global header, not as a reason for global-header drift.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cross-surface header parity tests and screenshots", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "create a third benchmark base class while cleaning up two existing bases", "why": "remove, alias, or justify the secondary base."}, {"INSTEAD": "classify first.", "dont": "interpret duplicate wrapper reachability as accidental", "why": "tests assert facade behavior"}, {"INSTEAD": "(not specified)", "dont": "split core by moving files mechanically", "why": "dependency direction and public API status must drive the boundary."}], "approach": "Start with API classification and small ambiguity removals. Treat a\nbenchmark-family plugin seam as a design gate before extraction, and use tests\nto keep wrappers, registries, and loaders aligned.\n", "blocked_reason": null, "category": "Core Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-api-and-core-boundary-cleanup", "id": 349, "reason": "Large structural refactor should follow the public API decision and plugin-seam ADR.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Split all of benchbox/core into engine and benchmark packages"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-api-and-core-boundary-cleanup", "id": 350, "reason": "Wrappers are tested as public facades unless explicitly deprecated.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly rejects removal: wrapper facades are tested public surfaces unless explicitly deprecated.", "summary": "Remove all wrapper facades"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Benchmark APIs are split across tested top-level wrappers, a public\nbenchbox.base.BaseBenchmark, a small remaining core.base_benchmark class,\nlazy registries, loaders, and benchmark-family code inside broad core/base\nmixins. The wrappers are not accidental; tests assert facade behavior. The\ncleanup target is the remaining ambiguous base/loader split and the lack of a\nbenchmark-family extension boundary, not a wrapper purge.\n\nThis item classifies benchmark API surfaces, retires or justifies the second\nbase class, and prepares a benchmark-family plugin seam without a broad\nrewrite.", "id": "benchmark-api-and-core-boundary-cleanup", "preserves": ["Do not break top-level wrapper facade tests unless an explicit deprecation path lands in the same PR.", "Do not introduce platform adapter imports into core engine modules.", "Do not move benchmark-family code in a way that changes result bundle shape."], "prior_art": [{"concept": "docs/development/run-lifecycle-map.md: existing lifecycle documentation -- align any plugin seam with this flow.", "decision": "reuse", "path": "docs/development/run-lifecycle-map.md"}, {"concept": "docs/reference/backward-compatibility.md: existing base-class lifecycle entries -- update rather than invent separate lifecycle docs.", "decision": "reuse", "path": "docs/reference/backward-compatibility.md"}, {"concept": "tests/unit/test_wrapper_facades_fast.py: existing wrapper contract tests -- preserve as public API evidence.", "decision": "reuse", "path": "tests/unit/test_wrapper_facades_fast.py"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/"}, {"kind": "only_modify", "path_glob": "docs/design/"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "tests/docs/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/test_wrapper_facades_fast.py"}], "state": "done", "title": "Architecture hardening: classify benchmark APIs and reduce core boundary drift", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_wrapper_facades_fast.py tests -k 'benchmark_loader or base_benchmark' --tb=short", "description": "Run wrapper facade and benchmark loader tests.", "expected": "Wrapper and loader tests pass.", "item_id": "benchmark-api-and-core-boundary-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests -k 'import_boundary or core_boundary' --tb=short", "description": "Run import-boundary check added by this item.", "expected": "Core/platform dependency boundary tests pass or known exemptions are documented.", "item_id": "benchmark-api-and-core-boundary-cleanup", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": [], "notes": "Completed evidence at checked SHA\n1d454632ba73911bc4ff0cf0a3fb8ec22227a7a8:\n - benchbox/base.py remains the beta-public BaseBenchmark surface;\n run_with_platform remains the beta-public adapter execution hook.\n - benchbox/core/base_benchmark.py remains a distinct deprecated\n internal compatibility base, not an alias.\n - Production imports of the deprecated core base are limited to\n benchbox/core/datavault/benchmark.py and\n benchbox/core/tpcds_obt/benchmark.py.\n - tests/unit/test_wrapper_facades_fast.py asserts wrapper behavior;\n wrapper reachability is not accidental.\n - benchbox/__init__.py exposes 21 top-level benchmark facades: 7 eager\n imports and 14 lazy _BENCHMARK_REGISTRY entries.\n - benchbox/core/benchmark_loader.py resolves 23 core benchmark families\n from registry-backed CORE_BENCHMARK_CLASS_NAMES.\n - docs/reference/backward-compatibility.md now records the remaining\n core-base migration target explicitly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate benchmark API surfaces, wrapper tests, registries, and core boundary issues", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w0"], "notes": "Decision recorded in docs/reference/public-contracts.md:\n - CLI, top-level benchmark wrappers, benchbox.base.BaseBenchmark,\n BaseBenchmark.run_with_platform, registry metadata, and MCP remain\n beta-public surfaces.\n - benchbox.core.benchmark_loader is internal.\n - benchbox.core.base_benchmark.BaseBenchmark is deprecated internal\n compatibility.\n\nNo wrapper facade was collapsed or deprecated.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decision gate: define wrapper-vs-CLI-vs-loader public API contract", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w1"], "notes": "Justified as a documented deprecated internal compatibility base. It is\nretained only for datavault and tpcds_obt until those implementations\nmigrate to benchbox.base.BaseBenchmark. Runtime modules expose surface\nmarkers and docs/reference/backward-compatibility.md carries the target.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire, alias, or justify benchbox/core/base_benchmark.py", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w1"], "notes": "Added tests/unit/core/test_benchmark_api_contract.py to lock:\n - 23 registry metadata entries.\n - 23 loader-resolved core families.\n - 22 public discovery entries.\n - 21 top-level Python benchmark facades.\n - 14 lazy facades and 7 eager facades.\n - 2 core-only benchmark IDs: ai_primitives and joinorder_synthetic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile benchmark registry, wrapper, and loader counts", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w1"], "notes": "Added docs/design/future-state/benchmark-family-plugin-seam/README.md and\nlinked it from the future-state index. The design keeps wrappers and\nresult bundles stable, defines family-owned versus shared-runtime\nresponsibilities, and requires one pilot family before any broad package\nextraction.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the benchmark-family plugin seam as an ADR before extraction", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w4"], "notes": "Added a narrow AST import-boundary check over benchbox/base.py,\nbenchbox/core/base_benchmark.py, benchbox/core/benchmark_loader.py,\nbenchbox/core/__init__.py, and benchbox/core/tuning/. The check forbids\nconcrete benchbox.platforms imports in those benchmark API and tuning\nboundary files while avoiding unrelated existing core/platform couplings.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an import-boundary check for core modules that depend on platform adapters", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "enforce SimpleBenchmarkMixin requirements so broadly that intermediate mixins fail before concrete benchmark classes are defined.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "leave two editable copies of result/validation helpers without a boundary test and migration owner.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "migrate datavault or tpcds_obt without preserving their current output path conventions.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "refactor unrelated benchmark query logic while changing base contracts.", "why": "(unstated)"}], "approach": "Start by deciding whether migration is feasible in one bounded PR. If it is,\nmigrate the two deprecated-base consumers and delete the duplicated base. If\nit is not, extract shared result/validation behavior and add a hard boundary\ntest so the deprecated surface cannot attract new benchmarks. For the simple\nmixin, reuse the existing runtime contract style instead of inventing a broad\nframework.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-22T00:00:00Z", "deferrals": [], "deps": ["canonical-benchmark-id-resolution"], "description": "Independent verification at checked SHA e60986923 confirmed the deprecated\ninternal base remains load-bearing for exactly two loader benchmarks:\n\n datavault -> benchbox.core.datavault.benchmark.DataVaultBenchmark\n tpcds_obt -> benchbox.core.tpcds_obt.benchmark.TPCDSOBTBenchmark\n\n`benchbox/core/base_benchmark.py` is labeled deprecated and retained for\ninternal compatibility, but it duplicates result creation and validation\nhelpers from `benchbox/base.py`. Verified duplicated method names include\n`create_enhanced_benchmark_result`, `create_minimal_benchmark_result`,\n`_resolve_output_dir`, `validate_preflight`, `validate_manifest`, and\n`validate_loaded_data`. Their signatures are currently similar, but they are\ndifferent method objects, and `_attach_performance_snapshot` exists only on\nthe public base.\n\nIndependent verification also confirmed `SimpleBenchmarkMixin` is a\ndocstring-only duck-typed contract. A dummy class inheriting\n`SimpleBenchmarkMixin` defines successfully and fails only at runtime with\n`AttributeError: 'BadSimple' object has no attribute 'query_manager'`. This\ncontrasts with `CursorValidationQueryExecutionMixin`, which uses\n`__init_subclass__` enforcement for required methods.", "id": "benchmark-base-and-mixin-contract-consolidation", "preserves": ["Datavault and TPC-DS OBT remain importable loader benchmarks.", "Existing AMPLab, ClickBench, and SSB simple benchmark behavior remains unchanged.", "Result schema fields and validation helper return types remain backward compatible.", "TPC-DS OBT keeps separate source-data and OBT output-directory behavior."], "prior_art": [{"concept": "benchbox/base.py: public BaseBenchmark result/validation helpers - prefer as the future implementation surface.", "decision": "reuse", "path": "benchbox/base.py"}, {"concept": "benchbox/core/base_benchmark.py: deprecated compatibility base - migrate or shrink to a thin shared layer.", "decision": "reuse", "path": "benchbox/core/base_benchmark.py"}, {"concept": "benchbox/core/benchmark_mixins.py: CursorValidationQueryExecutionMixin.__init_subclass__ - reuse enforcement style where suitable.", "decision": "reuse", "path": "benchbox/core/benchmark_mixins.py"}, {"concept": "benchbox/core/simple_benchmark_mixin.py: documented SimpleBenchmarkMixin requirements - convert from prose-only to enforced contract.", "decision": "reuse", "path": "benchbox/core/simple_benchmark_mixin.py"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: import-boundary tests - natural home for deprecated-base boundary assertions.", "decision": "reuse", "path": "tests/unit/core/test_benchmark_api_contract.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_result_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/simple_benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}], "state": "done", "title": "Consolidate benchmark base and mixin contracts", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/core/test_loader_registry_parity.py tests/test_base.py -q", "description": "Base contract and loader parity tests cover deprecated-base migration or boundary.", "expected": "All selected contract tests pass.", "item_id": "benchmark-base-and-mixin-contract-consolidation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core -k 'simple or benchmark_api_contract or loader_registry_parity' -q", "description": "Simple benchmark consumers still pass their focused tests.", "expected": "Focused unit tests pass without runtime AttributeError regressions.", "item_id": "benchmark-base-and-mixin-contract-consolidation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite remains green after base/mixin changes.", "expected": "Fast tests pass.", "item_id": "benchmark-base-and-mixin-contract-consolidation", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": [], "notes": "Re-run a read-only check that iterates loader classes and prints any\nsubclass of `benchbox.core.base_benchmark.BaseBenchmark`. Also define a\nthrowaway `SimpleBenchmarkMixin` consumer with missing attributes and\nconfirm whether failure occurs at class definition or runtime. Keep raw\noutput outside git.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate deprecated-base consumers and simple-mixin enforcement gap", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w0"], "notes": "Evaluate two paths:\n 1. Migrate `datavault` and `tpcds_obt` to `benchbox.base.BaseBenchmark`\n and delete or drastically shrink `benchbox/core/base_benchmark.py`.\n 2. Extract shared result/validation methods into one mixin/helper used\n by both bases while migration remains pending.\nPrefer full migration if constructor/output-dir differences are tractable\ninside one PR. If not, use shared extraction with a dated migration TODO.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose between full migration and shared helper extraction for deprecated-base result/validation logic", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w1"], "notes": "Ensure `create_enhanced_benchmark_result`, minimal result creation,\npreflight validation, manifest validation, and loaded-data validation use\none implementation path. Preserve `datavault` and `tpcds_obt` default\noutput-directory semantics, including source-data paths for TPC-DS OBT.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate or de-duplicate result creation and validation helpers", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w2"], "notes": "If full migration is completed, assert no loader benchmark subclasses the\ndeprecated base and consider deleting the file. If shared extraction is\nchosen, assert only the explicitly approved migration stragglers use it\nand link that approval to a TODO or documented deadline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an import-boundary test for deprecated-base usage", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w0"], "notes": "Apply the existing `CursorValidationQueryExecutionMixin.__init_subclass__`\npattern if practical. If attribute enforcement is too noisy for mixin\ncomposition, define a `Protocol` and add focused tests for AMPLab,\nClickBench, and SSB. The enforcement should catch missing\n`query_manager`, `_table_load_order`, `_get_table_schema`, and query\nexecution methods before `_load_data` or `run_benchmark` fails deep at\nruntime.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enforce `SimpleBenchmarkMixin` structural requirements at class definition or via Protocol tests", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w2", "w3", "w4"], "notes": "Cover `datavault` and `tpcds_obt` construction, output directory\ndefaults, validation fallback identifiers, and result creation if they\nmigrate. Add a negative test for an invalid `SimpleBenchmarkMixin`\nconsumer and positive coverage for the existing consumers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for migrated base behavior and mixin contract failures", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "catch every constructor TypeError to detect missing `parallel`", "why": "internal constructor bugs must surface directly."}, {"INSTEAD": "(not specified)", "dont": "keep two same-named functions with different return surfaces without an explicit compatibility story.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "remove a public alias without checking docs/tests for import expectations.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "switch CLI preview code to wrapper classes by accident", "why": "sample query preview currently needs core implementation internals."}], "approach": "Rename or parameterize for intent first, then remove the broad constructor\nfallback. Keep compatibility aliases only where external callers may already\nimport the old name. The key design rule is that function names and exception\nhandling should describe the contract instead of relying on caller memory.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-22T00:00:00Z", "deferrals": [], "deps": ["canonical-benchmark-id-resolution"], "description": "Independent verification at checked SHA e60986923 confirmed that two functions\nnamed `get_benchmark_class` intentionally return different surfaces:\n\n benchbox.core.benchmark_registry.get_benchmark_class(\"tpch\")\n -> benchbox.tpch.TPCH\n benchbox.core.benchmark_loader.get_benchmark_class(\"tpch\")\n -> benchbox.core.tpch.benchmark.TPCHBenchmark\n\nThe same split was confirmed for `vector_search` and `datavault`. Both\nreturns are callable benchmark classes, so importing the wrong function can\nsilently select the public wrapper surface where a core implementation was\nexpected, or vice versa. Existing tests already alias these functions to keep\nthem straight, which is evidence that the ambiguity is real.\n\nIndependent verification also confirmed the constructor fallback concern in\n`benchbox/core/benchmark_loader.py`: a class whose constructor accepts\n`parallel` but raises `TypeError` internally is retried without `parallel`,\nreturning successfully in the mocked harness after constructor calls\n`[1, None]`. This means \"constructor does not accept `parallel`\" and\n\"constructor accepted `parallel` but had an internal TypeError\" are currently\nindistinguishable.", "id": "benchmark-class-loading-surface-contract", "preserves": ["Benchmarks that genuinely lack a `parallel` constructor argument still instantiate successfully.", "Core loader lookup remains available for internal runtime construction.", "Import errors and unknown benchmark IDs continue to include available benchmark IDs.", "Public wrapper lookup remains available for documented public surfaces."], "prior_art": [{"concept": "benchbox/cli/benchmarks.py: _show_sample_queries imports loader class lookup - preserve the core implementation requirement.", "decision": "reuse", "path": "benchbox/cli/benchmarks.py"}, {"concept": "benchbox/core/benchmark_loader.py: get_benchmark_class and get_benchmark_instance - currently resolves core classes and retries broad TypeError.", "decision": "reuse", "path": "benchbox/core/benchmark_loader.py"}, {"concept": "benchbox/core/benchmark_registry.py: get_benchmark_class - currently resolves the top-level wrapper surface.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "tests/unit/core/test_loader_registry_parity.py: existing aliases for registry_get_benchmark_class and loader_get_benchmark_class - use as the initial contract location.", "decision": "reuse", "path": "tests/unit/core/test_loader_registry_parity.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/benchmarks.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/convert.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/execution.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/discovery.py"}, {"kind": "only_modify", "path_glob": "tests/uat/test_no_cli_surface_drift.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_manager_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_selection.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_orchestrator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_convert_command.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_loader_registry_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_benchmark_stdout_suppression.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_tool_handlers.py"}], "state": "done", "title": "Clarify public versus core benchmark class loading contracts", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_loader_registry_parity.py tests/unit/core/test_benchmark_api_contract.py -q", "description": "Class lookup parity and API contract tests pass.", "expected": "All selected class lookup tests pass.", "item_id": "benchmark-class-loading-surface-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_loader_registry_parity.py -q", "description": "Benchmark loader constructor fallback tests pass.", "expected": "Constructor signature/failure mode tests pass.", "item_id": "benchmark-class-loading-surface-contract", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite remains green after loader changes.", "expected": "Fast tests pass.", "item_id": "benchmark-class-loading-surface-contract", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": [], "notes": "Run a compact read-only check for at least `tpch`, `vector_search`, and\n`datavault`, printing registry-returned class and loader-returned class.\nReproduce the constructor fallback with a mock benchmark class that raises\n`TypeError` only when `parallel` is provided. Keep raw output in `/tmp`\nand summarize the behavior in the PR.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate the public/core class surface split and constructor fallback behavior", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w0"], "notes": "Evaluate two options:\n 1. Rename functions to `get_public_benchmark_class` and\n `get_core_benchmark_class`, preserving compatibility aliases with\n deprecation notes.\n 2. Collapse lookup into one registry-owned function with\n `surface=\\\"public\\\"|\\\"core\\\"`.\nPrefer the option with the smallest public API churn and clearest import\nsites. Document which call sites intentionally need wrappers versus core\nimplementations.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose an explicit naming or API strategy for class lookup", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w1"], "notes": "Update internal imports in CLI, MCP, tests, and loader code so the call\nsite name communicates public/core intent. Keep backward-compatible\naliases only if they are part of the public or semi-public surface, and\nadd comments where alias retention is intentional.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor call sites to use intent-revealing class lookup", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w1"], "notes": "Use `inspect.signature()` or a benchmark capability marker to determine\nwhether `parallel` is accepted before instantiation. Ensure a `TypeError`\nraised from inside `__init__` is not swallowed and retried as if it were a\nsignature mismatch. Keep compatibility for benchmarks that genuinely do\nnot accept `parallel`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace broad constructor `TypeError` fallback with signature-aware or declared capability handling", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w2", "w3"], "notes": "Add or extend tests so public lookup returns wrappers, core lookup returns\ncore classes, unknown benchmark IDs still fail clearly, and a constructor\nthat raises an internal `TypeError` is surfaced without a misleading\nfallback attempt.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests that distinguish class surface and constructor failure modes", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "dispatch to it for dual-surface benchmarks.", "dont": "duplicate the cross-surface gate here", "why": "it has its own item"}, {"INSTEAD": "advisory until proven.", "dont": "land the coverage check as a hard blocker first", "why": "a noisy gate erodes trust"}, {"INSTEAD": "prefer live oracles.", "dont": "mandate full expected-results answer keys everywhere", "why": "the maintenance cost is why coverage is thin today"}, {"INSTEAD": "use an independent reference (other surface, other engine, or curated key).", "dont": "verify a benchmark against the surface under test", "why": "that is circular"}], "approach": "Inventory first (w0): a single, ideally generated, coverage matrix is the\nartifact that makes the gap legible and dispatchable. Then route dual-surface\ngaps to the cross-surface gate (w1) and solve single-surface gaps with the\ncheapest credible per-benchmark oracle (w2), reusing existing harnesses\n(cross-surface, cross-dialect, expected-results registry) rather than new\nmachinery. Close with a policy + advisory check (w3) so coverage cannot\nsilently erode.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-correctness-oracle-coverage-map", "id": 11, "reason": "Expensive and brittle to maintain; live oracles (cross-surface / differential) cover most of the value at a fraction of the upkeep. Only curate expected-results where no live oracle is feasible.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason explicitly rejects full backfill favoring live oracles; a scoped-out sub-note of the still-open parent oracle-coverage-map item. (tracked by benchmark-correctness-oracle-coverage-map)", "summary": "Backfill full expected-results answer keys for every benchmark"}], "deps": [], "description": "The TPC-Havoc equivalence work established that a benchmark can have a real\ncorrectness oracle without a hand-maintained answer key (variant-vs-canonical\nand SQL-vs-DataFrame, compared live on a bounded DuckDB cell). Applying that\nlens across the suite exposes how little of it is actually guarded:\n\n- Stored expected results exist for ONLY tpch and tpcds\n (benchbox/core/expected_results/tpch_results.py, tpcds_results.py).\n- Live equivalence gates exist for ONLY tpchavoc (SQL variant + DataFrame),\n plus cross-engine sampling for tpchavoc (PRs #803/#806).\n- Every other benchmark - ssb, clickbench, amplab, coffeeshop, datavault,\n flightdata, joinorder, joinorder_synthetic, nyctaxi, tsbs_devops, tpcds_obt,\n tpch_skew, and the primitive suites (read/write/transaction/metadata/ai),\n tpcdi - executes \"green\" with NO automated check that any result is correct.\n\nTwo complementary mechanisms cover most of this gap. The cross-surface\nSQL<->DataFrame gate (owned by benchmark-cross-surface-equivalence-gate)\nhandles dual-surface benchmarks. THIS item owns (a) the coverage map itself -\na single authoritative matrix of which oracle, if any, guards each shipped\nbenchmark - and (b) the benchmarks that cross-surface cannot reach: single-\nsurface benchmarks (one of SQL or DataFrame only) and benchmarks whose two\nsurfaces are not result-comparable. Those need a different oracle: a\ndifferential check against a second engine on the same SQL, a small curated\nexpected-results subset, or a documented structural invariant - chosen per\nbenchmark from evidence.\n\nThe end state is a policy: every shipped benchmark declares its correctness\noracle, and a benchmark with \"none\" is visible (and ideally flagged), so the\nsuite cannot quietly grow more unverified surfaces.\n", "id": "benchmark-correctness-oracle-coverage-map", "preserves": ["Existing oracles (tpch/tpcds expected results, tpchavoc gates) are unchanged and remain authoritative for their benchmarks.", "No new hard CI gate lands noisy/flaky; advisory-first, promote to blocking only once stable.", "benchbox run default behavior is unchanged for every benchmark; coverage work is additive."], "prior_art": [{"concept": "_project/TODO/main/planning/benchmark-cross-surface-equivalence-gate.yaml - the dual-surface mechanism this map dispatches to; do not duplicate it.", "decision": "reuse", "path": "_project/TODO/main/planning/benchmark-cross-surface-equivalence-gate.yaml"}, {"concept": "_project/TODO/main/planning/testing-approach-correctness-gates.yaml - the existing real-result gate / test-quality work; this map is complementary (coverage inventory + ungated-benchmark campaign), not a re-do of it.", "decision": "reuse", "path": "_project/TODO/main/planning/testing-approach-correctness-gates.yaml"}, {"concept": "_project/TODO/main/planning/tpchavoc-cross-dialect-equivalence-sampling.yaml and tpchavoc-third-engine-equivalence-sampling.yaml - the cross-engine/differential exemplars for the single-surface fallback.", "decision": "reuse", "path": "_project/TODO/main/planning/tpchavoc-cross-dialect-equivalence-sampling.yaml"}, {"concept": "benchbox/core/benchmark_registry.py - the authoritative list of shipped benchmarks to enumerate the map against.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "benchbox/core/expected_results/registry.py - register_benchmark_provider / list of registered providers; the place a coverage check would read 'has expected results'.", "decision": "reuse", "path": "benchbox/core/expected_results/registry.py"}, {"concept": "benchbox/core/tpchavoc/equivalence.py and dataframe_equivalence.py - the live-oracle exemplars (variant + cross-surface).", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/"}, {"kind": "only_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "active", "title": "Map correctness-oracle coverage across all benchmarks and close the unguarded ones", "verifications": [{"command": "run the coverage map generator; diff against the registry", "description": "Coverage matrix enumerates every registered benchmark with its current oracle", "expected": "every shipped benchmark appears with an oracle classification (including NONE)", "item_id": "benchmark-correctness-oracle-coverage-map", "last_result": null, "last_run": null, "seq": 1}, {"command": "pick one w2 benchmark, plant a wrong result, run its new oracle", "description": "A single-surface benchmark gains an independent oracle that catches a planted defect", "expected": "the oracle flags the defect", "item_id": "benchmark-correctness-oracle-coverage-map", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": [], "notes": "Implemented `_project/scripts/generate_oracle_coverage_map.py` (per the\ngating open_question default: GENERATED from live sources, not a curated\ndoc). It reads the benchmark registry (surfaces via `num_queries` /\n`supports_dataframe`), the expected-results provider registry, the\ncross-surface `GATES` registry, and TPC-Havoc gate-module presence, and\nemits checked-in artifacts `_project/analysis/oracle-coverage-map.{md,json}`.\n\nCurrent state (23 benchmarks): 4 GUARDED \u2014 tpch/tpcds (expected-results),\ntpchavoc (variant-equivalence + cross-surface-variant), ssb (cross-surface)\n\u2014 and 19 UNGUARDED. Of the 19: 17 are dual-surface (cross-surface gate\ncandidates, w1) and 2 are single-surface SQL-only (ai_primitives,\nvector_search) needing a fallback oracle (w2).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build the authoritative correctness-oracle coverage matrix for every shipped benchmark", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": ["w0"], "notes": "For benchmarks the matrix shows as dual-surface with oracle=NONE, the fix is\nbenchmark-cross-surface-equivalence-gate. Cross-reference rather than\nre-implement; this item only tracks that they get covered there.\n\nDispatch list from the w0 map (17 dual-surface UNGUARDED): amplab,\nclickbench, coffeeshop, datavault, flightdata, h2odb, joinorder,\njoinorder_synthetic, metadata_primitives, nyctaxi, read_primitives, tpcdi,\ntpcds_obt, tpch_skew, transaction_primitives, tsbs_devops, write_primitives.\nCAVEAT (see map): write/DML/nondeterministic ones (write_primitives,\ntransaction_primitives, metadata_primitives, tpcdi) may not be\nresult-comparable across surfaces and likely belong in w2 (invariant\noracle) rather than the cross-surface gate.\n\nSTATUS SYNC (2026-06-21) from benchmark-cross-surface-equivalence-gate: ssb,\namplab, coffeeshop are GATED, and clickbench + joinorder_synthetic are now\nPROMOTED to enforced GATES (campaign w4) - their prerequisites landed (w9\nloader dtype/empty-string semantics, w8 tie-aware top-N comparator), both run\nas blocking pr.yml steps with medium-marker fast-lane tests, and the oracle\ncoverage map (regenerated here) shows both as cross-surface \"guarded\".\n\nSTATUS SYNC (2026-06-24): h2odb is now an enforced cross-surface GATE\n(single trips table, ids Q1..Q10 1:1; blocking pr.yml step\n`make h2odb-cross-surface-equivalence-report` + medium-marker fast-lane test;\ngated on a bounded SF=0.01 ~100k-row cell because its generator base is the\n10M-row small tier). Wiring it caught and fixed a REAL DataFrame bug: the\nPolars Q9 percentile defaulted to \"nearest\" interpolation (disagreeing with\nboth pandas and SQL PERCENTILE_CONT) - fixed to linear. Q9's only residue is a\nclassified DECIMAL(8,2)-scale percentile difference (DuckDB returns the\npercentile at the column's 2-decimal scale; the DataFrame computes it over\nfloat64). So 6 dual-surface benchmarks are guarded. Remaining dual-surface\ndispatch targets (one focused gate PR each, several need id normalization or a\nbounded data-source check first): datavault, flightdata, joinorder (SF=1-only -\nuse joinorder_synthetic), nyctaxi, read_primitives, tpcds_obt, tpch_skew,\ntsbs_devops. This item only tracks coverage; the gates themselves are owned there.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Dispatch dual-surface, no-oracle benchmarks to the cross-surface gate", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": ["w0"], "notes": "For benchmarks with only one comparable surface (or non-comparable\nsurfaces), pick the cheapest credible oracle per benchmark: differential\nvs a second engine on the same SQL (reuse the cross-dialect harness), a\nsmall curated expected-results subset via the existing registry, or a\ndocumented structural invariant where results are nondeterministic.\nPrioritize the canonical/most-used benchmarks. Verify each against an\nindependent reference, never against the surface under test alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Choose and apply a fallback oracle for single-surface, no-oracle benchmarks", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": ["w1", "w2"], "notes": "Delivered ahead of w1/w2 (the visibility guard is independent of building\nthe oracles). `make oracle-coverage-map-check` (and the generator's\n`--check`) fail when the checked-in map is stale. CI surfacing on develop\nPRs is via TWO paths: the fast unit test\n`tests/unit/test_oracle_coverage_map.py::test_checked_in_artifacts_are_current`\nruns in the pr.yml `code-test` job (`-m fast`), and an explicit\n`make oracle-coverage-map-check` step sits in the pr.yml `code-lint` job\nnext to `compat-docs-check`. It is also in the `ci-lint` aggregate\n(post-merge / local `make pr-preflight` parity). Mechanism: adding a\nbenchmark changes the generated map -> the drift check fails -> the author\nmust regenerate and commit, which surfaces the new benchmark's UNGUARDED\nclassification in review rather than letting it slip in silently. The map's\nUNGUARDED list is the always-current \"no oracle\" inventory. Advisory in the\nsense that UNGUARDED is allowed-and-visible; only staleness is blocked.\n\nw1/w2 (actually building oracles for the 19 UNGUARDED benchmarks) remain the\nopen campaign; the w0 map makes them dispatchable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make 'no oracle' visible and prevent silent regressions in coverage", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use the DataFrame runner/mixin for query dispatch AND the production loader/adapters for data loading (w5).", "dont": "execute DataFrame queries OR load their data via a bespoke path", "why": "the gate must reflect the real product run path"}, {"INSTEAD": "one bounded small-SF DuckDB cell per benchmark.", "dont": "gate a full platform matrix", "why": "routine PRs must stay cheap"}, {"INSTEAD": "fix the wrong surface.", "dont": "silence a real divergence by adding it to the baseline", "why": "the baseline is for intentional differences only"}, {"INSTEAD": "generalize and reuse it.", "dont": "write a new result comparator or data builder", "why": "the TPC-Havoc harness already does this"}], "approach": "Generalize the proven TPC-Havoc DataFrame gate rather than inventing a new\ncomparator: extract a shared harness (w0), inventory applicability (w1), stand\nup a production-faithful DataFrame-loading substrate (w5) and fix the loader\nbugs it surfaces (w6), re-base ssb onto it (w7), then land report-mode across\nthe ungated dual-surface benchmarks (w2), burn down (w3), and flip bounded CI\ngates with fast-lane companions (w4). SQL is the live reference for its own\nDataFrame surface; no hand-maintained answer keys; the DataFrame surface runs\non its real production data path so divergences are real, not dtype artifacts.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-cross-surface-equivalence-gate", "id": 6, "reason": "Cross-engine sampling is its own discipline (see tpchavoc-cross-dialect-equivalence-sampling); keep these gates DuckDB-bounded first.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Extending equivalence gating beyond DuckDB matches the scope of the open cross-surface gate breadth rollout item. (tracked by cross-surface-gate-rollout-followups)", "summary": "Cross-surface equivalence on a second engine"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-cross-surface-equivalence-gate", "id": 7, "reason": "Cross-surface cannot apply; their oracle gap is owned by benchmark-correctness-oracle-coverage-map.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason states this gap is owned by benchmark-correctness-oracle-coverage-map, which is open and tracks it.", "summary": "Benchmarks that ship only one surface"}], "deps": [], "description": "TPC-Havoc now ships two result-equivalence gates that proved a discipline:\nexecute a candidate query and a trusted reference over the SAME bounded\nsmall-SF DuckDB dataset and compare results live (order-insensitive, float-\ntolerant), with no hand-maintained expected answers. The SQL gate\n(benchbox/core/tpchavoc/equivalence.py) compares each SQL variant to canonical\nTPC-H; the DataFrame gate (benchbox/core/tpchavoc/dataframe_equivalence.py)\ncompares each DataFrame variant to canonical TPC-H SQL. Together they found and\nfixed ~42 silent correctness defects.\n\nThat discipline is currently TPC-Havoc-only, by construction. But ~14\nbenchmarks ship a DataFrame query surface (amplab, clickbench, coffeeshop,\ndatavault, flightdata, joinorder, nyctaxi, read_primitives, ssb, tpcds,\ntpcds_obt, tpch, tpch_skew, tsbs_devops) in ADDITION to their SQL surface, and\nnothing checks that the two surfaces agree. Worse, stored expected results\nexist for ONLY tpch and tpcds (benchbox/core/expected_results/tpch_results.py,\ntpcds_results.py) - so for every other dual-surface benchmark there is NO\ncorrectness oracle of any kind: SQL and DataFrame implementations both execute\n\"green\" while their results are unverified and may silently disagree.\n\nCross-surface equivalence is the cheap, maintenance-free oracle for exactly\nthis gap: SQL and DataFrame are independent implementations of the same logical\nquery, so a divergence proves at least one is wrong - without needing a\nhand-curated answer key. This work generalizes the TPC-Havoc DataFrame-gate\nharness into a shared, benchmark-agnostic helper, then rolls out a bounded\nSQL<->DataFrame equivalence gate to the dual-surface benchmarks, SQL as the\nreference for its own DataFrame surface.\n\nNote: for tpch/tpcds the expected-results registry already validates each\nsurface against a stored answer key (transitive equivalence), so they are\nlower priority here; the highest value is the dual-surface benchmarks with NO\noracle today (ssb first - canonical and small - then clickbench, joinorder,\ncoffeeshop, etc.).\n", "id": "benchmark-cross-surface-equivalence-gate", "preserves": ["ResultValidator tolerance/checksum semantics are unchanged; reuse, do not fork, the comparator and the DuckDB SQL-reference data builder.", "The TPC-Havoc SQL and DataFrame gates stay green with empty KNOWN_DIVERGENCES and unchanged behavior.", "The production DataFrame loader/adapters are reused, not forked; any change there (w6) is a genuine loader bug fix that keeps the real DataFrame run path working (re-verify the DataFrame benchmark run), never gate-only special-casing.", "benchbox run default (execution-only) behavior is unchanged for every benchmark; the gates are additive."], "prior_art": [{"concept": "benchbox/core//dataframe_queries(.py or package) and /queries.py (or get_queries) for each dual-surface benchmark - the two surfaces to compare (note: some, e.g. ssb/tpcds/clickbench, ship dataframe_queries as a package directory rather than a single module).", "decision": "reuse", "path": "benchbox/core//dataframe_queries(.py"}, {"concept": "benchbox/core/expected_results/ (registry.py, tpch_results.py, tpcds_results.py) - the only existing correctness oracles; documents that coverage stops at tpch/tpcds.", "decision": "reuse", "path": "benchbox/core/expected_results/"}, {"concept": "benchbox/core/runner/dataframe_runner.py and benchbox/platforms/dataframe/benchmark_mixin.py - how DataFrame queries actually execute and materialize; the gate must use the same execution path, not a bespoke one.", "decision": "reuse", "path": "benchbox/core/runner/dataframe_runner.py"}, {"concept": "benchbox/core/tpchavoc/dataframe_equivalence.py - build_dataframe_contexts, materialize_rows, fetch_canonical_rows, find_dataframe_divergences, main; the template to generalize (it already compares a DataFrame result to a SQL reference on DuckDB).", "decision": "reuse", "path": "benchbox/core/tpchavoc/dataframe_equivalence.py"}, {"concept": "benchbox/core/tpchavoc/equivalence.py - build_duckdb_with_tpch, find_divergences, strip_top_n, KNOWN_DIVERGENCES, main; the SQL-side harness and the empty-baseline fail-on-divergence pattern.", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py"}, {"concept": "tests/integration/test_tpchavoc_variant_equivalence.py - the fast default-lane regression companion; mirror it per gated benchmark.", "decision": "reuse", "path": "tests/integration/test_tpchavoc_variant_equivalence.py"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "active", "title": "Generalize the result-equivalence harness and gate SQL<->DataFrame equivalence for dual-surface benchmarks", "verifications": [{"command": "make tpchavoc-dataframe-equivalence-report", "description": "Generalized harness reproduces the existing TPC-Havoc DataFrame gate result", "expected": "still 0 divergent (re-expressed on the shared harness, unchanged)", "item_id": "benchmark-cross-surface-equivalence-gate", "last_result": null, "last_run": null, "seq": 1}, {"command": "introduce a wrong projection in one SSB DataFrame query, run the SSB cross-surface gate", "description": "A new dual-surface gate fails when one surface is deliberately broken", "expected": "gate FAILS (non-zero), naming the divergent query", "item_id": "benchmark-cross-surface-equivalence-gate", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": [], "notes": "Factor the data-build + run-SQL + run-DataFrame + compare loop out of\ntpchavoc/dataframe_equivalence.py into a shared core helper parameterized by\nbenchmark (DDL/data builder, SQL query accessor, DataFrame query accessor,\nvalidator). Re-express the existing TPC-Havoc DataFrame gate on top of it\nwith NO behavior change (its gate must stay green and byte-equivalent in\nresult). Reuse validate_variant_equivalence / the result validator and the\nDuckDB data builder; do not fork them.\n\nDONE: shared harness landed at benchbox/core/equivalence/dataframe_surface.py\n(build_dataframe_contexts, materialize_rows, fetch_reference_rows,\nfind_surface_divergences, SurfaceDivergence, _normalize_value). The TPC-Havoc\nDataFrame gate (benchbox/core/tpchavoc/dataframe_equivalence.py) is now a thin\nwrapper over it: it injects the canonical-TPC-H reference, the variant/backend\ncell enumeration, and the reused validate_variant_equivalence comparator +\nbuild_duckdb_with_tpch data builder. Gate re-verified byte-equivalent:\n\"checked 440 variant-backend cells - 0 divergent, 440 equivalent\". Fast-lane\ntests/integration/test_tpchavoc_dataframe_variant_equivalence.py still passes;\nnew fast-lane unit coverage at tests/unit/core/equivalence/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract a benchmark-agnostic cross-surface equivalence harness from the TPC-Havoc DataFrame gate", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w0"], "notes": "NOT YET COMPLETE \u2014 kept open so downstream work units that `needs: [w1]` are not\nunblocked with unverified query mappings: nyctaxi and tsbs still need independent\nmapping verification, and clickbench's SQL id set / construction is unconfirmed.\nFlip to done only once those three are verified.\n\nFor each benchmark with both surfaces, confirm the SQL and DataFrame query\nsets cover the same logical queries (matching ids/keys) and can run on a\nshared DuckDB cell. Record which are in scope, which are single-surface\n(out of scope here - owned by benchmark-correctness-oracle-coverage-map),\nand any that need per-benchmark normalization (top-N/LIMIT, column order).\n\nIN PROGRESS. Surface scan (dataframe_queries module OR package) confirms a\nDataFrame surface for: amplab, clickbench, coffeeshop, datavault,\nflightdata, joinorder, joinorder_synthetic, nyctaxi, read_primitives, ssb,\ntpcds, tpcds_obt, tpch, tpch_skew, tsbs_devops, h2odb (+ tpchavoc). The\nungated, no-expected-results dual-surface targets per the TODO are: ssb,\nclickbench, joinorder, coffeeshop, amplab, nyctaxi, flightdata, tsbs_devops,\ntpcds_obt, tpch_skew, read_primitives.\n\nConfirmed and gated: ssb - SQL get_queries() and the SSB_DATAFRAME_QUERIES\nregistry both key by identical ids Q1.1..Q4.3 (13 queries, 1:1). Per-\nbenchmark normalization finding: schema shape is heterogeneous - TPC-H/\ntpchavoc expose typed Table/Column objects, but ssb/amplab/coffeeshop/etc.\nexpose dict-shaped TABLES. The shared harness now offers\nbuild_dataframe_contexts_from_specs((name, [(col, sql_type)])) so both\nshapes reuse one DuckDB->Arrow->register loop.\n\nVERIFIED id-correspondence + feasibility map for the ungated targets\n(probed SQL get_queries() vs each DataFrame registry's get_query_ids()):\n - ssb 13 ids, 1:1 (Q1.1..Q4.3) GATED (#815).\n - coffeeshop 11 ids, 1:1 ready; but DuckDB\n ->Arrow date32 contexts break its pandas impls (date-vs-string\n compare) -> needs the data-path decision below before gating.\n - flightdata 20 ids, 1:1 (friendly names) ready; NOTE no\n synthetic generator module (uses real downloaded data) - confirm a\n bounded data source before gating.\n - read_primitives 157 SQL vs 152 DF gate the 152-id\n intersection (5 SQL-only ids have no DF impl).\n - amplab 8 ids, count matches; DF ids are Q-prefixed ('1'->'Q1')\n -> mechanical id normalization in the spec.\n - tpch_skew 22 ids, same; DF Q-prefixed ('1'->'Q1') -> same.\n - nyctaxi 25 ids; SQL friendly-names vs DF 'Q1'..'Q25' -> AMBIGUOUS\n ordering map; needs independent verification of which SQL id == which\n Qn before asserting (do NOT guess).\n - tsbs_devops 18 ids; same friendly-name vs 'Qn' ambiguity as nyctaxi.\n - joinorder 113 ids, 1:1 string keys BUT scale_factor=1.0-only\n (canonical IMDb) -> does NOT fit the SF=0.1 bounded cell; use SF=1 or\n joinorder_synthetic, a separate decision.\n - clickbench 43 DF ids; benchmark construction differs (abstract base)\n -> confirm SQL id set + construction before gating.\n - tpcds_obt (per source TODO) DF is a 3-query stub vs ~89 SQL - gate\n only the implemented intersection or defer.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "in_progress", "summary": "Inventory dual-surface benchmarks and confirm SQL<->DataFrame query-id correspondence", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5", "w6", "w7"], "notes": "Prioritize benchmarks with NO expected-results oracle: ssb, clickbench,\njoinorder, coffeeshop, amplab, nyctaxi, flightdata, tsbs_devops, tpcds_obt,\ntpch_skew, read_primitives. Report (benchmark, query, kind, deltas) without\nasserting. Capture stdout to /tmp and summarize counts in the PR/TODO.\n\nIN PROGRESS. ssb reported clean on the INTERIM DuckDB->Arrow substrate (#815):\n26 query-backend cells (13 queries x 2 backends) at SF=0.1, 0 divergent. This\nre-runs on the production-loader substrate (w5-w7); ssb is expected to stay\nclean but is re-confirmed there. Remaining ungated benchmarks are reported on\nthe production substrate as each gate is built (now needs w5/w6/w7).\n\namplab reported on the production substrate (SF=0.1, 16 query-backend cells =\n8 queries x 2 backends): 10/16 cells divergent on the FIRST report - exactly\nthe date-vs-string defect class w6 predicted (the 5 queries touching the\ndate32-typed uservisits.visitDate: Q2/Q2a/Q3/Q4/Q5, both backends), each\nraising \"cannot compare date to a string value\" under the production loader's\ndtypes. Burned down in w3 (see below); amplab now reports 0/16 divergent.\n\nAPPLICABILITY SWEEP: a reproducible sweep\n(_project/scripts/cross_surface_applicability_sweep.py ->\n_project/analysis/cross-surface-applicability.md, locked by\ntests/unit/test_cross_surface_applicability.py) classifies the dual-surface\nUNGUARDED benchmarks by whether they ship a DataFrame query `QueryRegistry`\n(the registry the cross-surface gate builders consume directly) - the\nauthoritative gate-applicability signal. `supports_dataframe` over-counts\n(it is a LOADING flag); the production query *resolver* under-counts (it\nmisses per-benchmark `_DATAFRAME_QUERIES` registries - e.g. it\nreturned 0 for coffeeshop, which #842 nonetheless gated). Registry detection\nfixes both.\n\nRESULT (16 unguarded candidates; coffeeshop already gated/excluded):\n - GATEABLE, ids overlap as-is (6): clickbench, flightdata, h2odb,\n joinorder, joinorder_synthetic, read_primitives (152/157 ids).\n - GATEABLE, needs id normalization first (6): amplab, datavault, nyctaxi,\n tpch_skew, tsbs_devops (ids Q-prefixed/friendly vs SQL ids), tpcds_obt\n (only 3 DF queries vs 89 SQL - partial).\n - w2 FALLBACK, NO DataFrame query registry (4): metadata_primitives,\n tpcdi, transaction_primitives, write_primitives. Only these four need a\n differential second-engine or curated expected-results oracle.\nSo the cross-surface gate burn-down is ~12 benchmarks, not 2 (the earlier\nresolver-based \"only 2\" was wrong).\n\nCAVEAT proven during the sweep: a GENERIC build is not load-faithful (it\nloads DataFrame contexts without the schema/headers the production loader\napplies, yielding spurious \"column not found\" divergences), so enumerating\nREAL divergence counts requires a load-faithful per-benchmark builder like\nbuild_ssb_duckdb. That builder work is w3 (clickbench done; the other 11\ngateable benchmarks next, normalizing ids where the sweep flags it).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "in_progress", "summary": "Run the cross-surface gate in report mode over the ungated dual-surface benchmarks and enumerate divergences", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w2"], "notes": "DONE. Burn-down complete for every wired/staged gate: ssb (0), amplab (10\ndate-vs-string impl bugs fixed in the DataFrame surface), coffeeshop (0),\njoinorder_synthetic (loader dtype/null fixes, w9 -> 26/26), clickbench (the\nremaining cells were the tie-ambiguous top-N class, not logic bugs - resolved\nby the w8 tie-aware comparator, with Q18's order-less LIMIT classified as a\njustified known-divergence). No real divergence was masked into a baseline;\nthe only baseline entry is ClickBench Q18 (genuinely order-less LIMIT). The\nlast in_progress reason was the ClickBench tie cells, which w8 cleared - so\nw3 closes with w8. Original plan below.\n\nFor each divergence, determine which surface is wrong and fix it; verify the\nfix makes the two surfaces agree. Where a difference is intentional and\ndefensible (e.g. a surface deliberately omits a presentational LIMIT),\nrecord it as a classified, justified known-divergence entry, mirroring the\nTPC-Havoc baseline discipline (default: empty baseline, every query matches).\n\nIN PROGRESS. ssb: no burndown required (0 divergences); baseline empty.\n\namplab: 10 divergences, all the projection-faithfulness/dtype class (NOT a\nloader bug). Root cause: the amplab DataFrame impls compared the date-typed\nvisitDate column to ISO-STRING literals (\"2000-01-01\"), which raises under\nthe production loader's date32 dtype - i.e. the production amplab DataFrame\nrun was itself broken for Q2/Q2a/Q3/Q4/Q5. Fixed the WRONG surface (the\nDataFrame impls), mirroring the TPC-H DataFrame queries whose date params are\ntyped datetime.date objects: AMPLAB_DEFAULT_PARAMS (and the impl fallbacks)\nnow carry date(2000,1,1)/date(2000,1,3) instead of strings, so the\ncomparison succeeds on both backends. No gate-local cast, no loader change,\nno baseline entry. Verified against CANONICAL SQL (the gate), not a sibling\nDataFrame impl: 0/16 divergent, with Q2 (70 rows), Q2a (6), Q4 (100)\nmeaningfully exercising the date filter+join (non-vacuous). baseline empty.\n\nclickbench: load-faithful gate WIRED (build_clickbench_duckdb), STAGED_GATES\n(NOT GATES), runnable via `make clickbench-cross-surface-equivalence-report`.\n\nBURN-DOWN PROGRESS (full detail:\n_project/analysis/clickbench-cross-surface-divergences.md):\n - DETERMINISM FIX (prerequisite): the ClickBench generator was UNSEEDED, so\n every run produced different data and top-N/tie-sensitive cells flipped\n divergence run-to-run (the gate was flaky). Added random.seed(42) in\n _generate_data_local (mirrors SSBDataGenerator); the gate is now\n deterministic (verified: two runs report the identical divergence set).\n - ALL 10 engine/error-category cells FIXED: added UnifiedStrExpr.replace\n (Q29) and UnifiedLazyFrame.slice (Q39-42) to the shared abstraction;\n compared EventDate to native date literals not strings (Q37-43); pandas\n `<> ''` filters now exclude NULL via .notna() to match SQL/Polars\n (Q25/27). 1129 dataframe/clickbench unit tests still pass.\n - REMAINING: 27 stable value-divergence cells across 14 queries. TRIAGE\n (verified by direct SQL-vs-DataFrame set/count comparison): these are\n mostly NOT logic bugs. The DataFrame computes CORRECT aggregates; the\n divergences are the gate's order-sensitive row-by-row comparison of\n tie-ambiguous top-N (`ORDER BY col ... LIMIT N` with ties spanning the\n cutoff). E.g. Q32/Q33 top-10 are all count=1 (thousands of count-1\n groups, LIMIT keeps an arbitrary 10); Q16 count distribution is IDENTICAL\n on both surfaces (only tied rows reorder). So the fix is GATE-LEVEL\n (deterministic total-order tie-breaker on both surfaces, OR a tie-aware\n validator, OR classify tie-ambiguous queries), NOT per-query ClickBench\n fixes. Exception: Q17/Q24 empty-string-vs-None is the deferred loader\n null/dtype materialization issue (see joinorder_synthetic). Detail:\n _project/analysis/clickbench-cross-surface-divergences.md.\nSo ClickBench has ~no genuine DataFrame logic bugs; promote STAGED_GATES->GATES\nonce the gate handles tie-ambiguity + the loader dtype fix lands (w4).\njoinorder_synthetic and the other registry-bearing benchmarks (per the\ncorrected applicability sweep) are the next gates to wire.\n\nGENERATOR SEEDING AUDIT (_project/analysis/gateable-generator-seeding-audit.md):\nan unseeded generator makes any cross-surface gate flaky (as ClickBench was).\nAudited every gateable benchmark's generator. Most are seeded (ssb, amplab,\nh2odb, tpch_skew, tsbs_devops, clickbench) or RNG-free (coffeeshop static\nseed-data, read_primitives). FIX: joinorder_synthetic used unseeded random\nand is the next gate to wire - added random.seed(42) in its\n_generate_data_local; verified its data tables are now byte-identical across\nruns (the only residual file-byte diff is the manifest timestamp, which the\ngate does not read). The five external/derived-data benchmarks (datavault,\nflightdata, joinorder, nyctaxi, tpcds_obt) have no synthetic RNG; confirm\ntheir bounded data source is stable when wiring each.\n\njoinorder_synthetic: gate WIRED (build_joinorder_synthetic_duckdb, STAGED_GATES,\n`make joinorder-synthetic-cross-surface-equivalence-report`; 13 queries x 2\nbackends). Wiring surfaced a SHARED DataFrame-loader bug: the production loader\napplies column names from get_benchmark_schema_columns, but that extractor did\nnot understand joinorder_synthetic's schema shape (columns are raw DDL strings\nlike \"id INTEGER PRIMARY KEY\") -> returned no columns -> every table loaded\nHEADERLESS, all 26 cells errored. Fixed schema_utils.py (column_name/\ncolumn_sql_type) to parse DDL-string columns - a general fix benefiting any\nstring-schema benchmark - dropping divergences 26 -> 10 (16 cells now pass;\n791 dataframe/equivalence unit tests still pass). REMAINING 10 cells are\ndtype-mismatch errors (loader applies column names but type-INFERS dtypes\ninstead of applying schema types: \"expected String type\" on 1a/1b/5a/8a/9a/10a;\n\"string vs numeric\" on 4a/12a). Follow-up: apply schema column TYPES when\nloading the DataFrame surface (types now available from column_sql_type). Detail:\n_project/analysis/joinorder-synthetic-cross-surface-divergences.md.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Burn down divergences (fix the wrong surface) or declare intentional ones", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w3", "w8", "w9"], "notes": "DONE for the two STAGED gates (the prerequisite-driven promotion w8/w9\nunblocked). clickbench and joinorder_synthetic moved STAGED_GATES -> GATES in\ncross_surface.py; both wired as BLOCKING steps in the pr.yml correctness-gate\njob (make clickbench-cross-surface-equivalence-report,\nmake joinorder-synthetic-cross-surface-equivalence-report) beside ssb/amplab/\ncoffeeshop; medium-marker fast-lane regressions added\n(tests/integration/test_{clickbench,joinorder_synthetic}_cross_surface_equivalence.py)\nmirroring test_ssb_cross_surface_equivalence.py; unit test updated\n(test_clickbench_and_joinorder_are_enforced_gates). Proved each FAILS on a\nplanted divergence (clickbench Q1 count+1000 -> Q1_expression+Q1_pandas;\njoinorder 1a value+99999 -> 1a_expression+1a_pandas; no collateral).\nclickbench's only baseline entry is the order-less Q18; joinorder_synthetic is\nclean. Regenerated the oracle coverage map (make oracle-coverage-map) - both\nnow show cross-surface \"guarded\" (15 UNGUARDED, was 17 dual-surface) - and\ncommitted it; updated the coverage-map TODO w1 dispatch note. STAGED_GATES is\nnow empty. ssb/amplab/coffeeshop and TPC-Havoc gates re-verified green.\n\nFOLLOW-ON (separate focused gate PRs, tracked by\nbenchmark-correctness-oracle-coverage-map w1, NOT a blocker for this item):\nwire the remaining gateable dual-surface benchmarks one PR each - datavault,\nflightdata, nyctaxi, read_primitives (152-id intersection), tpcds_obt\n(partial), tpch_skew, tsbs_devops (several need id normalization or a bounded\ndata-source check first), and joinorder (SF=1-only; covered by\njoinorder_synthetic). Each follows the same recipe: build__duckdb,\nregister in GATES, add a pr.yml step + a fast-lane test, prove a planted\ndivergence fails, regenerate the coverage map.\n\nDONE so far (follow-on gate PRs): h2odb (2026-06-24) - enforced GATE,\nbuild_h2odb_duckdb on a bounded SF=0.01 ~100k-row cell (its generator base is\nthe 10M-row small tier, so the shared SF=0.1 would emit ~1M rows; every query\nstays discriminating at 100k). SQL/DataFrame ids correspond 1:1 (Q1..Q10).\nWiring caught a REAL DataFrame bug: the Polars Q9 percentile defaulted to\n\"nearest\" interpolation, disagreeing with both pandas (linear) and SQL\nPERCENTILE_CONT - fixed the WRONG surface by passing interpolation=\"linear\"\n(added an additive `interpolation` arg to UnifiedExpr.quantile, default\n\"nearest\" so other callers are unchanged). Q9's only residue is one classified\nknown-divergence per backend: DuckDB's PERCENTILE_CONT over the DECIMAL(8,2)\nfare_amount column returns the percentile at the column's 2-decimal scale\n(77.87) while the DataFrame computes the same linear percentile over float64\n(77.874) - verified presentational (casting the SQL arg to DOUBLE yields the\nDataFrame value), not a logic bug. Evidence: gate green + deterministic across\n3 runs (18/20 cells strictly compared, 2 classified); planted Q4 corruption\nflags only Q4_expression (no collateral); coverage map shows h2odb guarded\n(UNGUARDED 15 -> 14). The applicability sweep's gateable example was retargeted\nh2odb -> flightdata (h2odb left the unguarded-candidate set).\n\nOriginal plan: one bounded small-SF DuckDB cell per gated benchmark, wired into\nthe correctness-gate CI job and a Make target, mirroring\ntpchavoc-(dataframe-)equivalence-report. Add a medium-marker pytest per\nbenchmark like tests/integration/test_tpchavoc_variant_equivalence.py.\nProve each gate fails when a deliberate divergence is introduced into one\nsurface.\n\nIN PROGRESS. Generic cross-surface gate landed at\nbenchbox/core/equivalence/cross_surface.py (find_cross_surface_divergences,\nCrossSurfaceGate, GATES registry) reusing the shared harness +\nResultValidator; the DataFrame side executes through the real\nDataFrameQuery.get_impl_for_family dispatch. First gated benchmark: ssb -\n`make ssb-cross-surface-equivalence-report` (added to the correctness-gate\nCI job in pr.yml), fast-lane tests/integration/test_ssb_cross_surface_equivalence.py\n(medium marker), and unit coverage tests/unit/core/equivalence/test_cross_surface.py.\nVerified the gate FAILS on a planted SSB defect (only Q1.1_expression\ndiverges, no collateral).\n\nSecond gated benchmark: amplab - `make amplab-cross-surface-equivalence-report`\n(added to the correctness-gate CI job in pr.yml beside ssb), fast-lane\ntests/integration/test_amplab_cross_surface_equivalence.py (medium marker).\nRegistered build_amplab_duckdb in the GATES registry (maps SQL ids \"1\"..\"5\"\nto DataFrame ids \"Q1\"..\"Q5\"). Verified the gate FAILS on a planted amplab\ndefect (Q4 expression total_revenue doubled -> only 4_expression diverges, no\ncollateral). Remaining ungated benchmarks: clickbench, joinorder, coffeeshop,\nnyctaxi, flightdata, tsbs_devops, tpcds_obt, tpch_skew, read_primitives - one\nfocused gate PR each.\n\nBLOCKED-ON for the two STAGED gates: clickbench and joinorder_synthetic are\nWIRED in STAGED_GATES (report mode) but cannot promote to GATES until the two\ncross-cutting prerequisite fixes land - w8 (tie-aware top-N comparator, clears\nClickBench's 27 tie-ambiguous cells) and w9 (loader dtype-from-schema, clears\njoinorder_synthetic's 10 cells + ClickBench Q17/Q24). Hence w4 now needs\n[w3, w8, w9]. ssb/amplab/coffeeshop were already promoted under this item and\nare unaffected by w8/w9.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Promote to bounded CI gates and add fast-lane regressions", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w1"], "notes": "DONE. Added build_production_contexts() in core/equivalence/cross_surface.py:\nper backend it instantiates the real adapter and calls the new public\nBenchmarkExecutionMixin.load_benchmark_into_context() - the exact data path\nrun_benchmark uses (_extract_benchmark_config + _load_data_phase, prefer_parquet\ndefault), stopping short of query execution. CrossSurfaceData now carries the\nbenchmark instance + data_dir (not table_specs); the DuckDB SQL reference and\nboth DataFrame backends read the SAME generated files. The comparator and the\nDuckDB SQL-reference builder are unchanged. (No new fork of the loader: the\ngate calls the production mixin.) Original plan below.\n\nReplace the gate's bespoke DuckDB->Arrow context-build\n(build_dataframe_contexts_from_specs) with one that loads each backend's\nDataFrame context via the REAL production loader, so the gate exercises the\nsame data path (and dtypes) a production DataFrame run uses - honoring the\n\"use the real runner/mixin path, not a bespoke one\" anti-pattern for DATA\nLOADING as well as query execution.\n\nReuse, do not fork: the reference adapters' load path\n(benchbox/platforms/dataframe/{polars_df,pandas_df}.py load_table /\nload_tables_from_data_source / _load_tables_into_context) and\nbenchbox/core/dataframe/data_loader.py DataFrameDataLoader. Drive it with\nthe benchmark's schema_info (column names/types) and the correct\ndelimiter/trailing-delimiter/parquet handling that a bare\nload_tables_from_data_source call skips (this was the cause of the SSB\nmis-load). Generate data once; the SQL reference (DuckDB) and BOTH DataFrame\nbackends must read the SAME generated data so \"same bounded cell\" still holds.\nKeep the comparator and the DuckDB SQL-reference builder reused (unchanged).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a production-faithful DataFrame-context substrate for the gate (reuse the real loader)", "wid": "w5"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5"], "notes": "DONE (#821). Made the Pandas loader's date-column inference TYPE-AWARE:\n_pandas_parse_date_columns now takes the schema column types and never\ntreats a numeric-typed column as a date even if its name ends in date/time,\nso SSB's INTEGER lo_orderdate/lo_commitdate load as integers (real string\ndates like d_date VARCHAR still parse). Behavior is unchanged when types are\nabsent. Threaded benchmark -> load_table -> _load_csv_files -> read_csv ->\n_pandas_parse_date_columns; abstract load_table signature made honest. SSB\nnow loads via the production loader (gate 26/26). Routing coffeeshop through\nthe real loader surfaced GENUINE divergences (date32-vs-string impl bugs,\nvalue mismatches, a KeyError) - logged for w2/w3 burndown, NOT masked.\nOriginal plan + diagnosis below.\n\nDriving the real loader over real benchmark data exposes loader defects the\ngate must not paper over. Known, reproduced:\n * Pandas adapter date heuristic: _pandas_parse_date_columns\n (benchbox/platforms/dataframe/pandas_df.py:62) infers date columns by\n NAME suffix only (\"*date\"/\"*_date\") with no type check, then\n _coerce_date_columns casts them to date32. SSB's lo_orderdate /\n lo_commitdate are INTEGER YYYYMMDD datekeys, so the loader tries to\n date-parse integers and raises (\"month must be in 1..12\"). Fix: gate\n date-coercion on the column's SCHEMA TYPE (thread schema_info types\n through), never the name alone, so integer datekeys load as integers.\nFor each benchmark, confirm its impls run under the loader's production\ndtypes; where an impl genuinely fails under the dtype its production run\nuses (e.g. coffeeshop pandas impls compare a date column to a STRING\nliteral), that is a REAL divergence/bug to report and burn down in w3 - do\nNOT add a gate-local cast to mask it. Only fix loader code where the LOADER\nis wrong (e.g. the type-blind date heuristic), not to accommodate a wrong impl.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix production-loader correctness bugs surfaced by real benchmarks (now in-scope: benchbox/platforms/dataframe/)", "wid": "w6"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5", "w6"], "notes": "DONE. The SSB gate + its fast-lane test now build DataFrame contexts via\nbuild_production_contexts (w5); re-verified 26/26, 0 divergent at SF=0.1 and\nre-proved the planted-defect failure (corrupting Q1.1's expression impl\nflags exactly Q1.1_expression, no collateral) on the new substrate. The\ntpchavoc DataFrame gate is LEFT on its DuckDB->Arrow path (it is a\nvariant-vs-canonical gate, not cross-surface) and re-confirmed green. The\nDuckDB->Arrow builder (build_dataframe_contexts_from_specs) remains in\ndataframe_surface for tpchavoc; only the cross-surface gate moved. Original\nplan below.\n\nSwitch the merged SSB gate (#815) from DuckDB->Arrow to the w5 substrate and\nre-verify it stays green (26/26) under production dtypes; this is the\nreference example every later benchmark follows. Re-confirm the TPC-Havoc\nSQL + DataFrame gates are unchanged (the tpchavoc DataFrame gate may keep its\nown DuckDB->Arrow path - decide whether to migrate it too or leave it, since\nit is a variant-vs-canonical gate, not cross-surface). Update the fast-lane\ntests accordingly. Re-prove the planted-defect failure on the new substrate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-base the SSB gate (and the shared harness) onto the production-loader substrate; re-verify", "wid": "w7"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w3"], "notes": "DONE. Made ResultValidator.validate_results_exact opt-in tie_aware (additive,\ndefault False so the TPC-Havoc gates are byte-unchanged). Chose option (b) (a\ntie-aware comparator) over (a) (the ClickBench SQL queries are also the perf\nqueries, so adding tie-breaker ORDER BY columns would pollute the benchmark)\nand over weakening to set-equality.\n\nMECHANISM: the relaxation engages ONLY after the strict sorted-positional\ncomparison fails, and accepts a divergence iff it is exactly an ambiguous\ntop-N boundary tie - deterministically, independent of run-to-run tie\nreordering:\n - the differing rows are the multiset difference each way (the swapped tie\n members); the rest are byte-identical (the deterministic remainder);\n - there must be an order-key column c that is monotonic across the\n reference result (the ORDER BY key is; a grouped non-key column is not),\n whose value on EVERY swapped row equals the reference's boundary value\n original[-1][c] (stable even when tied rows reorder), the boundary value\n occurs >=2 times (a genuine tie, not a unique last row), and is an\n extreme (min/max) of the column.\nSo a real value-misplacement bug outside the tie (wrong order-key value,\nwrong non-key value on a determined row, wrong row count, unique-last-row\nchange) puts a non-boundary value into the swap and still FAILS. The\ncross-surface gate passes tie_aware=True; TPC-Havoc keeps the default.\n\nAn EARLIER attempt inferred the key by splitting on all monotonic columns\njointly; DuckDB's non-deterministic tie order made a non-key column\n(SearchPhrase) coincidentally monotonic in ~1/3 of runs, so the gate was\nflaky. The multiset-difference + boundary-value formulation is order-stable\n(the count column always qualifies); verified 5 consecutive clean runs.\n\nQ18 (`... GROUP BY ... LIMIT 10` with NO ORDER BY over ~97k groups) is the\none genuinely order-less query - any 10 rows are valid and the two surfaces\npick different subsets (both verified subsets of the full grouped result).\nIt is the last-resort option (c): classified in clickbench's\nknown_divergences with justification, NOT masked (the tie-aware comparator\nhandles every ORDER BY ... LIMIT case; only this order-less LIMIT needs a\nbaseline entry).\n\nEVIDENCE: make clickbench-cross-surface-equivalence-report 27 -> 0\nunclassified (Q18's 1-2 classified cells aside), stable across 5 runs; ssb\n26/26, amplab 16/16, coffeeshop 22/22, joinorder_synthetic 26/26 all 0\ndivergent on the tie_aware path; TPC-Havoc SQL 220/220 and DataFrame 440/440\nunchanged (default strict path); a planted REAL ClickBench defect (Q1 count\n+1000) still flags Q1_expression+Q1_pandas; 80 tpchavoc+equivalence unit\ntests pass (6 new tie_aware cases lock accept/reject behavior). Files:\nbenchbox/core/tpchavoc/validation.py, benchbox/core/equivalence/cross_surface.py.\nOriginal triage below.\n\nPREREQUISITE for promoting ClickBench STAGED_GATES->GATES (w4). Surfaced and\ntriaged during w3 ClickBench burn-down (PR #848; detail in\n_project/analysis/clickbench-cross-surface-divergences.md).\n\nROOT CAUSE (verified by direct SQL-vs-DataFrame set/count comparison): the 27\nremaining ClickBench divergences are NOT logic bugs. The DataFrame computes\nCORRECT aggregates; the gate's comparator (ResultValidator.validate_results_exact\nin benchbox/core/tpchavoc/validation.py) sorts both sides then compares\nrow-by-row, so for `ORDER BY ... LIMIT N` queries whose ties span the\nLIMIT cutoff it reports a false divergence: SQL and the DataFrame each keep a\ndifferent (equally valid) subset of the tied rows. E.g. Q32/Q33 top-10 are all\ncount=1 (thousands of count-1 groups; LIMIT 10 keeps an arbitrary 10); Q16's\ncount distribution 4,4,4,4,4,4,3,3,3,3 is IDENTICAL on both surfaces, only the\ntied rows reorder/swap at the boundary.\n\nFIX (gate/validator-level, NOT 27 per-query ClickBench changes - pick one):\n (a) append a deterministic total-order tie-breaker (remaining columns / a\n stable key) to the ORDER BY on BOTH surfaces before comparison, or\n (b) make the comparator tie-aware: compare the MULTISET of rows sharing the\n boundary order-key value instead of position-by-position, or\n (c) classify genuinely tie-ambiguous queries as accepted/justified\n divergences (last resort - prefer a real total order).\nReuse, do not fork, ResultValidator; keep it able to catch genuine\nvalue-misplacement bugs (do not weaken to set-equality across all columns).\n\nVERIFY: after the fix, `make clickbench-cross-surface-equivalence-report` drops\nfrom 27 divergent cells to only the w9 loader-dtype residue (Q17/Q24), and a\nplanted real ClickBench defect still fails the gate (no masking).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the cross-surface comparator tie-aware for top-N queries (clears ClickBench's 27 tie-ambiguous cells)", "wid": "w8"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5", "w6"], "notes": "DONE. Made the production DataFrame loader apply the schema's declared column\nTYPES (and DuckDB-matching empty-field semantics) instead of per-file\ninference. Three loader-layer fixes, all additive, no per-query changes:\n\n 1. SQL-type -> Arrow-type map was incomplete. The CSV->Parquet conversion\n (the Polars/expression path, which reads read_parquet) only typed the\n handful of names in SchemaMapper.PYARROW_TYPE_MAP (VARCHAR/CHAR/INTEGER/\n DATE/...), so declared TEXT/STRING/BIGINT/NUMERIC/... columns fell\n through to PyArrow inference - an all-NULL TEXT column (joinorder\n movie_companies.note) inferred as Null, a numeric-looking TEXT column\n (movie_info_idx.info ratings) as float64. Added\n SchemaMapper.sql_type_to_pyarrow() covering every SQL type family\n (TEXT/STRING/CHAR -> string, INT family -> int64, DECIMAL/FLOAT family\n -> float64, DATE -> date32, TIMESTAMP/DATETIME -> timestamp[us]) and\n routed _extract_arrow_types through it. Cleared the 6 Polars\n \"expected String type\" cells (1a/1b/5a/8a/9a/10a) and the float64-vs-str\n cells (4a/12a).\n 2. Pandas reads CSV, not Parquet (its recommended_data_format is CSV), so\n the Parquet fix did not reach it. read_csv now (a) reads declared text\n columns as object (so numeric-looking text is not inferred float64) and\n (b) parses a declared DATE/TIMESTAMP column as temporal even when its\n NAME does not match the date-suffix heuristic (ClickBench\n ClientEventTime TIMESTAMP was read as a raw string vs DuckDB's datetime).\n Date detection is now TYPE-aware (extends the w6 numeric-exclusion).\n 3. Empty-string vs NULL (ClickBench Q17/Q24): the DataFrame surface emitted\n None where the SQL surface emits \"\". Root cause was the per-benchmark\n CSV null_marker, NOT dtype: DuckDB resolves nullstr via\n resolve_csv_dialect (ClickBench null_marker=None -> empty kept \"\";\n JoinOrder null_marker=\"\" -> empty -> NULL), while the DataFrame path\n always nulled empties (coincidentally matching JoinOrder, diverging from\n ClickBench). Threaded the SAME resolved null_marker into both surfaces:\n the Parquet conversion sets strings_can_be_null = (null_marker is not\n None); pandas restores \"\" on declared string columns when null_marker is\n None; pandas load_table now resolves null_marker via resolve_csv_dialect\n (not the file-extension heuristic) so JoinOrder still nulls empties.\n Bumped the DataFrame Parquet cache v3->v4 (conversion rules changed).\n\nEVIDENCE:\n - make joinorder-synthetic-cross-surface-equivalence-report: 26/26, 0\n divergent (was 16/26). Deterministic (seeded generator).\n - make clickbench-cross-surface-equivalence-report: Q24 fully cleared (both\n backends); Q17's empty-string/dtype residue cleared - Q17's remaining\n divergence is the tie-ambiguous top-N (ORDER BY COUNT(*) DESC LIMIT 10,\n different tied UserIDs per backend), which is w8's domain, not w9. Overall\n clickbench 27 -> 22 divergent; the 22 are exactly the w8 tie-ambiguous set.\n - Enforced gates unaffected: ssb 26/26, amplab 16/16, coffeeshop 22/22, all\n 0 divergent (their null_marker is \"\" or .tbl, or None with no compared\n empties). tpchavoc SQL gate 220/220 and DataFrame gate 440/440, 0\n divergent (the latter uses DuckDB->Arrow, unaffected).\n - 1863 tests/unit/core/dataframe + core/equivalence + platforms/dataframe\n pass. ruff clean; no new ty diagnostics. Real benchbox-run DataFrame load\n path re-verified (data loads via the production loader unchanged; the\n NOT_RUN CLI query-selection quirk pre-exists on develop).\nFiles: benchbox/core/dataframe/data_loader.py (sql_type_to_pyarrow,\n_get_null_markers, null_marker threading, cache v4),\nbenchbox/platforms/dataframe/pandas_df.py (string-col dtype, type-aware date\nparse, empty-string restore), benchbox/platforms/dataframe/pandas_family.py\n(resolve_csv_dialect for the benchmark-instance null_marker). Original plan\nbelow.\n\nPREREQUISITE for promoting joinorder_synthetic STAGED_GATES->GATES (w4) and for\nClickBench Q17/Q24 (empty-string-vs-None). DEFERRED by user decision\n(2026-06-21) to keep that session focused; it remains REQUIRED for w4 and is\ntracked here so it is not lost. Detail:\n_project/analysis/joinorder-synthetic-cross-surface-divergences.md and the\nQ17/Q24 exception in clickbench-cross-surface-divergences.md.\n\nROOT CAUSE: the production DataFrame loader applies column NAMES from the schema\n(get_benchmark_schema_columns, fixed in #846 to parse DDL-string columns) but\nthen TYPE-INFERS dtypes per file rather than applying the schema's declared\ncolumn TYPES. So columns whose content looks numeric, or is null-heavy, get the\nwrong dtype and the query's comparisons fail. joinorder_synthetic: 10 cells -\n\"expected String type\" (Polars) on 1a/1b/5a/8a/9a/10a; \"string vs numeric\" on\n4a/12a. ClickBench: SQL emits \"\" where the DataFrame emits None (Q17 col 1,\nQ24 col 14) - same null/dtype materialization issue. NOTE the divergence\nreproduces on the read_parquet path (the loader converts CSV->parquet via\nDataFrameDataLoader, then read_parquet), NOT read_csv - an earlier CSV-layer\nschema_overrides attempt was on the wrong layer and was reverted (#847); the\nfix must apply schema types on the parquet/materialization path the loader\nactually uses.\n\nFIX (loader-layer, not per-query): apply the schema column TYPES when loading\nthe DataFrame surface (types are available from\nbenchbox/core/dataframe/schema_utils.py column_sql_type), so a column declared\nVARCHAR/TEXT loads as string and INTEGER as int instead of relying on per-file\ndtype inference. Reuse the production loader/adapters (benchbox/platforms/\ndataframe/, benchbox/core/dataframe/data_loader.py); keep the change additive\nand re-verify the normal DataFrame benchmark run path (must_preserve).\n\nVERIFY: `make joinorder-synthetic-cross-surface-equivalence-report` reaches\n26/26 (from 16/26); ClickBench Q17/Q24 clear; dataframe/equivalence unit tests\nstay green.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply schema column TYPES (not just names) when loading the DataFrame surface (clears joinorder_synthetic's 10 dtype cells)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "copy-paste three visually similar headers and call them unified", "instead": "(not specified)", "why": "the current defect is independent implementations drifting."}, {"dont": "make the Results global header almost identical except for spacing or CTA behavior", "instead": "(not specified)", "why": "the user requirement is identical."}, {"dont": "remove Furo's theme controls or search while harmonizing docs/blog", "instead": "(not specified)", "why": "docs usability is outside the header defect."}, {"dont": "start the full dark/light retheme in this phase", "instead": "(not specified)", "why": "phase 1 needs a stable shell contract first."}, {"dont": "use the Results secondary nav to justify changing the global header", "instead": "(not specified)", "why": "the secondary nav is a separate product navigation layer."}], "approach": "Start by deciding the canonical global header from real rendered surfaces,\nthen make every surface consume or test-enforce that contract. Treat Results\nExplorer's secondary nav as a product-specific child nav, not as part of the\nglobal shell. Keep CSS token names compatible with the follow-up light/dark\ntheme work so phase 2 can swap values without rewriting header markup again.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-05-13T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-site-header-unification", "id": 347, "reason": "Owned by dependent TODO `benchbox-light-dark-theme-system`; this phase must avoid blocking it but should not implement it opportunistically.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Dependent TODO benchbox-light-dark-theme-system is Completed (2026-05-14); full light/dark theme switching already shipped.", "summary": "Full light/dark theme switching"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchbox-site-header-unification", "id": 348, "reason": "The request is header parity, not new navigation architecture. Keep the Results secondary nav unless a concrete usability defect requires a separate TODO.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Deferred only if a concrete usability defect surfaces requiring a new Explorer IA TODO; none identified.", "summary": "Redesign the Results Explorer information architecture"}], "deps": [], "description": "The public BenchBox site currently ships multiple independent header\nimplementations. The landing page uses `landing/index.html` and\n`landing/style.css`; docs/blog inject a Furo-adjacent header from\n`docs/_templates/page.html` and `docs/_static/custom.css`; Results Explorer\nrenders a separate Preact header in `results-explorer/src/components/Layout.tsx`.\nThose headers differ in link set, CTA presence, max width, fixed/sticky/static\npositioning, spacing, mobile behavior, active state, and theme assumptions.\n\nThis is a product-shell defect: the user should experience one BenchBox\nglobal header everywhere, including `/`, `/docs/`, `/blog/`, and `/results/`.\nResults Explorer may keep a secondary Results-specific navigation row below\nthe global header, but the global header itself must be identical in content,\nlayout, spacing, colors, active-link behavior, and responsive behavior.\n\nThis phase intentionally solves only the shared header contract. It should\nnot attempt the full light/dark retheme; it must, however, avoid introducing\nnew header styling that will block the follow-up theme phase.", "id": "benchbox-site-header-unification", "preserves": ["Docs/blog Furo search, sidebar, edit/view buttons, keyboard navigation, and built-in theme controls remain functional.", "Existing public URLs continue to work: `/`, `/docs/`, `/blog/`, `/results/`, GitHub, and installation/run-benchmark destination.", "Header changes do not alter Results Explorer data queries, result routes, compare flow, or DuckDB loading.", "Landing smooth-scroll/copy interactions and mobile navigation remain functional.", "One global BenchBox header contract applies to landing, docs/blog, and Results Explorer.", "Results Explorer keeps a clearly separate secondary Results navigation row unless a separate IA decision changes it."], "prior_art": [{"concept": "docs/_static/custom.css:10 \u2014 current docs nav CSS, including forced dark overrides", "decision": "reuse", "path": "docs/_static/custom.css:10"}, {"concept": "docs/_templates/page.html:14 \u2014 current Sphinx/Furo injected site navigation", "decision": "reuse", "path": "docs/_templates/page.html:14"}, {"concept": "landing/index.html:20 \u2014 current public landing header markup; candidate canonical global link set", "decision": "reuse", "path": "landing/index.html:20"}, {"concept": "landing/script.js:3 \u2014 current landing smooth-scroll and mobile menu behavior", "decision": "reuse", "path": "landing/script.js:3"}, {"concept": "landing/shared/ \u2014 preferred new home for shared static shell CSS/JS if this TODO creates reusable assets", "decision": "reuse", "path": "landing/shared/"}, {"concept": "landing/style.css:55 \u2014 current landing nav spacing, fixed positioning, backdrop, and responsive rules", "decision": "reuse", "path": "landing/style.css:55"}, {"concept": "results-explorer/e2e/routes/home.spec.ts:28 \u2014 existing mixed-theme assertion that phase 2 must supersede", "decision": "supersede", "path": "results-explorer/e2e/routes/home.spec.ts:28"}, {"concept": "results-explorer/src/components/Layout.tsx:18 \u2014 current Results Explorer global header and secondary nav", "decision": "reuse", "path": "results-explorer/src/components/Layout.tsx:18"}, {"concept": "results-explorer/src/components/__tests__/Layout.test.tsx:30 \u2014 current Results header/link/active-state unit coverage", "decision": "reuse", "path": "results-explorer/src/components/__tests__/Layout.test.tsx:30"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "docs/_static/"}, {"kind": "only_modify", "path_glob": "docs/_templates/"}, {"kind": "only_modify", "path_glob": "docs/conf.py"}, {"kind": "only_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "landing/shared/"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/Layout.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/Layout.test.tsx"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Unify BenchBox site header across landing, docs/blog, and Results Explorer", "verifications": [{"command": "cd results-explorer && npm test -- src/components/__tests__/Layout.test.tsx", "description": "Results Explorer layout tests pass after header refactor", "expected": "Layout tests pass and assert global-header parity plus Results subnav separation", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 1}, {"command": "rg -n 'benchbox-site-header|site-header|nav-logo|Run benchmark|Results' landing docs/_templates docs/_static results-explorer/src/components/Layout.tsx", "description": "Landing header source uses the shared header contract", "expected": "review output shows one shared/global header contract, not divergent local implementations", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- sphinx-build -b html docs docs/_build/html", "description": "Docs build still succeeds", "expected": "Sphinx build succeeds without template or static asset errors", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run build", "description": "Results Explorer builds", "expected": "Vite build succeeds", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 4}, {"command": "Run the header parity browser check added by this TODO for landing, docs/blog, and Results at desktop and mobile widths", "description": "Cross-surface visual parity check passes", "expected": "global header labels, spacing, active state, CTA, logo, and mobile behavior match across surfaces", "item_id": "benchbox-site-header-unification", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "benchbox-site-header-unification", "needs": [], "notes": "Inspect the current landing, docs/blog, and Results Explorer header\nimplementations and capture a compact comparison matrix covering:\n- link order and labels;\n- external URL casing and target behavior;\n- CTA presence and destination;\n- max width, height, padding, gap, typography, border, backdrop, and z-index;\n- fixed vs sticky vs static positioning;\n- active-link semantics for Home, Docs, Blog, Results, and GitHub;\n- mobile breakpoint behavior and keyboard/focus behavior.\n\nDecision gate: choose the canonical header contract before editing code.\nThe default should be the public landing header shape unless a specific\naccessibility or routing issue requires a documented adjustment. Record\nthe chosen contract in a short analysis note or in the implementation PR\ndescription; do not let each surface make local design choices.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture current header variants and define the exact global-header contract", "wid": "w0"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w0"], "notes": "Extract the canonical header markup and CSS into reusable site-shell\nassets that landing and docs/blog can consume without duplicating class\ndefinitions. Prefer a small shared CSS file and template/include strategy\nthat works with the existing static landing deployment and Sphinx/Furo\ndocs build.\n\nPreferred durable home: keep static shared shell assets under\n`landing/shared/` and copy/reference them from docs as part of the normal\ndocs static build, or document why that path is not viable before choosing\na different location. The implementation must leave a clear source of\ntruth so the next header change has one obvious file to edit.\n\nThe shared contract must include:\n- the exact global link set and order;\n- active-state attributes/classes that can be applied by page context;\n- mobile menu behavior if the canonical header needs it;\n- no hard dependency on the Results Explorer Vite build;\n- no duplicated magic numbers between landing and docs custom CSS.\n\nDecision gate: if true shared includes are impractical because of the\ndeployment pipeline, document the constraint and keep one generated source\nof truth or a test-enforced mirror. Do not manually fork the header again.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create a shared static header asset contract for non-React pages", "wid": "w1"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w1"], "notes": "Replace the landing page's local header markup/styles with the shared\nglobal-header contract while preserving landing-only section links only\nif the contract explicitly allows an in-page secondary section nav. If\nsection links remain, they must be visually distinct from the global\nheader and must not appear as global site navigation on other surfaces.\n\nPreserve existing copy buttons, smooth-scroll behavior, mobile menu\nbehavior, and first-viewport hero layout. Any JS moved from\n`landing/script.js` must remain static-file friendly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the shared header to the landing page", "wid": "w2"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w1"], "notes": "Replace the docs/blog custom nav in `docs/_templates/page.html` and\n`docs/_static/custom.css` with the shared global-header contract. Keep\nFuro's sidebars, search, edit/view buttons, keyboard navigation, and\nbuilt-in theme controls working.\n\nRemove or revise the existing docs CSS that forces the BenchBox nav to\nstay dark under `[data-theme=\\\"light\\\"]`, because phase 2 will make the\nheader theme-aware. For this phase, preserve the current visual result\nunless the chosen canonical contract says otherwise.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the shared header to docs and blog without breaking Furo controls", "wid": "w3"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w1"], "notes": "Refactor `results-explorer/src/components/Layout.tsx` so the global\nheader matches the canonical shared header. Keep the Results Explorer\nsecondary nav (`Leaderboards`, `Benchmarks`, `Platforms`, `Compare`,\n`Query`) below the global header as a separate product nav with its own\naria-label and active-state tests.\n\nThe Results global header must not invent different spacing, link order,\nCTA text, max width, border treatment, or mobile behavior. It may use a\nPreact component wrapper only to integrate routing and active state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the shared global header to Results Explorer and preserve the Results subnav", "wid": "w4"}, {"evidence": null, "item_id": "benchbox-site-header-unification", "needs": ["w2", "w3", "w4"], "notes": "Add verification that compares the global header across:\n- landing `/`;\n- docs `/docs/`;\n- blog `/blog/` if locally buildable;\n- Results `/results/`.\n\nAt minimum, assert identical visible global nav labels/order, CTA label\nand URL, logo URL, active-link semantics, and mobile behavior. Add\ndesktop and mobile visual screenshots or DOM snapshots sufficient to\ncatch future drift. Results-specific secondary nav must be asserted as\nseparate from the global header, not as a reason for global-header drift.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cross-surface header parity tests and screenshots", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "create a third benchmark base class while cleaning up two existing bases", "instead": "(not specified)", "why": "remove, alias, or justify the secondary base."}, {"dont": "interpret duplicate wrapper reachability as accidental", "instead": "classify first.", "why": "tests assert facade behavior"}, {"dont": "split core by moving files mechanically", "instead": "(not specified)", "why": "dependency direction and public API status must drive the boundary."}], "approach": "Start with API classification and small ambiguity removals. Treat a\nbenchmark-family plugin seam as a design gate before extraction, and use tests\nto keep wrappers, registries, and loaders aligned.\n", "blocked_reason": null, "category": "Core Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-api-and-core-boundary-cleanup", "id": 349, "reason": "Large structural refactor should follow the public API decision and plugin-seam ADR.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Split all of benchbox/core into engine and benchmark packages"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-api-and-core-boundary-cleanup", "id": 350, "reason": "Wrappers are tested as public facades unless explicitly deprecated.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly rejects removal: wrapper facades are tested public surfaces unless explicitly deprecated.", "summary": "Remove all wrapper facades"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Benchmark APIs are split across tested top-level wrappers, a public\nbenchbox.base.BaseBenchmark, a small remaining core.base_benchmark class,\nlazy registries, loaders, and benchmark-family code inside broad core/base\nmixins. The wrappers are not accidental; tests assert facade behavior. The\ncleanup target is the remaining ambiguous base/loader split and the lack of a\nbenchmark-family extension boundary, not a wrapper purge.\n\nThis item classifies benchmark API surfaces, retires or justifies the second\nbase class, and prepares a benchmark-family plugin seam without a broad\nrewrite.", "id": "benchmark-api-and-core-boundary-cleanup", "preserves": ["Do not break top-level wrapper facade tests unless an explicit deprecation path lands in the same PR.", "Do not introduce platform adapter imports into core engine modules.", "Do not move benchmark-family code in a way that changes result bundle shape."], "prior_art": [{"concept": "docs/development/run-lifecycle-map.md: existing lifecycle documentation -- align any plugin seam with this flow.", "decision": "reuse", "path": "docs/development/run-lifecycle-map.md"}, {"concept": "docs/reference/backward-compatibility.md: existing base-class lifecycle entries -- update rather than invent separate lifecycle docs.", "decision": "reuse", "path": "docs/reference/backward-compatibility.md"}, {"concept": "tests/unit/test_wrapper_facades_fast.py: existing wrapper contract tests -- preserve as public API evidence.", "decision": "reuse", "path": "tests/unit/test_wrapper_facades_fast.py"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/"}, {"kind": "only_modify", "path_glob": "docs/design/"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "tests/docs/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/test_wrapper_facades_fast.py"}], "state": "done", "title": "Architecture hardening: classify benchmark APIs and reduce core boundary drift", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_wrapper_facades_fast.py tests -k 'benchmark_loader or base_benchmark' --tb=short", "description": "Run wrapper facade and benchmark loader tests.", "expected": "Wrapper and loader tests pass.", "item_id": "benchmark-api-and-core-boundary-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests -k 'import_boundary or core_boundary' --tb=short", "description": "Run import-boundary check added by this item.", "expected": "Core/platform dependency boundary tests pass or known exemptions are documented.", "item_id": "benchmark-api-and-core-boundary-cleanup", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": [], "notes": "Completed evidence at checked SHA\n1d454632ba73911bc4ff0cf0a3fb8ec22227a7a8:\n - benchbox/base.py remains the beta-public BaseBenchmark surface;\n run_with_platform remains the beta-public adapter execution hook.\n - benchbox/core/base_benchmark.py remains a distinct deprecated\n internal compatibility base, not an alias.\n - Production imports of the deprecated core base are limited to\n benchbox/core/datavault/benchmark.py and\n benchbox/core/tpcds_obt/benchmark.py.\n - tests/unit/test_wrapper_facades_fast.py asserts wrapper behavior;\n wrapper reachability is not accidental.\n - benchbox/__init__.py exposes 21 top-level benchmark facades: 7 eager\n imports and 14 lazy _BENCHMARK_REGISTRY entries.\n - benchbox/core/benchmark_loader.py resolves 23 core benchmark families\n from registry-backed CORE_BENCHMARK_CLASS_NAMES.\n - docs/reference/backward-compatibility.md now records the remaining\n core-base migration target explicitly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate benchmark API surfaces, wrapper tests, registries, and core boundary issues", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w0"], "notes": "Decision recorded in docs/reference/public-contracts.md:\n - CLI, top-level benchmark wrappers, benchbox.base.BaseBenchmark,\n BaseBenchmark.run_with_platform, registry metadata, and MCP remain\n beta-public surfaces.\n - benchbox.core.benchmark_loader is internal.\n - benchbox.core.base_benchmark.BaseBenchmark is deprecated internal\n compatibility.\n\nNo wrapper facade was collapsed or deprecated.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decision gate: define wrapper-vs-CLI-vs-loader public API contract", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w1"], "notes": "Justified as a documented deprecated internal compatibility base. It is\nretained only for datavault and tpcds_obt until those implementations\nmigrate to benchbox.base.BaseBenchmark. Runtime modules expose surface\nmarkers and docs/reference/backward-compatibility.md carries the target.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire, alias, or justify benchbox/core/base_benchmark.py", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w1"], "notes": "Added tests/unit/core/test_benchmark_api_contract.py to lock:\n - 23 registry metadata entries.\n - 23 loader-resolved core families.\n - 22 public discovery entries.\n - 21 top-level Python benchmark facades.\n - 14 lazy facades and 7 eager facades.\n - 2 core-only benchmark IDs: ai_primitives and joinorder_synthetic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile benchmark registry, wrapper, and loader counts", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w1"], "notes": "Added docs/design/future-state/benchmark-family-plugin-seam/README.md and\nlinked it from the future-state index. The design keeps wrappers and\nresult bundles stable, defines family-owned versus shared-runtime\nresponsibilities, and requires one pilot family before any broad package\nextraction.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the benchmark-family plugin seam as an ADR before extraction", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-api-and-core-boundary-cleanup", "needs": ["w4"], "notes": "Added a narrow AST import-boundary check over benchbox/base.py,\nbenchbox/core/base_benchmark.py, benchbox/core/benchmark_loader.py,\nbenchbox/core/__init__.py, and benchbox/core/tuning/. The check forbids\nconcrete benchbox.platforms imports in those benchmark API and tuning\nboundary files while avoiding unrelated existing core/platform couplings.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an import-boundary check for core modules that depend on platform adapters", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "enforce SimpleBenchmarkMixin requirements so broadly that intermediate mixins fail before concrete benchmark classes are defined.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "leave two editable copies of result/validation helpers without a boundary test and migration owner.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "migrate datavault or tpcds_obt without preserving their current output path conventions.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "refactor unrelated benchmark query logic while changing base contracts.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Start by deciding whether migration is feasible in one bounded PR. If it is,\nmigrate the two deprecated-base consumers and delete the duplicated base. If\nit is not, extract shared result/validation behavior and add a hard boundary\ntest so the deprecated surface cannot attract new benchmarks. For the simple\nmixin, reuse the existing runtime contract style instead of inventing a broad\nframework.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-22T00:00:00Z", "deferrals": [], "deps": ["canonical-benchmark-id-resolution"], "description": "Independent verification at checked SHA e60986923 confirmed the deprecated\ninternal base remains load-bearing for exactly two loader benchmarks:\n\n datavault -> benchbox.core.datavault.benchmark.DataVaultBenchmark\n tpcds_obt -> benchbox.core.tpcds_obt.benchmark.TPCDSOBTBenchmark\n\n`benchbox/core/base_benchmark.py` is labeled deprecated and retained for\ninternal compatibility, but it duplicates result creation and validation\nhelpers from `benchbox/base.py`. Verified duplicated method names include\n`create_enhanced_benchmark_result`, `create_minimal_benchmark_result`,\n`_resolve_output_dir`, `validate_preflight`, `validate_manifest`, and\n`validate_loaded_data`. Their signatures are currently similar, but they are\ndifferent method objects, and `_attach_performance_snapshot` exists only on\nthe public base.\n\nIndependent verification also confirmed `SimpleBenchmarkMixin` is a\ndocstring-only duck-typed contract. A dummy class inheriting\n`SimpleBenchmarkMixin` defines successfully and fails only at runtime with\n`AttributeError: 'BadSimple' object has no attribute 'query_manager'`. This\ncontrasts with `CursorValidationQueryExecutionMixin`, which uses\n`__init_subclass__` enforcement for required methods.", "id": "benchmark-base-and-mixin-contract-consolidation", "preserves": ["Datavault and TPC-DS OBT remain importable loader benchmarks.", "Existing AMPLab, ClickBench, and SSB simple benchmark behavior remains unchanged.", "Result schema fields and validation helper return types remain backward compatible.", "TPC-DS OBT keeps separate source-data and OBT output-directory behavior."], "prior_art": [{"concept": "benchbox/base.py: public BaseBenchmark result/validation helpers - prefer as the future implementation surface.", "decision": "reuse", "path": "benchbox/base.py"}, {"concept": "benchbox/core/base_benchmark.py: deprecated compatibility base - migrate or shrink to a thin shared layer.", "decision": "reuse", "path": "benchbox/core/base_benchmark.py"}, {"concept": "benchbox/core/benchmark_mixins.py: CursorValidationQueryExecutionMixin.__init_subclass__ - reuse enforcement style where suitable.", "decision": "reuse", "path": "benchbox/core/benchmark_mixins.py"}, {"concept": "benchbox/core/simple_benchmark_mixin.py: documented SimpleBenchmarkMixin requirements - convert from prose-only to enforced contract.", "decision": "reuse", "path": "benchbox/core/simple_benchmark_mixin.py"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: import-boundary tests - natural home for deprecated-base boundary assertions.", "decision": "reuse", "path": "tests/unit/core/test_benchmark_api_contract.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_result_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/simple_benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}], "state": "done", "title": "Consolidate benchmark base and mixin contracts", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/core/test_loader_registry_parity.py tests/test_base.py -q", "description": "Base contract and loader parity tests cover deprecated-base migration or boundary.", "expected": "All selected contract tests pass.", "item_id": "benchmark-base-and-mixin-contract-consolidation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core -k 'simple or benchmark_api_contract or loader_registry_parity' -q", "description": "Simple benchmark consumers still pass their focused tests.", "expected": "Focused unit tests pass without runtime AttributeError regressions.", "item_id": "benchmark-base-and-mixin-contract-consolidation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite remains green after base/mixin changes.", "expected": "Fast tests pass.", "item_id": "benchmark-base-and-mixin-contract-consolidation", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": [], "notes": "Re-run a read-only check that iterates loader classes and prints any\nsubclass of `benchbox.core.base_benchmark.BaseBenchmark`. Also define a\nthrowaway `SimpleBenchmarkMixin` consumer with missing attributes and\nconfirm whether failure occurs at class definition or runtime. Keep raw\noutput outside git.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate deprecated-base consumers and simple-mixin enforcement gap", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w0"], "notes": "Evaluate two paths:\n 1. Migrate `datavault` and `tpcds_obt` to `benchbox.base.BaseBenchmark`\n and delete or drastically shrink `benchbox/core/base_benchmark.py`.\n 2. Extract shared result/validation methods into one mixin/helper used\n by both bases while migration remains pending.\nPrefer full migration if constructor/output-dir differences are tractable\ninside one PR. If not, use shared extraction with a dated migration TODO.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose between full migration and shared helper extraction for deprecated-base result/validation logic", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w1"], "notes": "Ensure `create_enhanced_benchmark_result`, minimal result creation,\npreflight validation, manifest validation, and loaded-data validation use\none implementation path. Preserve `datavault` and `tpcds_obt` default\noutput-directory semantics, including source-data paths for TPC-DS OBT.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate or de-duplicate result creation and validation helpers", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w2"], "notes": "If full migration is completed, assert no loader benchmark subclasses the\ndeprecated base and consider deleting the file. If shared extraction is\nchosen, assert only the explicitly approved migration stragglers use it\nand link that approval to a TODO or documented deadline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an import-boundary test for deprecated-base usage", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w0"], "notes": "Apply the existing `CursorValidationQueryExecutionMixin.__init_subclass__`\npattern if practical. If attribute enforcement is too noisy for mixin\ncomposition, define a `Protocol` and add focused tests for AMPLab,\nClickBench, and SSB. The enforcement should catch missing\n`query_manager`, `_table_load_order`, `_get_table_schema`, and query\nexecution methods before `_load_data` or `run_benchmark` fails deep at\nruntime.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enforce `SimpleBenchmarkMixin` structural requirements at class definition or via Protocol tests", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-base-and-mixin-contract-consolidation", "needs": ["w2", "w3", "w4"], "notes": "Cover `datavault` and `tpcds_obt` construction, output directory\ndefaults, validation fallback identifiers, and result creation if they\nmigrate. Add a negative test for an invalid `SimpleBenchmarkMixin`\nconsumer and positive coverage for the existing consumers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for migrated base behavior and mixin contract failures", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "catch every constructor TypeError to detect missing `parallel`", "instead": "(not specified)", "why": "internal constructor bugs must surface directly."}, {"dont": "keep two same-named functions with different return surfaces without an explicit compatibility story.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "remove a public alias without checking docs/tests for import expectations.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "switch CLI preview code to wrapper classes by accident", "instead": "(not specified)", "why": "sample query preview currently needs core implementation internals."}], "approach": "Rename or parameterize for intent first, then remove the broad constructor\nfallback. Keep compatibility aliases only where external callers may already\nimport the old name. The key design rule is that function names and exception\nhandling should describe the contract instead of relying on caller memory.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-22T00:00:00Z", "deferrals": [], "deps": ["canonical-benchmark-id-resolution"], "description": "Independent verification at checked SHA e60986923 confirmed that two functions\nnamed `get_benchmark_class` intentionally return different surfaces:\n\n benchbox.core.benchmark_registry.get_benchmark_class(\"tpch\")\n -> benchbox.tpch.TPCH\n benchbox.core.benchmark_loader.get_benchmark_class(\"tpch\")\n -> benchbox.core.tpch.benchmark.TPCHBenchmark\n\nThe same split was confirmed for `vector_search` and `datavault`. Both\nreturns are callable benchmark classes, so importing the wrong function can\nsilently select the public wrapper surface where a core implementation was\nexpected, or vice versa. Existing tests already alias these functions to keep\nthem straight, which is evidence that the ambiguity is real.\n\nIndependent verification also confirmed the constructor fallback concern in\n`benchbox/core/benchmark_loader.py`: a class whose constructor accepts\n`parallel` but raises `TypeError` internally is retried without `parallel`,\nreturning successfully in the mocked harness after constructor calls\n`[1, None]`. This means \"constructor does not accept `parallel`\" and\n\"constructor accepted `parallel` but had an internal TypeError\" are currently\nindistinguishable.", "id": "benchmark-class-loading-surface-contract", "preserves": ["Benchmarks that genuinely lack a `parallel` constructor argument still instantiate successfully.", "Core loader lookup remains available for internal runtime construction.", "Import errors and unknown benchmark IDs continue to include available benchmark IDs.", "Public wrapper lookup remains available for documented public surfaces."], "prior_art": [{"concept": "benchbox/cli/benchmarks.py: _show_sample_queries imports loader class lookup - preserve the core implementation requirement.", "decision": "reuse", "path": "benchbox/cli/benchmarks.py"}, {"concept": "benchbox/core/benchmark_loader.py: get_benchmark_class and get_benchmark_instance - currently resolves core classes and retries broad TypeError.", "decision": "reuse", "path": "benchbox/core/benchmark_loader.py"}, {"concept": "benchbox/core/benchmark_registry.py: get_benchmark_class - currently resolves the top-level wrapper surface.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "tests/unit/core/test_loader_registry_parity.py: existing aliases for registry_get_benchmark_class and loader_get_benchmark_class - use as the initial contract location.", "decision": "reuse", "path": "tests/unit/core/test_loader_registry_parity.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/benchmarks.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/convert.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/execution.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/discovery.py"}, {"kind": "only_modify", "path_glob": "tests/uat/test_no_cli_surface_drift.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_manager_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_selection.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_orchestrator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_convert_command.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_loader_registry_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_benchmark_stdout_suppression.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_tool_handlers.py"}], "state": "done", "title": "Clarify public versus core benchmark class loading contracts", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_loader_registry_parity.py tests/unit/core/test_benchmark_api_contract.py -q", "description": "Class lookup parity and API contract tests pass.", "expected": "All selected class lookup tests pass.", "item_id": "benchmark-class-loading-surface-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_loader_registry_parity.py -q", "description": "Benchmark loader constructor fallback tests pass.", "expected": "Constructor signature/failure mode tests pass.", "item_id": "benchmark-class-loading-surface-contract", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite remains green after loader changes.", "expected": "Fast tests pass.", "item_id": "benchmark-class-loading-surface-contract", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": [], "notes": "Run a compact read-only check for at least `tpch`, `vector_search`, and\n`datavault`, printing registry-returned class and loader-returned class.\nReproduce the constructor fallback with a mock benchmark class that raises\n`TypeError` only when `parallel` is provided. Keep raw output in `/tmp`\nand summarize the behavior in the PR.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate the public/core class surface split and constructor fallback behavior", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w0"], "notes": "Evaluate two options:\n 1. Rename functions to `get_public_benchmark_class` and\n `get_core_benchmark_class`, preserving compatibility aliases with\n deprecation notes.\n 2. Collapse lookup into one registry-owned function with\n `surface=\\\"public\\\"|\\\"core\\\"`.\nPrefer the option with the smallest public API churn and clearest import\nsites. Document which call sites intentionally need wrappers versus core\nimplementations.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose an explicit naming or API strategy for class lookup", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w1"], "notes": "Update internal imports in CLI, MCP, tests, and loader code so the call\nsite name communicates public/core intent. Keep backward-compatible\naliases only if they are part of the public or semi-public surface, and\nadd comments where alias retention is intentional.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor call sites to use intent-revealing class lookup", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w1"], "notes": "Use `inspect.signature()` or a benchmark capability marker to determine\nwhether `parallel` is accepted before instantiation. Ensure a `TypeError`\nraised from inside `__init__` is not swallowed and retried as if it were a\nsignature mismatch. Keep compatibility for benchmarks that genuinely do\nnot accept `parallel`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace broad constructor `TypeError` fallback with signature-aware or declared capability handling", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-class-loading-surface-contract", "needs": ["w2", "w3"], "notes": "Add or extend tests so public lookup returns wrappers, core lookup returns\ncore classes, unknown benchmark IDs still fail clearly, and a constructor\nthat raises an internal `TypeError` is surfaced without a misleading\nfallback attempt.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests that distinguish class surface and constructor failure modes", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "duplicate the cross-surface gate here", "instead": "dispatch to it for dual-surface benchmarks.", "why": "it has its own item"}, {"dont": "land the coverage check as a hard blocker first", "instead": "advisory until proven.", "why": "a noisy gate erodes trust"}, {"dont": "mandate full expected-results answer keys everywhere", "instead": "prefer live oracles.", "why": "the maintenance cost is why coverage is thin today"}, {"dont": "verify a benchmark against the surface under test", "instead": "use an independent reference (other surface, other engine, or curated key).", "why": "that is circular"}], "approach": "Inventory first (w0): a single, ideally generated, coverage matrix is the\nartifact that makes the gap legible and dispatchable. Then route dual-surface\ngaps to the cross-surface gate (w1) and solve single-surface gaps with the\ncheapest credible per-benchmark oracle (w2), reusing existing harnesses\n(cross-surface, cross-dialect, expected-results registry) rather than new\nmachinery. Close with a policy + advisory check (w3) so coverage cannot\nsilently erode.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-correctness-oracle-coverage-map", "id": 11, "reason": "Expensive and brittle to maintain; live oracles (cross-surface / differential) cover most of the value at a fraction of the upkeep. Only curate expected-results where no live oracle is feasible.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason explicitly rejects full backfill favoring live oracles; a scoped-out sub-note of the still-open parent oracle-coverage-map item. (tracked by benchmark-correctness-oracle-coverage-map)", "summary": "Backfill full expected-results answer keys for every benchmark"}], "deps": [], "description": "The TPC-Havoc equivalence work established that a benchmark can have a real\ncorrectness oracle without a hand-maintained answer key (variant-vs-canonical\nand SQL-vs-DataFrame, compared live on a bounded DuckDB cell). Applying that\nlens across the suite exposes how little of it is actually guarded:\n\n- Stored expected results exist for ONLY tpch and tpcds\n (benchbox/core/expected_results/tpch_results.py, tpcds_results.py).\n- Live equivalence gates exist for ONLY tpchavoc (SQL variant + DataFrame),\n plus cross-engine sampling for tpchavoc (PRs #803/#806).\n- Every other benchmark - ssb, clickbench, amplab, coffeeshop, datavault,\n flightdata, joinorder, joinorder_synthetic, nyctaxi, tsbs_devops, tpcds_obt,\n tpch_skew, and the primitive suites (read/write/transaction/metadata/ai),\n tpcdi - executes \"green\" with NO automated check that any result is correct.\n\nTwo complementary mechanisms cover most of this gap. The cross-surface\nSQL<->DataFrame gate (owned by benchmark-cross-surface-equivalence-gate)\nhandles dual-surface benchmarks. THIS item owns (a) the coverage map itself -\na single authoritative matrix of which oracle, if any, guards each shipped\nbenchmark - and (b) the benchmarks that cross-surface cannot reach: single-\nsurface benchmarks (one of SQL or DataFrame only) and benchmarks whose two\nsurfaces are not result-comparable. Those need a different oracle: a\ndifferential check against a second engine on the same SQL, a small curated\nexpected-results subset, or a documented structural invariant - chosen per\nbenchmark from evidence.\n\nThe end state is a policy: every shipped benchmark declares its correctness\noracle, and a benchmark with \"none\" is visible (and ideally flagged), so the\nsuite cannot quietly grow more unverified surfaces.\n", "id": "benchmark-correctness-oracle-coverage-map", "preserves": ["Existing oracles (tpch/tpcds expected results, tpchavoc gates) are unchanged and remain authoritative for their benchmarks.", "No new hard CI gate lands noisy/flaky; advisory-first, promote to blocking only once stable.", "benchbox run default behavior is unchanged for every benchmark; coverage work is additive."], "prior_art": [{"concept": "_project/TODO/main/planning/benchmark-cross-surface-equivalence-gate.yaml - the dual-surface mechanism this map dispatches to; do not duplicate it.", "decision": "reuse", "path": "_project/TODO/main/planning/benchmark-cross-surface-equivalence-gate.yaml"}, {"concept": "_project/TODO/main/planning/testing-approach-correctness-gates.yaml - the existing real-result gate / test-quality work; this map is complementary (coverage inventory + ungated-benchmark campaign), not a re-do of it.", "decision": "reuse", "path": "_project/TODO/main/planning/testing-approach-correctness-gates.yaml"}, {"concept": "_project/TODO/main/planning/tpchavoc-cross-dialect-equivalence-sampling.yaml and tpchavoc-third-engine-equivalence-sampling.yaml - the cross-engine/differential exemplars for the single-surface fallback.", "decision": "reuse", "path": "_project/TODO/main/planning/tpchavoc-cross-dialect-equivalence-sampling.yaml"}, {"concept": "benchbox/core/benchmark_registry.py - the authoritative list of shipped benchmarks to enumerate the map against.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "benchbox/core/expected_results/registry.py - register_benchmark_provider / list of registered providers; the place a coverage check would read 'has expected results'.", "decision": "reuse", "path": "benchbox/core/expected_results/registry.py"}, {"concept": "benchbox/core/tpchavoc/equivalence.py and dataframe_equivalence.py - the live-oracle exemplars (variant + cross-surface).", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/"}, {"kind": "only_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "active", "title": "Map correctness-oracle coverage across all benchmarks and close the unguarded ones", "verifications": [{"command": "run the coverage map generator; diff against the registry", "description": "Coverage matrix enumerates every registered benchmark with its current oracle", "expected": "every shipped benchmark appears with an oracle classification (including NONE)", "item_id": "benchmark-correctness-oracle-coverage-map", "last_result": null, "last_run": null, "seq": 1}, {"command": "pick one w2 benchmark, plant a wrong result, run its new oracle", "description": "A single-surface benchmark gains an independent oracle that catches a planted defect", "expected": "the oracle flags the defect", "item_id": "benchmark-correctness-oracle-coverage-map", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": [], "notes": "Implemented `_project/scripts/generate_oracle_coverage_map.py` (per the\ngating open_question default: GENERATED from live sources, not a curated\ndoc). It reads the benchmark registry (surfaces via `num_queries` /\n`supports_dataframe`), the expected-results provider registry, the\ncross-surface `GATES` registry, and TPC-Havoc gate-module presence, and\nemits checked-in artifacts `_project/analysis/oracle-coverage-map.{md,json}`.\n\nCurrent state (23 benchmarks): 4 GUARDED \u2014 tpch/tpcds (expected-results),\ntpchavoc (variant-equivalence + cross-surface-variant), ssb (cross-surface)\n\u2014 and 19 UNGUARDED. Of the 19: 17 are dual-surface (cross-surface gate\ncandidates, w1) and 2 are single-surface SQL-only (ai_primitives,\nvector_search) needing a fallback oracle (w2).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build the authoritative correctness-oracle coverage matrix for every shipped benchmark", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": ["w0"], "notes": "For benchmarks the matrix shows as dual-surface with oracle=NONE, the fix is\nbenchmark-cross-surface-equivalence-gate. Cross-reference rather than\nre-implement; this item only tracks that they get covered there.\n\nDispatch list from the w0 map (17 dual-surface UNGUARDED): amplab,\nclickbench, coffeeshop, datavault, flightdata, h2odb, joinorder,\njoinorder_synthetic, metadata_primitives, nyctaxi, read_primitives, tpcdi,\ntpcds_obt, tpch_skew, transaction_primitives, tsbs_devops, write_primitives.\nCAVEAT (see map): write/DML/nondeterministic ones (write_primitives,\ntransaction_primitives, metadata_primitives, tpcdi) may not be\nresult-comparable across surfaces and likely belong in w2 (invariant\noracle) rather than the cross-surface gate.\n\nSTATUS SYNC (2026-06-21) from benchmark-cross-surface-equivalence-gate: ssb,\namplab, coffeeshop are GATED, and clickbench + joinorder_synthetic are now\nPROMOTED to enforced GATES (campaign w4) - their prerequisites landed (w9\nloader dtype/empty-string semantics, w8 tie-aware top-N comparator), both run\nas blocking pr.yml steps with medium-marker fast-lane tests, and the oracle\ncoverage map (regenerated here) shows both as cross-surface \"guarded\".\n\nSTATUS SYNC (2026-06-24): h2odb is now an enforced cross-surface GATE\n(single trips table, ids Q1..Q10 1:1; blocking pr.yml step\n`make h2odb-cross-surface-equivalence-report` + medium-marker fast-lane test;\ngated on a bounded SF=0.01 ~100k-row cell because its generator base is the\n10M-row small tier). Wiring it caught and fixed a REAL DataFrame bug: the\nPolars Q9 percentile defaulted to \"nearest\" interpolation (disagreeing with\nboth pandas and SQL PERCENTILE_CONT) - fixed to linear. Q9's only residue is a\nclassified DECIMAL(8,2)-scale percentile difference (DuckDB returns the\npercentile at the column's 2-decimal scale; the DataFrame computes it over\nfloat64). So 6 dual-surface benchmarks are guarded. Remaining dual-surface\ndispatch targets (one focused gate PR each, several need id normalization or a\nbounded data-source check first): datavault, flightdata, joinorder (SF=1-only -\nuse joinorder_synthetic), nyctaxi, read_primitives, tpcds_obt, tpch_skew,\ntsbs_devops. This item only tracks coverage; the gates themselves are owned there.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Dispatch dual-surface, no-oracle benchmarks to the cross-surface gate", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": ["w0"], "notes": "For benchmarks with only one comparable surface (or non-comparable\nsurfaces), pick the cheapest credible oracle per benchmark: differential\nvs a second engine on the same SQL (reuse the cross-dialect harness), a\nsmall curated expected-results subset via the existing registry, or a\ndocumented structural invariant where results are nondeterministic.\nPrioritize the canonical/most-used benchmarks. Verify each against an\nindependent reference, never against the surface under test alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Choose and apply a fallback oracle for single-surface, no-oracle benchmarks", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-correctness-oracle-coverage-map", "needs": ["w1", "w2"], "notes": "Delivered ahead of w1/w2 (the visibility guard is independent of building\nthe oracles). `make oracle-coverage-map-check` (and the generator's\n`--check`) fail when the checked-in map is stale. CI surfacing on develop\nPRs is via TWO paths: the fast unit test\n`tests/unit/test_oracle_coverage_map.py::test_checked_in_artifacts_are_current`\nruns in the pr.yml `code-test` job (`-m fast`), and an explicit\n`make oracle-coverage-map-check` step sits in the pr.yml `code-lint` job\nnext to `compat-docs-check`. It is also in the `ci-lint` aggregate\n(post-merge / local `make pr-preflight` parity). Mechanism: adding a\nbenchmark changes the generated map -> the drift check fails -> the author\nmust regenerate and commit, which surfaces the new benchmark's UNGUARDED\nclassification in review rather than letting it slip in silently. The map's\nUNGUARDED list is the always-current \"no oracle\" inventory. Advisory in the\nsense that UNGUARDED is allowed-and-visible; only staleness is blocked.\n\nw1/w2 (actually building oracles for the 19 UNGUARDED benchmarks) remain the\nopen campaign; the w0 map makes them dispatchable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make 'no oracle' visible and prevent silent regressions in coverage", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "execute DataFrame queries OR load their data via a bespoke path", "instead": "use the DataFrame runner/mixin for query dispatch AND the production loader/adapters for data loading (w5).", "why": "the gate must reflect the real product run path"}, {"dont": "gate a full platform matrix", "instead": "one bounded small-SF DuckDB cell per benchmark.", "why": "routine PRs must stay cheap"}, {"dont": "silence a real divergence by adding it to the baseline", "instead": "fix the wrong surface.", "why": "the baseline is for intentional differences only"}, {"dont": "write a new result comparator or data builder", "instead": "generalize and reuse it.", "why": "the TPC-Havoc harness already does this"}], "approach": "Generalize the proven TPC-Havoc DataFrame gate rather than inventing a new\ncomparator: extract a shared harness (w0), inventory applicability (w1), stand\nup a production-faithful DataFrame-loading substrate (w5) and fix the loader\nbugs it surfaces (w6), re-base ssb onto it (w7), then land report-mode across\nthe ungated dual-surface benchmarks (w2), burn down (w3), and flip bounded CI\ngates with fast-lane companions (w4). SQL is the live reference for its own\nDataFrame surface; no hand-maintained answer keys; the DataFrame surface runs\non its real production data path so divergences are real, not dtype artifacts.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-cross-surface-equivalence-gate", "id": 6, "reason": "Cross-engine sampling is its own discipline (see tpchavoc-cross-dialect-equivalence-sampling); keep these gates DuckDB-bounded first.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Extending equivalence gating beyond DuckDB matches the scope of the open cross-surface gate breadth rollout item. (tracked by cross-surface-gate-rollout-followups)", "summary": "Cross-surface equivalence on a second engine"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-cross-surface-equivalence-gate", "id": 7, "reason": "Cross-surface cannot apply; their oracle gap is owned by benchmark-correctness-oracle-coverage-map.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason states this gap is owned by benchmark-correctness-oracle-coverage-map, which is open and tracks it.", "summary": "Benchmarks that ship only one surface"}], "deps": [], "description": "TPC-Havoc now ships two result-equivalence gates that proved a discipline:\nexecute a candidate query and a trusted reference over the SAME bounded\nsmall-SF DuckDB dataset and compare results live (order-insensitive, float-\ntolerant), with no hand-maintained expected answers. The SQL gate\n(benchbox/core/tpchavoc/equivalence.py) compares each SQL variant to canonical\nTPC-H; the DataFrame gate (benchbox/core/tpchavoc/dataframe_equivalence.py)\ncompares each DataFrame variant to canonical TPC-H SQL. Together they found and\nfixed ~42 silent correctness defects.\n\nThat discipline is currently TPC-Havoc-only, by construction. But ~14\nbenchmarks ship a DataFrame query surface (amplab, clickbench, coffeeshop,\ndatavault, flightdata, joinorder, nyctaxi, read_primitives, ssb, tpcds,\ntpcds_obt, tpch, tpch_skew, tsbs_devops) in ADDITION to their SQL surface, and\nnothing checks that the two surfaces agree. Worse, stored expected results\nexist for ONLY tpch and tpcds (benchbox/core/expected_results/tpch_results.py,\ntpcds_results.py) - so for every other dual-surface benchmark there is NO\ncorrectness oracle of any kind: SQL and DataFrame implementations both execute\n\"green\" while their results are unverified and may silently disagree.\n\nCross-surface equivalence is the cheap, maintenance-free oracle for exactly\nthis gap: SQL and DataFrame are independent implementations of the same logical\nquery, so a divergence proves at least one is wrong - without needing a\nhand-curated answer key. This work generalizes the TPC-Havoc DataFrame-gate\nharness into a shared, benchmark-agnostic helper, then rolls out a bounded\nSQL<->DataFrame equivalence gate to the dual-surface benchmarks, SQL as the\nreference for its own DataFrame surface.\n\nNote: for tpch/tpcds the expected-results registry already validates each\nsurface against a stored answer key (transitive equivalence), so they are\nlower priority here; the highest value is the dual-surface benchmarks with NO\noracle today (ssb first - canonical and small - then clickbench, joinorder,\ncoffeeshop, etc.).\n", "id": "benchmark-cross-surface-equivalence-gate", "preserves": ["ResultValidator tolerance/checksum semantics are unchanged; reuse, do not fork, the comparator and the DuckDB SQL-reference data builder.", "The TPC-Havoc SQL and DataFrame gates stay green with empty KNOWN_DIVERGENCES and unchanged behavior.", "The production DataFrame loader/adapters are reused, not forked; any change there (w6) is a genuine loader bug fix that keeps the real DataFrame run path working (re-verify the DataFrame benchmark run), never gate-only special-casing.", "benchbox run default (execution-only) behavior is unchanged for every benchmark; the gates are additive."], "prior_art": [{"concept": "benchbox/core//dataframe_queries(.py or package) and /queries.py (or get_queries) for each dual-surface benchmark - the two surfaces to compare (note: some, e.g. ssb/tpcds/clickbench, ship dataframe_queries as a package directory rather than a single module).", "decision": "reuse", "path": "benchbox/core//dataframe_queries(.py"}, {"concept": "benchbox/core/expected_results/ (registry.py, tpch_results.py, tpcds_results.py) - the only existing correctness oracles; documents that coverage stops at tpch/tpcds.", "decision": "reuse", "path": "benchbox/core/expected_results/"}, {"concept": "benchbox/core/runner/dataframe_runner.py and benchbox/platforms/dataframe/benchmark_mixin.py - how DataFrame queries actually execute and materialize; the gate must use the same execution path, not a bespoke one.", "decision": "reuse", "path": "benchbox/core/runner/dataframe_runner.py"}, {"concept": "benchbox/core/tpchavoc/dataframe_equivalence.py - build_dataframe_contexts, materialize_rows, fetch_canonical_rows, find_dataframe_divergences, main; the template to generalize (it already compares a DataFrame result to a SQL reference on DuckDB).", "decision": "reuse", "path": "benchbox/core/tpchavoc/dataframe_equivalence.py"}, {"concept": "benchbox/core/tpchavoc/equivalence.py - build_duckdb_with_tpch, find_divergences, strip_top_n, KNOWN_DIVERGENCES, main; the SQL-side harness and the empty-baseline fail-on-divergence pattern.", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py"}, {"concept": "tests/integration/test_tpchavoc_variant_equivalence.py - the fast default-lane regression companion; mirror it per gated benchmark.", "decision": "reuse", "path": "tests/integration/test_tpchavoc_variant_equivalence.py"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "active", "title": "Generalize the result-equivalence harness and gate SQL<->DataFrame equivalence for dual-surface benchmarks", "verifications": [{"command": "make tpchavoc-dataframe-equivalence-report", "description": "Generalized harness reproduces the existing TPC-Havoc DataFrame gate result", "expected": "still 0 divergent (re-expressed on the shared harness, unchanged)", "item_id": "benchmark-cross-surface-equivalence-gate", "last_result": null, "last_run": null, "seq": 1}, {"command": "introduce a wrong projection in one SSB DataFrame query, run the SSB cross-surface gate", "description": "A new dual-surface gate fails when one surface is deliberately broken", "expected": "gate FAILS (non-zero), naming the divergent query", "item_id": "benchmark-cross-surface-equivalence-gate", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": [], "notes": "Factor the data-build + run-SQL + run-DataFrame + compare loop out of\ntpchavoc/dataframe_equivalence.py into a shared core helper parameterized by\nbenchmark (DDL/data builder, SQL query accessor, DataFrame query accessor,\nvalidator). Re-express the existing TPC-Havoc DataFrame gate on top of it\nwith NO behavior change (its gate must stay green and byte-equivalent in\nresult). Reuse validate_variant_equivalence / the result validator and the\nDuckDB data builder; do not fork them.\n\nDONE: shared harness landed at benchbox/core/equivalence/dataframe_surface.py\n(build_dataframe_contexts, materialize_rows, fetch_reference_rows,\nfind_surface_divergences, SurfaceDivergence, _normalize_value). The TPC-Havoc\nDataFrame gate (benchbox/core/tpchavoc/dataframe_equivalence.py) is now a thin\nwrapper over it: it injects the canonical-TPC-H reference, the variant/backend\ncell enumeration, and the reused validate_variant_equivalence comparator +\nbuild_duckdb_with_tpch data builder. Gate re-verified byte-equivalent:\n\"checked 440 variant-backend cells - 0 divergent, 440 equivalent\". Fast-lane\ntests/integration/test_tpchavoc_dataframe_variant_equivalence.py still passes;\nnew fast-lane unit coverage at tests/unit/core/equivalence/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract a benchmark-agnostic cross-surface equivalence harness from the TPC-Havoc DataFrame gate", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w0"], "notes": "NOT YET COMPLETE \u2014 kept open so downstream work units that `needs: [w1]` are not\nunblocked with unverified query mappings: nyctaxi and tsbs still need independent\nmapping verification, and clickbench's SQL id set / construction is unconfirmed.\nFlip to done only once those three are verified.\n\nFor each benchmark with both surfaces, confirm the SQL and DataFrame query\nsets cover the same logical queries (matching ids/keys) and can run on a\nshared DuckDB cell. Record which are in scope, which are single-surface\n(out of scope here - owned by benchmark-correctness-oracle-coverage-map),\nand any that need per-benchmark normalization (top-N/LIMIT, column order).\n\nIN PROGRESS. Surface scan (dataframe_queries module OR package) confirms a\nDataFrame surface for: amplab, clickbench, coffeeshop, datavault,\nflightdata, joinorder, joinorder_synthetic, nyctaxi, read_primitives, ssb,\ntpcds, tpcds_obt, tpch, tpch_skew, tsbs_devops, h2odb (+ tpchavoc). The\nungated, no-expected-results dual-surface targets per the TODO are: ssb,\nclickbench, joinorder, coffeeshop, amplab, nyctaxi, flightdata, tsbs_devops,\ntpcds_obt, tpch_skew, read_primitives.\n\nConfirmed and gated: ssb - SQL get_queries() and the SSB_DATAFRAME_QUERIES\nregistry both key by identical ids Q1.1..Q4.3 (13 queries, 1:1). Per-\nbenchmark normalization finding: schema shape is heterogeneous - TPC-H/\ntpchavoc expose typed Table/Column objects, but ssb/amplab/coffeeshop/etc.\nexpose dict-shaped TABLES. The shared harness now offers\nbuild_dataframe_contexts_from_specs((name, [(col, sql_type)])) so both\nshapes reuse one DuckDB->Arrow->register loop.\n\nVERIFIED id-correspondence + feasibility map for the ungated targets\n(probed SQL get_queries() vs each DataFrame registry's get_query_ids()):\n - ssb 13 ids, 1:1 (Q1.1..Q4.3) GATED (#815).\n - coffeeshop 11 ids, 1:1 ready; but DuckDB\n ->Arrow date32 contexts break its pandas impls (date-vs-string\n compare) -> needs the data-path decision below before gating.\n - flightdata 20 ids, 1:1 (friendly names) ready; NOTE no\n synthetic generator module (uses real downloaded data) - confirm a\n bounded data source before gating.\n - read_primitives 157 SQL vs 152 DF gate the 152-id\n intersection (5 SQL-only ids have no DF impl).\n - amplab 8 ids, count matches; DF ids are Q-prefixed ('1'->'Q1')\n -> mechanical id normalization in the spec.\n - tpch_skew 22 ids, same; DF Q-prefixed ('1'->'Q1') -> same.\n - nyctaxi 25 ids; SQL friendly-names vs DF 'Q1'..'Q25' -> AMBIGUOUS\n ordering map; needs independent verification of which SQL id == which\n Qn before asserting (do NOT guess).\n - tsbs_devops 18 ids; same friendly-name vs 'Qn' ambiguity as nyctaxi.\n - joinorder 113 ids, 1:1 string keys BUT scale_factor=1.0-only\n (canonical IMDb) -> does NOT fit the SF=0.1 bounded cell; use SF=1 or\n joinorder_synthetic, a separate decision.\n - clickbench 43 DF ids; benchmark construction differs (abstract base)\n -> confirm SQL id set + construction before gating.\n - tpcds_obt (per source TODO) DF is a 3-query stub vs ~89 SQL - gate\n only the implemented intersection or defer.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "in_progress", "summary": "Inventory dual-surface benchmarks and confirm SQL<->DataFrame query-id correspondence", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5", "w6", "w7"], "notes": "Prioritize benchmarks with NO expected-results oracle: ssb, clickbench,\njoinorder, coffeeshop, amplab, nyctaxi, flightdata, tsbs_devops, tpcds_obt,\ntpch_skew, read_primitives. Report (benchmark, query, kind, deltas) without\nasserting. Capture stdout to /tmp and summarize counts in the PR/TODO.\n\nIN PROGRESS. ssb reported clean on the INTERIM DuckDB->Arrow substrate (#815):\n26 query-backend cells (13 queries x 2 backends) at SF=0.1, 0 divergent. This\nre-runs on the production-loader substrate (w5-w7); ssb is expected to stay\nclean but is re-confirmed there. Remaining ungated benchmarks are reported on\nthe production substrate as each gate is built (now needs w5/w6/w7).\n\namplab reported on the production substrate (SF=0.1, 16 query-backend cells =\n8 queries x 2 backends): 10/16 cells divergent on the FIRST report - exactly\nthe date-vs-string defect class w6 predicted (the 5 queries touching the\ndate32-typed uservisits.visitDate: Q2/Q2a/Q3/Q4/Q5, both backends), each\nraising \"cannot compare date to a string value\" under the production loader's\ndtypes. Burned down in w3 (see below); amplab now reports 0/16 divergent.\n\nAPPLICABILITY SWEEP: a reproducible sweep\n(_project/scripts/cross_surface_applicability_sweep.py ->\n_project/analysis/cross-surface-applicability.md, locked by\ntests/unit/test_cross_surface_applicability.py) classifies the dual-surface\nUNGUARDED benchmarks by whether they ship a DataFrame query `QueryRegistry`\n(the registry the cross-surface gate builders consume directly) - the\nauthoritative gate-applicability signal. `supports_dataframe` over-counts\n(it is a LOADING flag); the production query *resolver* under-counts (it\nmisses per-benchmark `_DATAFRAME_QUERIES` registries - e.g. it\nreturned 0 for coffeeshop, which #842 nonetheless gated). Registry detection\nfixes both.\n\nRESULT (16 unguarded candidates; coffeeshop already gated/excluded):\n - GATEABLE, ids overlap as-is (6): clickbench, flightdata, h2odb,\n joinorder, joinorder_synthetic, read_primitives (152/157 ids).\n - GATEABLE, needs id normalization first (6): amplab, datavault, nyctaxi,\n tpch_skew, tsbs_devops (ids Q-prefixed/friendly vs SQL ids), tpcds_obt\n (only 3 DF queries vs 89 SQL - partial).\n - w2 FALLBACK, NO DataFrame query registry (4): metadata_primitives,\n tpcdi, transaction_primitives, write_primitives. Only these four need a\n differential second-engine or curated expected-results oracle.\nSo the cross-surface gate burn-down is ~12 benchmarks, not 2 (the earlier\nresolver-based \"only 2\" was wrong).\n\nCAVEAT proven during the sweep: a GENERIC build is not load-faithful (it\nloads DataFrame contexts without the schema/headers the production loader\napplies, yielding spurious \"column not found\" divergences), so enumerating\nREAL divergence counts requires a load-faithful per-benchmark builder like\nbuild_ssb_duckdb. That builder work is w3 (clickbench done; the other 11\ngateable benchmarks next, normalizing ids where the sweep flags it).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "in_progress", "summary": "Run the cross-surface gate in report mode over the ungated dual-surface benchmarks and enumerate divergences", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w2"], "notes": "DONE. Burn-down complete for every wired/staged gate: ssb (0), amplab (10\ndate-vs-string impl bugs fixed in the DataFrame surface), coffeeshop (0),\njoinorder_synthetic (loader dtype/null fixes, w9 -> 26/26), clickbench (the\nremaining cells were the tie-ambiguous top-N class, not logic bugs - resolved\nby the w8 tie-aware comparator, with Q18's order-less LIMIT classified as a\njustified known-divergence). No real divergence was masked into a baseline;\nthe only baseline entry is ClickBench Q18 (genuinely order-less LIMIT). The\nlast in_progress reason was the ClickBench tie cells, which w8 cleared - so\nw3 closes with w8. Original plan below.\n\nFor each divergence, determine which surface is wrong and fix it; verify the\nfix makes the two surfaces agree. Where a difference is intentional and\ndefensible (e.g. a surface deliberately omits a presentational LIMIT),\nrecord it as a classified, justified known-divergence entry, mirroring the\nTPC-Havoc baseline discipline (default: empty baseline, every query matches).\n\nIN PROGRESS. ssb: no burndown required (0 divergences); baseline empty.\n\namplab: 10 divergences, all the projection-faithfulness/dtype class (NOT a\nloader bug). Root cause: the amplab DataFrame impls compared the date-typed\nvisitDate column to ISO-STRING literals (\"2000-01-01\"), which raises under\nthe production loader's date32 dtype - i.e. the production amplab DataFrame\nrun was itself broken for Q2/Q2a/Q3/Q4/Q5. Fixed the WRONG surface (the\nDataFrame impls), mirroring the TPC-H DataFrame queries whose date params are\ntyped datetime.date objects: AMPLAB_DEFAULT_PARAMS (and the impl fallbacks)\nnow carry date(2000,1,1)/date(2000,1,3) instead of strings, so the\ncomparison succeeds on both backends. No gate-local cast, no loader change,\nno baseline entry. Verified against CANONICAL SQL (the gate), not a sibling\nDataFrame impl: 0/16 divergent, with Q2 (70 rows), Q2a (6), Q4 (100)\nmeaningfully exercising the date filter+join (non-vacuous). baseline empty.\n\nclickbench: load-faithful gate WIRED (build_clickbench_duckdb), STAGED_GATES\n(NOT GATES), runnable via `make clickbench-cross-surface-equivalence-report`.\n\nBURN-DOWN PROGRESS (full detail:\n_project/analysis/clickbench-cross-surface-divergences.md):\n - DETERMINISM FIX (prerequisite): the ClickBench generator was UNSEEDED, so\n every run produced different data and top-N/tie-sensitive cells flipped\n divergence run-to-run (the gate was flaky). Added random.seed(42) in\n _generate_data_local (mirrors SSBDataGenerator); the gate is now\n deterministic (verified: two runs report the identical divergence set).\n - ALL 10 engine/error-category cells FIXED: added UnifiedStrExpr.replace\n (Q29) and UnifiedLazyFrame.slice (Q39-42) to the shared abstraction;\n compared EventDate to native date literals not strings (Q37-43); pandas\n `<> ''` filters now exclude NULL via .notna() to match SQL/Polars\n (Q25/27). 1129 dataframe/clickbench unit tests still pass.\n - REMAINING: 27 stable value-divergence cells across 14 queries. TRIAGE\n (verified by direct SQL-vs-DataFrame set/count comparison): these are\n mostly NOT logic bugs. The DataFrame computes CORRECT aggregates; the\n divergences are the gate's order-sensitive row-by-row comparison of\n tie-ambiguous top-N (`ORDER BY col ... LIMIT N` with ties spanning the\n cutoff). E.g. Q32/Q33 top-10 are all count=1 (thousands of count-1\n groups, LIMIT keeps an arbitrary 10); Q16 count distribution is IDENTICAL\n on both surfaces (only tied rows reorder). So the fix is GATE-LEVEL\n (deterministic total-order tie-breaker on both surfaces, OR a tie-aware\n validator, OR classify tie-ambiguous queries), NOT per-query ClickBench\n fixes. Exception: Q17/Q24 empty-string-vs-None is the deferred loader\n null/dtype materialization issue (see joinorder_synthetic). Detail:\n _project/analysis/clickbench-cross-surface-divergences.md.\nSo ClickBench has ~no genuine DataFrame logic bugs; promote STAGED_GATES->GATES\nonce the gate handles tie-ambiguity + the loader dtype fix lands (w4).\njoinorder_synthetic and the other registry-bearing benchmarks (per the\ncorrected applicability sweep) are the next gates to wire.\n\nGENERATOR SEEDING AUDIT (_project/analysis/gateable-generator-seeding-audit.md):\nan unseeded generator makes any cross-surface gate flaky (as ClickBench was).\nAudited every gateable benchmark's generator. Most are seeded (ssb, amplab,\nh2odb, tpch_skew, tsbs_devops, clickbench) or RNG-free (coffeeshop static\nseed-data, read_primitives). FIX: joinorder_synthetic used unseeded random\nand is the next gate to wire - added random.seed(42) in its\n_generate_data_local; verified its data tables are now byte-identical across\nruns (the only residual file-byte diff is the manifest timestamp, which the\ngate does not read). The five external/derived-data benchmarks (datavault,\nflightdata, joinorder, nyctaxi, tpcds_obt) have no synthetic RNG; confirm\ntheir bounded data source is stable when wiring each.\n\njoinorder_synthetic: gate WIRED (build_joinorder_synthetic_duckdb, STAGED_GATES,\n`make joinorder-synthetic-cross-surface-equivalence-report`; 13 queries x 2\nbackends). Wiring surfaced a SHARED DataFrame-loader bug: the production loader\napplies column names from get_benchmark_schema_columns, but that extractor did\nnot understand joinorder_synthetic's schema shape (columns are raw DDL strings\nlike \"id INTEGER PRIMARY KEY\") -> returned no columns -> every table loaded\nHEADERLESS, all 26 cells errored. Fixed schema_utils.py (column_name/\ncolumn_sql_type) to parse DDL-string columns - a general fix benefiting any\nstring-schema benchmark - dropping divergences 26 -> 10 (16 cells now pass;\n791 dataframe/equivalence unit tests still pass). REMAINING 10 cells are\ndtype-mismatch errors (loader applies column names but type-INFERS dtypes\ninstead of applying schema types: \"expected String type\" on 1a/1b/5a/8a/9a/10a;\n\"string vs numeric\" on 4a/12a). Follow-up: apply schema column TYPES when\nloading the DataFrame surface (types now available from column_sql_type). Detail:\n_project/analysis/joinorder-synthetic-cross-surface-divergences.md.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Burn down divergences (fix the wrong surface) or declare intentional ones", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w3", "w8", "w9"], "notes": "DONE for the two STAGED gates (the prerequisite-driven promotion w8/w9\nunblocked). clickbench and joinorder_synthetic moved STAGED_GATES -> GATES in\ncross_surface.py; both wired as BLOCKING steps in the pr.yml correctness-gate\njob (make clickbench-cross-surface-equivalence-report,\nmake joinorder-synthetic-cross-surface-equivalence-report) beside ssb/amplab/\ncoffeeshop; medium-marker fast-lane regressions added\n(tests/integration/test_{clickbench,joinorder_synthetic}_cross_surface_equivalence.py)\nmirroring test_ssb_cross_surface_equivalence.py; unit test updated\n(test_clickbench_and_joinorder_are_enforced_gates). Proved each FAILS on a\nplanted divergence (clickbench Q1 count+1000 -> Q1_expression+Q1_pandas;\njoinorder 1a value+99999 -> 1a_expression+1a_pandas; no collateral).\nclickbench's only baseline entry is the order-less Q18; joinorder_synthetic is\nclean. Regenerated the oracle coverage map (make oracle-coverage-map) - both\nnow show cross-surface \"guarded\" (15 UNGUARDED, was 17 dual-surface) - and\ncommitted it; updated the coverage-map TODO w1 dispatch note. STAGED_GATES is\nnow empty. ssb/amplab/coffeeshop and TPC-Havoc gates re-verified green.\n\nFOLLOW-ON (separate focused gate PRs, tracked by\nbenchmark-correctness-oracle-coverage-map w1, NOT a blocker for this item):\nwire the remaining gateable dual-surface benchmarks one PR each - datavault,\nflightdata, nyctaxi, read_primitives (152-id intersection), tpcds_obt\n(partial), tpch_skew, tsbs_devops (several need id normalization or a bounded\ndata-source check first), and joinorder (SF=1-only; covered by\njoinorder_synthetic). Each follows the same recipe: build__duckdb,\nregister in GATES, add a pr.yml step + a fast-lane test, prove a planted\ndivergence fails, regenerate the coverage map.\n\nDONE so far (follow-on gate PRs): h2odb (2026-06-24) - enforced GATE,\nbuild_h2odb_duckdb on a bounded SF=0.01 ~100k-row cell (its generator base is\nthe 10M-row small tier, so the shared SF=0.1 would emit ~1M rows; every query\nstays discriminating at 100k). SQL/DataFrame ids correspond 1:1 (Q1..Q10).\nWiring caught a REAL DataFrame bug: the Polars Q9 percentile defaulted to\n\"nearest\" interpolation, disagreeing with both pandas (linear) and SQL\nPERCENTILE_CONT - fixed the WRONG surface by passing interpolation=\"linear\"\n(added an additive `interpolation` arg to UnifiedExpr.quantile, default\n\"nearest\" so other callers are unchanged). Q9's only residue is one classified\nknown-divergence per backend: DuckDB's PERCENTILE_CONT over the DECIMAL(8,2)\nfare_amount column returns the percentile at the column's 2-decimal scale\n(77.87) while the DataFrame computes the same linear percentile over float64\n(77.874) - verified presentational (casting the SQL arg to DOUBLE yields the\nDataFrame value), not a logic bug. Evidence: gate green + deterministic across\n3 runs (18/20 cells strictly compared, 2 classified); planted Q4 corruption\nflags only Q4_expression (no collateral); coverage map shows h2odb guarded\n(UNGUARDED 15 -> 14). The applicability sweep's gateable example was retargeted\nh2odb -> flightdata (h2odb left the unguarded-candidate set).\n\nOriginal plan: one bounded small-SF DuckDB cell per gated benchmark, wired into\nthe correctness-gate CI job and a Make target, mirroring\ntpchavoc-(dataframe-)equivalence-report. Add a medium-marker pytest per\nbenchmark like tests/integration/test_tpchavoc_variant_equivalence.py.\nProve each gate fails when a deliberate divergence is introduced into one\nsurface.\n\nIN PROGRESS. Generic cross-surface gate landed at\nbenchbox/core/equivalence/cross_surface.py (find_cross_surface_divergences,\nCrossSurfaceGate, GATES registry) reusing the shared harness +\nResultValidator; the DataFrame side executes through the real\nDataFrameQuery.get_impl_for_family dispatch. First gated benchmark: ssb -\n`make ssb-cross-surface-equivalence-report` (added to the correctness-gate\nCI job in pr.yml), fast-lane tests/integration/test_ssb_cross_surface_equivalence.py\n(medium marker), and unit coverage tests/unit/core/equivalence/test_cross_surface.py.\nVerified the gate FAILS on a planted SSB defect (only Q1.1_expression\ndiverges, no collateral).\n\nSecond gated benchmark: amplab - `make amplab-cross-surface-equivalence-report`\n(added to the correctness-gate CI job in pr.yml beside ssb), fast-lane\ntests/integration/test_amplab_cross_surface_equivalence.py (medium marker).\nRegistered build_amplab_duckdb in the GATES registry (maps SQL ids \"1\"..\"5\"\nto DataFrame ids \"Q1\"..\"Q5\"). Verified the gate FAILS on a planted amplab\ndefect (Q4 expression total_revenue doubled -> only 4_expression diverges, no\ncollateral). Remaining ungated benchmarks: clickbench, joinorder, coffeeshop,\nnyctaxi, flightdata, tsbs_devops, tpcds_obt, tpch_skew, read_primitives - one\nfocused gate PR each.\n\nBLOCKED-ON for the two STAGED gates: clickbench and joinorder_synthetic are\nWIRED in STAGED_GATES (report mode) but cannot promote to GATES until the two\ncross-cutting prerequisite fixes land - w8 (tie-aware top-N comparator, clears\nClickBench's 27 tie-ambiguous cells) and w9 (loader dtype-from-schema, clears\njoinorder_synthetic's 10 cells + ClickBench Q17/Q24). Hence w4 now needs\n[w3, w8, w9]. ssb/amplab/coffeeshop were already promoted under this item and\nare unaffected by w8/w9.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Promote to bounded CI gates and add fast-lane regressions", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w1"], "notes": "DONE. Added build_production_contexts() in core/equivalence/cross_surface.py:\nper backend it instantiates the real adapter and calls the new public\nBenchmarkExecutionMixin.load_benchmark_into_context() - the exact data path\nrun_benchmark uses (_extract_benchmark_config + _load_data_phase, prefer_parquet\ndefault), stopping short of query execution. CrossSurfaceData now carries the\nbenchmark instance + data_dir (not table_specs); the DuckDB SQL reference and\nboth DataFrame backends read the SAME generated files. The comparator and the\nDuckDB SQL-reference builder are unchanged. (No new fork of the loader: the\ngate calls the production mixin.) Original plan below.\n\nReplace the gate's bespoke DuckDB->Arrow context-build\n(build_dataframe_contexts_from_specs) with one that loads each backend's\nDataFrame context via the REAL production loader, so the gate exercises the\nsame data path (and dtypes) a production DataFrame run uses - honoring the\n\"use the real runner/mixin path, not a bespoke one\" anti-pattern for DATA\nLOADING as well as query execution.\n\nReuse, do not fork: the reference adapters' load path\n(benchbox/platforms/dataframe/{polars_df,pandas_df}.py load_table /\nload_tables_from_data_source / _load_tables_into_context) and\nbenchbox/core/dataframe/data_loader.py DataFrameDataLoader. Drive it with\nthe benchmark's schema_info (column names/types) and the correct\ndelimiter/trailing-delimiter/parquet handling that a bare\nload_tables_from_data_source call skips (this was the cause of the SSB\nmis-load). Generate data once; the SQL reference (DuckDB) and BOTH DataFrame\nbackends must read the SAME generated data so \"same bounded cell\" still holds.\nKeep the comparator and the DuckDB SQL-reference builder reused (unchanged).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a production-faithful DataFrame-context substrate for the gate (reuse the real loader)", "wid": "w5"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5"], "notes": "DONE (#821). Made the Pandas loader's date-column inference TYPE-AWARE:\n_pandas_parse_date_columns now takes the schema column types and never\ntreats a numeric-typed column as a date even if its name ends in date/time,\nso SSB's INTEGER lo_orderdate/lo_commitdate load as integers (real string\ndates like d_date VARCHAR still parse). Behavior is unchanged when types are\nabsent. Threaded benchmark -> load_table -> _load_csv_files -> read_csv ->\n_pandas_parse_date_columns; abstract load_table signature made honest. SSB\nnow loads via the production loader (gate 26/26). Routing coffeeshop through\nthe real loader surfaced GENUINE divergences (date32-vs-string impl bugs,\nvalue mismatches, a KeyError) - logged for w2/w3 burndown, NOT masked.\nOriginal plan + diagnosis below.\n\nDriving the real loader over real benchmark data exposes loader defects the\ngate must not paper over. Known, reproduced:\n * Pandas adapter date heuristic: _pandas_parse_date_columns\n (benchbox/platforms/dataframe/pandas_df.py:62) infers date columns by\n NAME suffix only (\"*date\"/\"*_date\") with no type check, then\n _coerce_date_columns casts them to date32. SSB's lo_orderdate /\n lo_commitdate are INTEGER YYYYMMDD datekeys, so the loader tries to\n date-parse integers and raises (\"month must be in 1..12\"). Fix: gate\n date-coercion on the column's SCHEMA TYPE (thread schema_info types\n through), never the name alone, so integer datekeys load as integers.\nFor each benchmark, confirm its impls run under the loader's production\ndtypes; where an impl genuinely fails under the dtype its production run\nuses (e.g. coffeeshop pandas impls compare a date column to a STRING\nliteral), that is a REAL divergence/bug to report and burn down in w3 - do\nNOT add a gate-local cast to mask it. Only fix loader code where the LOADER\nis wrong (e.g. the type-blind date heuristic), not to accommodate a wrong impl.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix production-loader correctness bugs surfaced by real benchmarks (now in-scope: benchbox/platforms/dataframe/)", "wid": "w6"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5", "w6"], "notes": "DONE. The SSB gate + its fast-lane test now build DataFrame contexts via\nbuild_production_contexts (w5); re-verified 26/26, 0 divergent at SF=0.1 and\nre-proved the planted-defect failure (corrupting Q1.1's expression impl\nflags exactly Q1.1_expression, no collateral) on the new substrate. The\ntpchavoc DataFrame gate is LEFT on its DuckDB->Arrow path (it is a\nvariant-vs-canonical gate, not cross-surface) and re-confirmed green. The\nDuckDB->Arrow builder (build_dataframe_contexts_from_specs) remains in\ndataframe_surface for tpchavoc; only the cross-surface gate moved. Original\nplan below.\n\nSwitch the merged SSB gate (#815) from DuckDB->Arrow to the w5 substrate and\nre-verify it stays green (26/26) under production dtypes; this is the\nreference example every later benchmark follows. Re-confirm the TPC-Havoc\nSQL + DataFrame gates are unchanged (the tpchavoc DataFrame gate may keep its\nown DuckDB->Arrow path - decide whether to migrate it too or leave it, since\nit is a variant-vs-canonical gate, not cross-surface). Update the fast-lane\ntests accordingly. Re-prove the planted-defect failure on the new substrate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-base the SSB gate (and the shared harness) onto the production-loader substrate; re-verify", "wid": "w7"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w3"], "notes": "DONE. Made ResultValidator.validate_results_exact opt-in tie_aware (additive,\ndefault False so the TPC-Havoc gates are byte-unchanged). Chose option (b) (a\ntie-aware comparator) over (a) (the ClickBench SQL queries are also the perf\nqueries, so adding tie-breaker ORDER BY columns would pollute the benchmark)\nand over weakening to set-equality.\n\nMECHANISM: the relaxation engages ONLY after the strict sorted-positional\ncomparison fails, and accepts a divergence iff it is exactly an ambiguous\ntop-N boundary tie - deterministically, independent of run-to-run tie\nreordering:\n - the differing rows are the multiset difference each way (the swapped tie\n members); the rest are byte-identical (the deterministic remainder);\n - there must be an order-key column c that is monotonic across the\n reference result (the ORDER BY key is; a grouped non-key column is not),\n whose value on EVERY swapped row equals the reference's boundary value\n original[-1][c] (stable even when tied rows reorder), the boundary value\n occurs >=2 times (a genuine tie, not a unique last row), and is an\n extreme (min/max) of the column.\nSo a real value-misplacement bug outside the tie (wrong order-key value,\nwrong non-key value on a determined row, wrong row count, unique-last-row\nchange) puts a non-boundary value into the swap and still FAILS. The\ncross-surface gate passes tie_aware=True; TPC-Havoc keeps the default.\n\nAn EARLIER attempt inferred the key by splitting on all monotonic columns\njointly; DuckDB's non-deterministic tie order made a non-key column\n(SearchPhrase) coincidentally monotonic in ~1/3 of runs, so the gate was\nflaky. The multiset-difference + boundary-value formulation is order-stable\n(the count column always qualifies); verified 5 consecutive clean runs.\n\nQ18 (`... GROUP BY ... LIMIT 10` with NO ORDER BY over ~97k groups) is the\none genuinely order-less query - any 10 rows are valid and the two surfaces\npick different subsets (both verified subsets of the full grouped result).\nIt is the last-resort option (c): classified in clickbench's\nknown_divergences with justification, NOT masked (the tie-aware comparator\nhandles every ORDER BY ... LIMIT case; only this order-less LIMIT needs a\nbaseline entry).\n\nEVIDENCE: make clickbench-cross-surface-equivalence-report 27 -> 0\nunclassified (Q18's 1-2 classified cells aside), stable across 5 runs; ssb\n26/26, amplab 16/16, coffeeshop 22/22, joinorder_synthetic 26/26 all 0\ndivergent on the tie_aware path; TPC-Havoc SQL 220/220 and DataFrame 440/440\nunchanged (default strict path); a planted REAL ClickBench defect (Q1 count\n+1000) still flags Q1_expression+Q1_pandas; 80 tpchavoc+equivalence unit\ntests pass (6 new tie_aware cases lock accept/reject behavior). Files:\nbenchbox/core/tpchavoc/validation.py, benchbox/core/equivalence/cross_surface.py.\nOriginal triage below.\n\nPREREQUISITE for promoting ClickBench STAGED_GATES->GATES (w4). Surfaced and\ntriaged during w3 ClickBench burn-down (PR #848; detail in\n_project/analysis/clickbench-cross-surface-divergences.md).\n\nROOT CAUSE (verified by direct SQL-vs-DataFrame set/count comparison): the 27\nremaining ClickBench divergences are NOT logic bugs. The DataFrame computes\nCORRECT aggregates; the gate's comparator (ResultValidator.validate_results_exact\nin benchbox/core/tpchavoc/validation.py) sorts both sides then compares\nrow-by-row, so for `ORDER BY ... LIMIT N` queries whose ties span the\nLIMIT cutoff it reports a false divergence: SQL and the DataFrame each keep a\ndifferent (equally valid) subset of the tied rows. E.g. Q32/Q33 top-10 are all\ncount=1 (thousands of count-1 groups; LIMIT 10 keeps an arbitrary 10); Q16's\ncount distribution 4,4,4,4,4,4,3,3,3,3 is IDENTICAL on both surfaces, only the\ntied rows reorder/swap at the boundary.\n\nFIX (gate/validator-level, NOT 27 per-query ClickBench changes - pick one):\n (a) append a deterministic total-order tie-breaker (remaining columns / a\n stable key) to the ORDER BY on BOTH surfaces before comparison, or\n (b) make the comparator tie-aware: compare the MULTISET of rows sharing the\n boundary order-key value instead of position-by-position, or\n (c) classify genuinely tie-ambiguous queries as accepted/justified\n divergences (last resort - prefer a real total order).\nReuse, do not fork, ResultValidator; keep it able to catch genuine\nvalue-misplacement bugs (do not weaken to set-equality across all columns).\n\nVERIFY: after the fix, `make clickbench-cross-surface-equivalence-report` drops\nfrom 27 divergent cells to only the w9 loader-dtype residue (Q17/Q24), and a\nplanted real ClickBench defect still fails the gate (no masking).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the cross-surface comparator tie-aware for top-N queries (clears ClickBench's 27 tie-ambiguous cells)", "wid": "w8"}, {"evidence": null, "item_id": "benchmark-cross-surface-equivalence-gate", "needs": ["w5", "w6"], "notes": "DONE. Made the production DataFrame loader apply the schema's declared column\nTYPES (and DuckDB-matching empty-field semantics) instead of per-file\ninference. Three loader-layer fixes, all additive, no per-query changes:\n\n 1. SQL-type -> Arrow-type map was incomplete. The CSV->Parquet conversion\n (the Polars/expression path, which reads read_parquet) only typed the\n handful of names in SchemaMapper.PYARROW_TYPE_MAP (VARCHAR/CHAR/INTEGER/\n DATE/...), so declared TEXT/STRING/BIGINT/NUMERIC/... columns fell\n through to PyArrow inference - an all-NULL TEXT column (joinorder\n movie_companies.note) inferred as Null, a numeric-looking TEXT column\n (movie_info_idx.info ratings) as float64. Added\n SchemaMapper.sql_type_to_pyarrow() covering every SQL type family\n (TEXT/STRING/CHAR -> string, INT family -> int64, DECIMAL/FLOAT family\n -> float64, DATE -> date32, TIMESTAMP/DATETIME -> timestamp[us]) and\n routed _extract_arrow_types through it. Cleared the 6 Polars\n \"expected String type\" cells (1a/1b/5a/8a/9a/10a) and the float64-vs-str\n cells (4a/12a).\n 2. Pandas reads CSV, not Parquet (its recommended_data_format is CSV), so\n the Parquet fix did not reach it. read_csv now (a) reads declared text\n columns as object (so numeric-looking text is not inferred float64) and\n (b) parses a declared DATE/TIMESTAMP column as temporal even when its\n NAME does not match the date-suffix heuristic (ClickBench\n ClientEventTime TIMESTAMP was read as a raw string vs DuckDB's datetime).\n Date detection is now TYPE-aware (extends the w6 numeric-exclusion).\n 3. Empty-string vs NULL (ClickBench Q17/Q24): the DataFrame surface emitted\n None where the SQL surface emits \"\". Root cause was the per-benchmark\n CSV null_marker, NOT dtype: DuckDB resolves nullstr via\n resolve_csv_dialect (ClickBench null_marker=None -> empty kept \"\";\n JoinOrder null_marker=\"\" -> empty -> NULL), while the DataFrame path\n always nulled empties (coincidentally matching JoinOrder, diverging from\n ClickBench). Threaded the SAME resolved null_marker into both surfaces:\n the Parquet conversion sets strings_can_be_null = (null_marker is not\n None); pandas restores \"\" on declared string columns when null_marker is\n None; pandas load_table now resolves null_marker via resolve_csv_dialect\n (not the file-extension heuristic) so JoinOrder still nulls empties.\n Bumped the DataFrame Parquet cache v3->v4 (conversion rules changed).\n\nEVIDENCE:\n - make joinorder-synthetic-cross-surface-equivalence-report: 26/26, 0\n divergent (was 16/26). Deterministic (seeded generator).\n - make clickbench-cross-surface-equivalence-report: Q24 fully cleared (both\n backends); Q17's empty-string/dtype residue cleared - Q17's remaining\n divergence is the tie-ambiguous top-N (ORDER BY COUNT(*) DESC LIMIT 10,\n different tied UserIDs per backend), which is w8's domain, not w9. Overall\n clickbench 27 -> 22 divergent; the 22 are exactly the w8 tie-ambiguous set.\n - Enforced gates unaffected: ssb 26/26, amplab 16/16, coffeeshop 22/22, all\n 0 divergent (their null_marker is \"\" or .tbl, or None with no compared\n empties). tpchavoc SQL gate 220/220 and DataFrame gate 440/440, 0\n divergent (the latter uses DuckDB->Arrow, unaffected).\n - 1863 tests/unit/core/dataframe + core/equivalence + platforms/dataframe\n pass. ruff clean; no new ty diagnostics. Real benchbox-run DataFrame load\n path re-verified (data loads via the production loader unchanged; the\n NOT_RUN CLI query-selection quirk pre-exists on develop).\nFiles: benchbox/core/dataframe/data_loader.py (sql_type_to_pyarrow,\n_get_null_markers, null_marker threading, cache v4),\nbenchbox/platforms/dataframe/pandas_df.py (string-col dtype, type-aware date\nparse, empty-string restore), benchbox/platforms/dataframe/pandas_family.py\n(resolve_csv_dialect for the benchmark-instance null_marker). Original plan\nbelow.\n\nPREREQUISITE for promoting joinorder_synthetic STAGED_GATES->GATES (w4) and for\nClickBench Q17/Q24 (empty-string-vs-None). DEFERRED by user decision\n(2026-06-21) to keep that session focused; it remains REQUIRED for w4 and is\ntracked here so it is not lost. Detail:\n_project/analysis/joinorder-synthetic-cross-surface-divergences.md and the\nQ17/Q24 exception in clickbench-cross-surface-divergences.md.\n\nROOT CAUSE: the production DataFrame loader applies column NAMES from the schema\n(get_benchmark_schema_columns, fixed in #846 to parse DDL-string columns) but\nthen TYPE-INFERS dtypes per file rather than applying the schema's declared\ncolumn TYPES. So columns whose content looks numeric, or is null-heavy, get the\nwrong dtype and the query's comparisons fail. joinorder_synthetic: 10 cells -\n\"expected String type\" (Polars) on 1a/1b/5a/8a/9a/10a; \"string vs numeric\" on\n4a/12a. ClickBench: SQL emits \"\" where the DataFrame emits None (Q17 col 1,\nQ24 col 14) - same null/dtype materialization issue. NOTE the divergence\nreproduces on the read_parquet path (the loader converts CSV->parquet via\nDataFrameDataLoader, then read_parquet), NOT read_csv - an earlier CSV-layer\nschema_overrides attempt was on the wrong layer and was reverted (#847); the\nfix must apply schema types on the parquet/materialization path the loader\nactually uses.\n\nFIX (loader-layer, not per-query): apply the schema column TYPES when loading\nthe DataFrame surface (types are available from\nbenchbox/core/dataframe/schema_utils.py column_sql_type), so a column declared\nVARCHAR/TEXT loads as string and INTEGER as int instead of relying on per-file\ndtype inference. Reuse the production loader/adapters (benchbox/platforms/\ndataframe/, benchbox/core/dataframe/data_loader.py); keep the change additive\nand re-verify the normal DataFrame benchmark run path (must_preserve).\n\nVERIFY: `make joinorder-synthetic-cross-surface-equivalence-report` reaches\n26/26 (from 16/26); ClickBench Q17/Q24 clear; dataframe/equivalence unit tests\nstay green.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply schema column TYPES (not just names) when loading the DataFrame surface (clears joinorder_synthetic's 10 dtype cells)", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"CLI execution pipeline now calls `benchbox.core.benchmark_loader.get_benchmark_instance` instead of importing/core class construction.\"\n- \"Added `tests/unit/cli/test_cli_benchmark_boundaries.py` to assert delegation to core loader.\"", "id": "benchmark-management-boundaries", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Benchmark Management Boundaries", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "use the generic --benchmark-option K=V mechanism instead", "dont": "add individual CLI flags per benchmark parameter (--taxi-types, --year, --end-year)", "why": "this creates N*M option explosion across 21 benchmarks and 30+ existing CLI options"}, {"INSTEAD": "create a parallel BenchmarkHookRegistry", "dont": "merge benchmark-option parsing into PlatformHookRegistry", "why": "platform and benchmark options have different lifecycles (platform options go to DatabaseConfig, benchmark options go to constructor kwargs)"}, {"INSTEAD": "only forward keys that have registered BenchmarkOptionSpecs", "dont": "pass raw BenchmarkConfig.options dict as **kwargs to constructors", "why": "options contains orchestrator-level settings (tuning_enabled, validation flags, estimated_time_range) that would cause TypeError on benchmark constructors"}, {"INSTEAD": "use explicit spec registration per benchmark (same as platform-option)", "dont": "use Python inspect/reflection to auto-discover constructor params", "why": "it's fragile across **kwargs, defaults, and type annotations"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-04T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-option-cli-support", "id": 351, "reason": "Tab-completion for --benchmark-option keys and choices would improve DX but requires Click shell completion integration. Not needed for initial release.\n", "resolution": "promoted", "resolved_item": "add-benchmark-option-shell-completion", "resolved_reason": null, "summary": "Add benchmark-option shell completion"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-option-cli-support", "id": 352, "reason": "Could catch mismatches between registered spec names and actual constructor param names. Low risk since specs are co-located with benchmark code and tested. Adds import-time overhead.\n", "resolution": "promoted", "resolved_item": "validate-benchmark-options-against-constructor-signature-at", "resolved_reason": null, "summary": "Validate benchmark options against constructor signature at registration time"}], "deps": [], "description": "Benchmark constructors accept parameters that are unreachable from the CLI. The orchestrator (_get_benchmark_instance) only forwards a fixed set of kwargs (scale_factor, compress_data, compression_type, compression_level, verbose, quiet, parallel). Benchmark-specific parameters like taxi_types, year, months, end_year, num_hosts, skew_preset, dimensions, hash_algorithm, seed, and force_regenerate never reach the constructor.\nThis affects 10 of 21 benchmarks. The fix is a --benchmark-option K=V mechanism mirroring the existing --platform-option K=V pattern. Each benchmark registers typed option specs; the orchestrator merges parsed options into constructor kwargs.", "id": "benchmark-option-cli-support", "preserves": ["--platform-option continues working independently", "All existing benchmark default behavior unchanged when no --benchmark-option provided", "BenchmarkConfig.options continues serving its existing purposes (tuning config, validation flags, power run settings)", "Benchmarks with only standard params (tpch, tpcds, ssb, clickbench, h2odb, etc.) unaffected", "seed and force_regenerate continue working in their current RunConfig/orchestrator paths - benchmark-option adds a parallel path to the constructor"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/*/downloader.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/*/queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/*/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/benchmark_hooks.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/datavault/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/flightdata/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds_obt/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch_skew/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tsbs_devops/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/vector_search/benchmark.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_hooks.py"}], "state": "done", "title": "Add --benchmark-option K=V CLI support for benchmark-specific parameters", "verifications": [{"command": "benchbox run --platform duckdb --benchmark nyctaxi --scale 0.01 --benchmark-option taxi_types=yellow,green,hvfhv --dry-run\n", "description": "NYC Taxi benchmark accepts taxi_types via CLI", "expected": "dry run shows 3 taxi types, 39 queries", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 1}, {"command": "benchbox run --platform duckdb --benchmark tpch --scale 0.01 --benchmark-option nonexistent=value --dry-run\n", "description": "Unknown benchmark option rejected with clear error", "expected": "error message listing available options (or none)", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -m fast -q", "description": "All existing tests pass (no regression)", "expected": "all pass, 0 failures", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/cli/test_benchmark_hooks.py -v", "description": "Benchmark hook registry unit tests pass", "expected": "all pass", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 4}, {"command": "benchbox run --help-topic benchmarks", "description": "Help-topic shows registered benchmark options", "expected": "output includes taxi_types, year, months for nyctaxi and skew_preset for tpch_skew", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "benchmark-option-cli-support", "needs": [], "notes": "Mirror platform_hooks.py structure: - BenchmarkOptionError (ValueError subclass) - BenchmarkOptionSpec dataclass (name, parser, default, help, choices, aliases) - List/enum parser functions: parse_int_list, parse_str_list, parse_enum_list factory - BenchmarkHookRegistry class: register_option_specs(benchmark, *specs),\n parse_options(benchmark, provided), describe_options(benchmark), get_default_options(benchmark)\n- Reuse existing parse_bool and _identity from platform_hooks or extract to shared utils\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create BenchmarkHookRegistry and BenchmarkOptionSpec in benchbox/cli/benchmark_hooks.py", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w1"], "notes": "In benchbox/cli/commands/run.py: - Add BenchmarkOptionParamType (same pattern as PlatformOptionParamType) - Add @click.option(\"--benchmark-option\", ..., type=BenchmarkOptionParamType(),\n multiple=True, hidden=True) - parallel to --platform-option\n- Parse benchmark options via BenchmarkHookRegistry.parse_options(benchmark, pairs) - Store parsed options in BenchmarkConfig.options under a namespaced key\n (e.g., options[\"benchmark_options\"] = {...}) to avoid collision with existing options\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --benchmark-option Click option to run.py and wire parsing", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w2"], "notes": "In benchbox/cli/orchestrator.py _get_benchmark_instance(): - Extract benchmark_options from config.options.get(\"benchmark_options\", {}) - Merge into kwargs dict before calling benchmark_class(**kwargs) - seed and force_regenerate: these already live in BenchmarkConfig.options\n but are NOT forwarded to constructors today. The orchestrator should check\n if the current benchmark has registered BenchmarkOptionSpecs for \"seed\" or\n \"force_regenerate\" (via BenchmarkHookRegistry.list_option_specs(benchmark)),\n and if so, include them in kwargs from config.options. Benchmarks that don't\n register these specs simply never receive them (no change). This avoids a\n dual-path where seed flows through both config.options and --benchmark-option.\n- Preserve existing try/except TypeError fallback for parallel param\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire orchestrator to forward benchmark options to constructor kwargs", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w1"], "notes": "These benchmarks require the custom parser functions built in w1 (parse_enum_list, parse_int_list, parse_datetime). This is the highest-risk work unit - validate parser edge cases (empty lists, single values, whitespace) here before w5.\nNYC Taxi (benchbox/core/nyctaxi/benchmark.py): - taxi_types: parse_enum_list(TaxiType), default=None, choices description - year: int parser, default=2019, choices=range(2019,2026) - months: parse_int_list, default=None - seed: int parser, default=None\nTSBS DevOps (benchbox/core/tsbs_devops/benchmark.py): - num_hosts: int parser, default=None - duration_days: int parser, default=None - interval_seconds: int parser, default=10 - start_time: parse_datetime, default=None\nRegistration via module-level BenchmarkHookRegistry.register_option_specs() calls (same pattern as platform option registration in benchbox/platforms/__init__.py).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register option specs for high-complexity benchmarks (NYC Taxi, TSBS DevOps - enum lists, datetime, int lists)", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w1"], "notes": "These benchmarks use only str, int, parse_bool, parse_str_list, and choices-constrained str parsers - all straightforward. Lower risk than w4.\nFlightData: end_year (int, default=LAST_AVAILABLE_YEAR) DataVault: hash_algorithm (str, choices=[\"md5\",\"sha256\",...]), record_source (str) TPC-DS OBT: tpcds_source_dir (str), dimension_mode (str, choices), channels (parse_str_list), output_format (str, choices) TPC-H Skew: skew_preset (str, choices=[\"none\",\"light\",\"moderate\",\"heavy\",\"extreme\",\"realistic\"]) Vector Search: dimensions (int, default=128) Join Order: queries_dir (str) TPC-DI: enable_parallel (parse_bool, default=False), max_workers (int)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register option specs for simple-type benchmarks (FlightData, DataVault, TPC-DS OBT, TPC-H Skew, Vector Search, Join Order, TPC-DI)", "wid": "w5"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w3", "w4"], "notes": "tests/unit/cli/test_benchmark_hooks.py: - Registry: register, parse, describe, defaults, unknown key rejection, duplicate rejection - Parsers: parse_int_list, parse_str_list, parse_enum_list (TaxiType), parse_datetime - Integration: BenchmarkOptionParamType Click type, end-to-end from CLI string to constructor kwarg - Regression: benchmarks without registered specs still construct normally\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for BenchmarkHookRegistry, parsers, and end-to-end wiring", "wid": "w6"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w4", "w5"], "notes": "Extend --help-topic system (benchbox/cli/help.py) to show available benchmark options per benchmark. Example output for --help-topic benchmarks or --help-topic nyctaxi:\n nyctaxi:\n taxi_types - Taxi data types to load (yellow,green,hvfhv) [default: yellow]\n year - Year of TLC data (2019-2025) [default: 2019]\n months - Months to include, comma-separated (1-12) [default: all]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --help-topic documentation for benchmark options", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "only sync generators that should follow the OBT/primary output root.", "dont": "blanket-sync every generator attribute", "why": "tpcds_obt/datavault intentionally diverge"}, {"INSTEAD": "delegate to the config helpers.", "dont": "duplicate ETL path math in the tpcdi setter", "why": "TPCDIConfig already owns it"}], "approach": "Prefer reusing GeneratorOutputDirMixin where the generator is a plain instance\nattribute (tpcds). For tpcdi, whose layout is config-driven and richer than a\nsingle generator, add a focused property setter that re-derives the ETL\ndirectories rather than shoehorning it into the mixin. Land tests first that\nfail on the current tree.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-11T00:00:00Z", "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [], "deps": ["benchmark-runs-output-root-propagation"], "description": "PR #759 made benchmark datagen roots honor BENCHBOX_OUTPUT_DIR at construction\nand added GeneratorOutputDirMixin so a post-construction output_dir\nreassignment propagates to nested generators. Several generator-backed\nbenchmarks (h2odb, ssb, tpch, nyctaxi, etc.) opted into the mixin; a few did\nnot because their generators are lazy or intentionally point elsewhere.\n\nTwo benchmarks build a generator (or derive directories) in __init__ but have\nno synchronizing output_dir setter, so an explicit CLI --output root that\ndiffers from the construction default does NOT reach them:\n\n- tpcds: builds its data_generator in __init__ and has no output_dir setter, so\n a post-construction reassignment (runner _resolve_output_dir_handler) leaves\n the generator pointed at the construction-time path.\n- tpcdi: derives source_dir/staging_dir/warehouse_dir and builds its\n data_generator in __init__ from self.config.output_dir; a later output_dir\n reassignment updates neither the ETL subdirs nor the generator.\n\nThis is a pre-existing residual gap, NOT a regression. The reported failure\nmode (BENCHBOX_OUTPUT_DIR ignored) is already fixed for both: their\nconstruction-time defaults resolve via the env-aware helper (tpcds via its\ndefault path, tpcdi via TPCDIConfig.get_benchmark_runs_datagen_path), and in\nthe managed/env flow the runner's reassignment target equals the construction\ndefault, so nothing diverges. Only an explicit `--output /custom` that differs\nfrom the default exposes the gap.\n\nClose it so an explicit override propagates correctly for these two\nbenchmarks, matching the contract the mixin already provides elsewhere. NOTE:\nbenchmark-output-root-resolve-at-construction-boundary proposes a deeper fix\n(resolve the root before construction) that would subsume this item; this is\nthe tactical fix if that refactor is deferred.", "id": "benchmark-output-root-explicit-override-propagation", "preserves": ["Env/managed-flow behavior for tpcds and tpcdi remains unchanged (construction default already env-aware).", "tpcdi data reuse: existing source/staging/warehouse layout names remain compatible.", "tpcds_obt and datavault lazy generators that intentionally write to a separate source dir are NOT force-synced."], "prior_art": [{"concept": "benchbox/base.py:GeneratorOutputDirMixin \u2014 reuse/extend; tpcds can adopt it directly (its data_generator is a plain instance attribute)", "decision": "extend", "path": "benchbox/base.py:GeneratorOutputDirMixin"}, {"concept": "benchbox/core/read_primitives/benchmark.py + benchbox/core/transactional/benchmark_base.py \u2014 reuse the existing output_dir setter pattern that re-derives nested state", "decision": "reuse", "path": "benchbox/core/read_primitives/benchmark.py"}, {"concept": "benchbox/core/runner/runner.py:_resolve_output_dir_handler \u2014 the post-construction assignment site this fix must satisfy", "decision": "reuse", "path": "benchbox/core/runner/runner.py:_resolve_output_dir_handler"}, {"concept": "benchbox/core/tpcdi/config.py:TPCDIConfig (output_dir + create_directories) \u2014 reuse for re-deriving source/staging/warehouse on reassignment instead of duplicating path math", "decision": "reuse", "path": "benchbox/core/tpcdi/config.py:TPCDIConfig"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/datavault/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds_obt/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}], "state": "done", "title": "Propagate explicit --output override to nested generators for tpcds and tpcdi", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py -q", "description": "Explicit override now propagates for tpcds and tpcdi", "expected": "tpcds and tpcdi override cases pass (previously xfail)", "item_id": "benchmark-output-root-explicit-override-propagation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpcds tests/unit/core/tpcdi -q", "description": "tpcds and tpcdi unit suites still pass", "expected": "all tests pass", "item_id": "benchmark-output-root-explicit-override-propagation", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": [], "notes": "Construct each, then assign a distinct explicit output_dir and assert the\nnested generator / ETL subdirs do NOT follow today. Capture the probe to\n/tmp/override-gap-w0.log (do not commit) and record results in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm the override gap reproduces for tpcds and tpcdi on current develop", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": ["w0"], "notes": "tpcds builds data_generator as a plain attribute in __init__, so the mixin\napplies cleanly. Verify the tpcds_obt source-dir behavior is unaffected\n(its generator is lazy and intentionally writes to a separate dir).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Adopt GeneratorOutputDirMixin for tpcds so its data_generator tracks output_dir", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": ["w0"], "notes": "On reassignment, update source_dir/staging_dir/warehouse_dir and\ndata_generator.output_dir, keeping self.config.output_dir consistent. Guard\nagainst running before config exists during __init__.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a tpcdi output_dir setter that re-derives ETL subdirs and syncs the generator", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": ["w1", "w2"], "notes": "Coordinate with benchmark-output-root-regression-guard, which marks these\ntwo xfail; this work removes the xfail once propagation is fixed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Flip the tpcds/tpcdi xfail markers in the regression guard to passing", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "assert the nested generator/downloader paths too.", "dont": "assert only benchmark.output_dir", "why": "the original bug lived in nested generators"}, {"INSTEAD": "enumerate via get_all_benchmarks() instead.", "dont": "hand-list benchmark ids", "why": "the point is to auto-cover future benchmarks"}], "approach": "Build the guard on top of the registry so it auto-covers new benchmarks. Reuse\nthe helper functions already in the propagation test for resolving nested\ngenerators. Prefer xfail with an explicit reason over skipping for the known\ntpcds/tpcdi override gap, so the tracker shows the residual work.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-11T00:00:00Z", "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-output-root-regression-guard", "id": 353, "reason": "Cleanup of existing on-disk artifacts belongs to benchmark-runs-duplicate-artifact-cleanup, not this regression-guard item.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Cleanup of orphaned non-canonical datagen dirs is explicitly assigned to benchmark-runs-duplicate-artifact-cleanup, which is open.", "summary": "Clean up orphaned non-canonical datagen dirs (e.g. nyctaxi_1.0) left by direct library users before PR #759 canonicalized the names"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-output-root-regression-guard", "id": 354, "reason": "That filesystem-at-runtime assertion is owned by benchmark-runs-local-artifact-guardrails; this item is the unit-test + static-source counterpart (construction contract and hardcoded-default lint).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Runtime guardrail explicitly owned by benchmark-runs-local-artifact-guardrails, which is already DONE (_project/DONE/main/planning/); no open work remains here.", "summary": "Runtime/UAT guardrail that fails a run when worktree-local benchmark_runs writes appear despite an external output root"}], "deps": ["benchmark-runs-output-root-propagation"], "description": "PR #759 fixed the lifecycle-ordering bug where generator-backed benchmarks\nignored BENCHBOX_OUTPUT_DIR and wrote datagen output under a worktree-local\nbenchmark_runs/. The fix made BaseBenchmark's default datagen root env-aware\nand added an opt-in GeneratorOutputDirMixin that propagates a resolved root to\nnested generators. The accompanying tests\n(tests/unit/core/test_benchmark_output_root_propagation.py,\ntests/unit/mcp/test_benchmark_output_root_propagation.py) only assert a\nhand-picked subset of benchmark families.\n\nThe footgun is structural: a new generator-backed benchmark that builds its\ngenerator in __init__ and forgets the mixin (or hardcodes\nPath.cwd()/\"benchmark_runs\") silently reintroduces the original bug, and no\ntest or lint catches it. A one-off manual probe across all 24 registered\nbenchmarks confirmed the contract currently holds, but that evidence is not\ndurable.\n\nConvert that probe into a registry-driven regression guard so the contract is\nenforced for every current and future benchmark, and add a cheap static guard\nthat fails when a benchmark constructor hardcodes a cwd-local benchmark_runs\ndefault instead of the shared resolver.", "id": "benchmark-output-root-regression-guard", "preserves": ["The existing per-family assertions in test_benchmark_output_root_propagation.py keep passing (extend, do not delete).", "tests remain in the fast lane and write nothing under the active worktree."], "prior_art": [{"concept": "_project/TODO/main/planning/benchmark-runs-local-artifact-guardrails.yaml \u2014 align with the runtime guardrail effort; this item is the test/lint counterpart", "decision": "reuse", "path": "_project/TODO/main/planning/benchmark-runs-local-artifact-guardrails.yaml"}, {"concept": "benchbox/base.py:GeneratorOutputDirMixin (OUTPUT_DIR_GENERATOR_ATTRS) \u2014 reuse to discover the nested-generator attributes to assert against", "decision": "reuse", "path": "benchbox/base.py:GeneratorOutputDirMixin"}, {"concept": "benchbox/core/benchmark_registry.py:get_all_benchmarks / get_public_benchmark_class \u2014 reuse to enumerate every benchmark under test", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py:get_all_benchmarks"}, {"concept": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path / resolve_benchmark_runs_dir \u2014 reuse as the canonical env-aware root for assertions", "decision": "reuse", "path": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path"}, {"concept": "tests/unit/core/test_benchmark_output_root_propagation.py \u2014 extend the per-family assertions into a registry-parametrized form rather than adding a parallel test file", "decision": "extend", "path": "tests/unit/core/test_benchmark_output_root_propagation.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/base.py"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": ".github/workflows/lint.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}], "state": "done", "title": "Registry-wide regression guard for benchmark output-root propagation", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py tests/unit/mcp/test_benchmark_output_root_propagation.py -q", "description": "New registry-wide propagation tests pass", "expected": "all tests pass; every registered benchmark is parametrized", "item_id": "benchmark-output-root-regression-guard", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_guard.py -q", "description": "Static guard fails on a deliberately hardcoded cwd default", "expected": "guard passes on current tree and fails when a benchmark hardcodes Path.cwd()/benchmark_runs", "item_id": "benchmark-output-root-regression-guard", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite still passes", "expected": "no new failures attributable to this change", "item_id": "benchmark-output-root-regression-guard", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": [], "notes": "Construct each id from get_all_benchmarks() with BENCHBOX_OUTPUT_DIR set to\na tmp path and assert output_dir plus any OUTPUT_DIR_GENERATOR_ATTRS\ngenerator resolves under it. Capture stdout to /tmp/output-root-guard-w0.log\n(do not commit) and record the pass count and any exceptions in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate that every registered benchmark currently honors the env output root", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w0"], "notes": "Parametrize over get_all_benchmarks(). For each, construct via the public\nclass (mirroring the MCP/library entry point) with BENCHBOX_OUTPUT_DIR set;\nassert output_dir and each present nested generator/downloader resolve under\nthe root and that Path.cwd() is not a parent. Resolve nested generators via\n_impl when the public wrapper delegates.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a registry-parametrized test asserting env-root honoring at construction for all benchmarks", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w1"], "notes": "For benchmarks that build a generator in __init__, assign a fresh explicit\nroot after construction and assert the nested generator follows. Mark the\nknown non-mixin benchmarks (tpcds, tpcdi) xfail and reference\nbenchmark-output-root-explicit-override-propagation so the gap is tracked,\nnot hidden.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a test asserting post-construction output_dir reassignment propagates to nested generators", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w0"], "notes": "A unit test (or ruff/grep-based check wired into ci-lint) that scans\nregistered benchmark class source files for Path.cwd()/\"benchmark_runs\"\ndefaults and fails with guidance to use get_benchmark_runs_datagen_path\ninstead. Allow a documented exception list only for intentional non-datagen\npaths.\n\nScan target must cover ALL registered benchmark sources, not only files named\nbenchmark.py. For example, TPCDSBenchmark lives in\nbenchbox/core/tpcds/benchmark/runner.py, not benchmark.py. Derive scan\ntargets from the registered class __module__ paths (via get_all_benchmarks())\nrather than a glob like benchbox/core/**/benchmark.py so future benchmarks\nadded in non-standard file names are automatically covered.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a static guard that fails on hardcoded cwd-local benchmark_runs defaults in constructors", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w1"], "notes": "Assert that a CloudStagingPath/DatabricksPath assigned as the output root\npropagates to the nested generator without being re-wrapped or losing cloud\nsemantics. Use a mocked/staging path so no live cloud access is required.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend coverage to a cloud/dbfs output root through at least one generator-backed benchmark", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "resolve through it at the boundary.", "dont": "add a third resolution helper", "why": "get_benchmark_runs_datagen_path already exists"}, {"INSTEAD": "keep a thin shim.", "dont": "remove the output_dir setter compatibility path outright", "why": "external/library callers assign it post-construction"}], "approach": "Treat this as an architecture-altitude change: prefer generalizing the\nresolve-once-at-boundary mechanism over layering more per-class setters. Land\nit behind the regression guard from benchmark-output-root-regression-guard so\nany behavior drift is caught. Sequence: prove the data-source root is\nresolvable pre-construction, inject at the boundary, then simplify the now-\nredundant mutation/setters.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-14T00:00:00Z", "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-output-root-resolve-at-construction-boundary", "id": 355, "reason": "Cloud roots resolve later via credentials in _apply_default_cloud_output_dir; needs its own analysis once the local-path boundary is settled.", "resolution": "promoted", "resolved_item": "cloud-staging-path-cloudstagingpath-databrickspath-construct", "resolved_reason": null, "summary": "Cloud staging path (CloudStagingPath/DatabricksPath) construction-time injection edge cases"}], "deps": ["benchmark-output-root-regression-guard"], "description": "PR #759 fixed the output-root propagation bug with two cooperating mechanisms:\nan env-aware default in BaseBenchmark, and an opt-in GeneratorOutputDirMixin\nwhose setter re-syncs nested generators when output_dir is reassigned after\nconstruction. This works, but it leaves the original architectural smell that\nthe TODO's w2 explicitly preferred to fix: the orchestrator still constructs\nthe benchmark BEFORE the final output root is resolved, then mutates\nbenchmark.output_dir afterward (orchestrator data-sharing redirect, runner\n_resolve_output_dir_handler).\n\nConsequences of keeping post-construction mutation as the contract:\n\n- Footgun: any new generator-backed benchmark that builds its generator in\n __init__ and forgets the mixin silently reintroduces the bug. The mixin is\n opt-in, so correctness depends on author memory.\n- Duplication: GeneratorOutputDirMixin now overlaps with the bespoke output_dir\n setters in read_primitives/benchmark.py and transactional/benchmark_base.py.\n- Residual gaps: benchmarks with lazy/derived layouts (tpcds, tpcdi) need the\n separate explicit-override fix tracked in\n benchmark-output-root-explicit-override-propagation.\n\nInvestigate resolving the benchmark-specific datagen directory once at the call\nboundary and injecting it into the constructor, so generators capture the\ncorrect root immediately and post-construction mutation (and the mixin) can be\nretired or reduced to a thin compatibility shim. If adopted, this would subsume\nthe explicit-override item and let the duplicated setters collapse onto one\nmechanism.", "id": "benchmark-output-root-resolve-at-construction-boundary", "preserves": ["BENCHBOX_OUTPUT_DIR remains the shared-root mechanism; cwd-local default preserved when neither is set.", "CLI --output remains the highest-precedence output root.", "Data-sharing benchmarks (read/write/transaction primitives) keep reusing the source benchmark's datagen dir.", "External callers that still assign benchmark.output_dir after construction continue to work (compatibility shim)."], "prior_art": [{"concept": "benchbox/base.py:GeneratorOutputDirMixin and benchbox/core/read_primitives + transactional output_dir setters \u2014 candidates to dedupe once mutation is removed", "decision": "reuse", "path": "benchbox/base.py:GeneratorOutputDirMixin"}, {"concept": "benchbox/cli/orchestrator.py:_get_benchmark_instance / _resolve_output_root / _resolve_custom_output_root \u2014 the construction + resolution sites to reorder", "decision": "reuse", "path": "benchbox/cli/orchestrator.py:_get_benchmark_instance"}, {"concept": "benchbox/core/benchmark_loader.py:instantiate_benchmark_class / constructor_accepts_argument \u2014 reuse to inject output_dir only where the constructor accepts it", "decision": "reuse", "path": "benchbox/core/benchmark_loader.py:instantiate_benchmark_class"}, {"concept": "benchbox/core/runner/runner.py:_resolve_output_dir_handler \u2014 the post-construction mutation this item aims to retire or shrink", "decision": "reuse", "path": "benchbox/core/runner/runner.py:_resolve_output_dir_handler"}, {"concept": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path \u2014 reuse as the single resolver at the boundary", "decision": "reuse", "path": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "only_modify", "path_glob": "benchbox/core/transactional/benchmark_base.py"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Resolve benchmark datagen root at the construction boundary to remove post-construction mutation", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py tests/unit/mcp/test_benchmark_output_root_propagation.py -q", "description": "Output-root regression guard passes after the refactor", "expected": "all propagation tests pass with no xfails remaining for generator-backed benchmarks", "item_id": "benchmark-output-root-resolve-at-construction-boundary", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/test_orchestrator_data_sharing.py tests/unit/core/test_runner_helpers.py -q", "description": "Orchestrator/runner and data-sharing suites pass", "expected": "all tests pass", "item_id": "benchmark-output-root-resolve-at-construction-boundary", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite passes", "expected": "no new failures", "item_id": "benchmark-output-root-resolve-at-construction-boundary", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": [], "notes": "Enumerate orchestrator, runner, MCP, UAT runner, and direct test\nconstructors. For each, record whether the final root is knowable before\nconstruction and what currently blocks passing it in (e.g. data-sharing\nsource resolution needing an instance).\nDONE: the two mutation sites were orchestrator._get_benchmark_instance\n(data-sharing redirect) and runner._resolve_output_dir_handler. The final\nlocal root is knowable before construction once the data-source name is a\nclass attribute; only cloud --output roots stay deferred (need platform\nconfig), handled by the post-construction shim.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Map every site that constructs a benchmark then mutates output_dir", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": ["w0"], "notes": "get_data_source_benchmark() is currently an instance method, which forces\nconstruct-then-redirect for read/write/transaction primitives. Evaluate a\nclass-level/registry lookup so the shared datagen dir is resolvable before\nconstruction.\nDONE: added BaseBenchmark.DATA_SOURCE_BENCHMARK class attribute. The\ninstance method now falls back to it, and read/transaction/write\nprimitives set it instead of overriding the method, so the shared root is\nresolvable from the class alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Resolve the data-source root without a constructed instance", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": ["w1"], "notes": "Pass output_dir through instantiate_benchmark_class where accepted. Preserve\nCLI --output precedence over BENCHBOX_OUTPUT_DIR and the cwd fallback when\nneither is set.\nDONE: orchestrator._resolve_construction_output_dir resolves the root\n(CLI local --output > shared data-source root > managed per-benchmark\npath) and injects it via optional_kwargs. Cloud --output returns None and\ndefers to the shim. BENCHBOX_OUTPUT_DIR / cwd fallback still apply via the\nBaseBenchmark default when no root is injected.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inject the resolved datagen dir into constructors at the orchestrator/runner boundary", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": ["w2"], "notes": "Once construction receives the final root, reduce _resolve_output_dir_handler\nto a no-op/compat shim and collapse GeneratorOutputDirMixin and the\nread_primitives/transactional setters onto a single mechanism, keeping a\ncompatibility path for external callers that still assign output_dir.\nDONE: _resolve_output_dir_handler is now a shim that only assigns when the\nresolved handler differs from the live one (no redundant re-wrap on the\nconstruction-injected path; still applies cloud handlers and external\noutput_root). read_primitives and transactional benchmark_base dropped\ntheir bespoke setters in favor of GeneratorOutputDirMixin; the mixin\nremains the single propagation mechanism and the public output_dir setter\nstays as the compatibility path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire or shrink post-construction mutation and dedupe the output_dir setters", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "clean before the output-root bug is fixed or guarded", "why": "that only frees space temporarily."}, {"INSTEAD": "(not specified)", "dont": "delete result bundles or manifests just because they live under a duplicate root", "why": "classify and preserve unique evidence first."}, {"INSTEAD": "(not specified)", "dont": "run `rm -rf` from a parent directory or with a glob", "why": "cleanup must list exact approved paths."}, {"INSTEAD": "(not specified)", "dont": "use Docker/system prune as part of this task", "why": "the target is BenchBox-generated filesystem data only."}], "approach": "Treat cleanup as a controlled operations task. First prove the root cause is\nfixed or guarded. Then inventory all repo and pool worktree local\n`benchmark_runs` trees, classify datagen-only duplicates versus durable\nresult evidence, compare against the shared root, and ask for explicit\napproval with exact paths and byte counts before deletion. Finish with a\nrepresentative external-root smoke to prove cleanup does not immediately\nregress.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-01T00:00:00Z", "deferrals": [], "deps": ["benchmark-runs-local-artifact-guardrails", "benchmark-runs-output-root-propagation"], "description": "Once output-root propagation and artifact guardrails are in place, audit and\nclean the already-written duplicate generated data from repo and pool\nworktrees.\n\nKnown duplicate/generated locations from the 2026-06-01 investigation:\n\n- `/Users/joe/Developer/BenchBox/benchmark_runs`: about 6.2G, mostly\n `datagen/`, containing SF1 data for TPCH variants, Vector Search, H2ODB,\n SSB, Nyctaxi, ClickBench, and related benchmarks.\n- `/Users/joe/Developer/BenchBox.pool-01/benchmark_runs`: about 4.2G,\n mostly `datagen/`, with large files including `tpch_skew_sf1/lineitem.tbl`,\n `vector_search_sf1/vectors.csv.zst`, `h2odb_sf1/trips.tbl.zst`,\n `ssb_sf1/lineorder.tbl.zst`, `nyctaxi_1.0/trips.csv.zst`, and\n `clickbench_sf1/hits.csv.zst`.\n\nCleanup must happen only after the root cause is fixed or protected, because\notherwise the next UAT/corpus run can recreate the same duplicate data.\nCleanup must also preserve any unique result bundles, manifests, or evidence\nthat are not redundant with the shared root under `~/Developer/benchmark_runs`.\n", "id": "benchmark-runs-duplicate-artifact-cleanup", "preserves": ["Cleanup commands are path-specific; no global filesystem prune or unbounded recursive deletion.", "Git-tracked files and pool worktree metadata are untouched.", "No deletion happens until `benchmark-runs-output-root-propagation` and `benchmark-runs-local-artifact-guardrails` are merged or explicitly confirmed active.", "Unique result JSON, manifests, lifecycle logs, and submission evidence are preserved before any datagen cleanup."], "prior_art": [{"concept": "benchmark_runs/ repository ignore conventions \u2014 preserve generated data out of git", "decision": "reuse", "path": "benchmark_runs/"}, {"concept": "docs/operations/uat-framework.md:Long-Running UAT \u2014 reuse shared-root conventions for durable UAT evidence", "decision": "reuse", "path": "docs/operations/uat-framework.md:Long-Running"}, {"concept": "tests/uat validation/submission tooling \u2014 use existing manifest/result validators before discarding any non-datagen evidence", "decision": "reuse", "path": "tests/uat"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "docs/operations/uat-framework.md"}], "state": "planning", "title": "Audit and clean duplicate worktree-local benchmark_runs artifacts", "verifications": [{"command": "find ~/Developer -maxdepth 2 -type d -name benchmark_runs -path '*BenchBox*' -print", "description": "Inventory report covers every BenchBox clone/pool worktree with a local benchmark_runs tree", "expected": "all candidate local benchmark_runs paths are listed and classified", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": null, "description": "Cleanup approval names exact paths and byte counts", "expected": "user approval explicitly covers each destructive path", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 2}, {"command": "du -sh /Users/joe/Developer/BenchBox/benchmark_runs /Users/joe/Developer/BenchBox.pool-*/benchmark_runs 2>/dev/null || true", "description": "Post-cleanup audit shows approved duplicate paths removed or reduced as expected", "expected": "approved duplicate datagen paths are absent or no longer contain redundant generated data", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 3}, {"command": "BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --queries Q1 --phases generate --non-interactive", "description": "External-root smoke does not recreate worktree-local generated data", "expected": "shared root receives expected output; active worktree-local benchmark_runs remains absent or unchanged", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": [], "notes": "Record the merged PR numbers or checked SHA(s). If either prerequisite is\nnot available, stop after producing an audit report and do not delete data.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Confirm root-cause fix and guardrails are merged or active before deleting anything", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w0"], "notes": "Produce a compact inventory with path, size, newest mtime, file count,\ntop large files, and whether each directory is datagen-only or contains\nresult bundles/manifests. Raw command output stays in `/tmp`; the durable\nsummary can go in a PR body, TODO note, or small markdown report.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Inventory repo and pool worktree `benchmark_runs` directories", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w1"], "notes": "Identify which generated data is redundant with the shared root and which\nevidence, if any, is unique. Preserve unique run results, manifests,\nlifecycle logs, and submission evidence before cleanup.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Compare worktree-local artifacts against `~/Developer/benchmark_runs`", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w2"], "notes": "Present the specific paths, byte counts, and preservation plan. Do not\ninfer approval from this TODO alone because deletion is intentionally\ndestructive.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Request explicit approval for the exact cleanup operation", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w3"], "notes": "Prefer removing redundant datagen-only directories. If unique evidence is\nfound, move it to the approved shared root or an agreed archive location\nbefore deleting the duplicate worktree-local tree.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Remove or archive only approved duplicate generated data", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w4"], "notes": "Run a lightweight external-root smoke after cleanup. Confirm the cleaned\nworktree-local path remains absent or unchanged and the shared root\nreceives any expected generated artifacts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify no new worktree-local artifacts appear after a representative external-root smoke", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "auto-delete unexpected artifacts", "why": "detection and reporting come first; cleanup is handled separately."}, {"INSTEAD": "(not specified)", "dont": "ban every `benchmark_runs/` directory globally", "why": "BenchBox still supports cwd-local output for explicit local/default runs."}, {"INSTEAD": "(not specified)", "dont": "make a broad disk scan mandatory for every unit test", "why": "keep expensive size checks in UAT/preflight paths."}, {"INSTEAD": "(not specified)", "dont": "rely on documentation alone", "why": "add executable checks that fail when the invariant is broken."}], "approach": "Add detection after the root-cause fix, not before. Start with a small\nregression test that proves external-root runs leave the worktree clean.\nThen add a UAT hygiene check that snapshots the worktree-local\n`benchmark_runs` path before a run and compares after each run or at teardown.\nDocument the invariant and wire enforcement only where an external root is\nconfigured, so default local usage remains compatible.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-01T00:00:00Z", "deferrals": [], "deps": ["benchmark-runs-output-root-propagation"], "description": "After `benchmark-runs-output-root-propagation` fixes the lifecycle bug,\nBenchBox needs regression guardrails so long UAT and corpus runs cannot\nsilently fill pool worktrees with duplicate generated data again.\n\nThe 2026-06-01 evidence showed the problematic shape clearly:\n\n- The corpus was launched from `/Users/joe/Developer/BenchBox.pool-01` with\n `BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs`.\n- Despite that, `/Users/joe/Developer/BenchBox.pool-01/benchmark_runs`\n accumulated about 4.2G of generated artifacts, mostly `datagen/`.\n- A similar stale/generated tree already existed in the primary clone under\n `/Users/joe/Developer/BenchBox/benchmark_runs`.\n\nThis TODO adds test and operational guardrails that treat unexpected\nworktree-local `benchmark_runs` writes as a failure when an external output\nroot is configured. The guardrails must be precise enough to avoid breaking\nordinary ad hoc runs that intentionally use the cwd-local default.", "id": "benchmark-runs-local-artifact-guardrails", "preserves": ["Guardrail failures are actionable: they print the unexpected local path and expected configured root.", "Long-running UAT continues to use `BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs` as the canonical shared root.", "No guardrail deletes or moves generated data automatically.", "Ordinary ad hoc runs with no `BENCHBOX_OUTPUT_DIR` and no explicit `--output` can still use cwd-local `benchmark_runs`."], "prior_art": [{"concept": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir \u2014 reuse the environment-aware root decision", "decision": "reuse", "path": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir"}, {"concept": "docs/operations/uat-framework.md:Long-Running UAT \u2014 extend the existing shared-root requirement instead of adding a competing run-root policy", "decision": "extend", "path": "docs/operations/uat-framework.md:Long-Running"}, {"concept": "make agent-write-preflight and make pr-preflight \u2014 reuse existing guard-command flow for repository hygiene checks", "decision": "reuse", "path": "make"}, {"concept": "tests/uat/runner.py \u2014 extend the runner that already sets `BENCHBOX_OUTPUT_DIR` and passes `--output`", "decision": "extend", "path": "tests/uat/runner.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/*/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/operations/uat-framework.md"}, {"kind": "only_modify", "path_glob": "tests/uat/"}, {"kind": "only_modify", "path_glob": "tests/uat/runner.py"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Add guardrails that fail UAT when worktree-local benchmark_runs artifacts appear", "verifications": [{"command": "uv run -- python -m pytest tests/uat/test_output_root_hygiene.py -q", "description": "External-root artifact hygiene regression passes", "expected": "test fails on unexpected cwd/benchmark_runs writes and passes when no local artifacts are created", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/uat/test_runner.py -q", "description": "UAT runner tests still pass", "expected": "all tests pass", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/test_runner_helpers.py tests/unit/cli/test_orchestrator_data_sharing.py tests/unit/utils/test_path_utils.py -q", "description": "Focused path tests from the root-cause TODO still pass", "expected": "all tests pass", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 3}, {"command": "make pr-preflight", "description": "PR preflight passes", "expected": "all preflight checks pass", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": [], "notes": "Confirm the fixed code no longer creates worktree-local datagen artifacts\nduring a representative external-root run. Record checked SHA, command,\nrun root, and before/after worktree-local path state in the PR body or\nTODO notes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the guardrail target after output-root propagation lands", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w0"], "notes": "The test should create an isolated cwd with a configured external root,\nrun a lightweight representative generation path, and fail if\n`cwd/benchmark_runs` exists or any nested generator path points there.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a targeted unit or integration guard for explicit/env output roots", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w1"], "notes": "When `BENCHBOX_OUTPUT_DIR` or UAT `--output` points outside the current\nworktree, capture the initial state of `cwd/benchmark_runs` and fail the\nrun if new files or size growth appear there. The failure message must\nname both the expected root and the unexpected local path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a UAT-level pre/post artifact hygiene check for external-root runs", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w2"], "notes": "Update operations docs with the invariant: long-running UAT writes under\n`~/Developer/benchmark_runs`, never under a clone or pool worktree. Include\ncompact audit commands for `du`, recent writes, and large-file summaries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the external-root invariant and emergency triage command", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w2"], "notes": "Prefer UAT/preflight-scoped enforcement over a global prohibition on\n`benchmark_runs/`. If adding to `make pr-preflight`, gate only unexpected\ngrowth or explicit external-root scenarios so developers can still run\nsmall local ad hoc benchmarks.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the hygiene check into the appropriate local gate without making ordinary local runs noisy", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add another output-root environment variable", "why": "BenchBox already has `BENCHBOX_OUTPUT_DIR` and path resolver helpers."}, {"INSTEAD": "(not specified)", "dont": "globally remove the cwd-local default for ordinary ad hoc runs unless no explicit/env output root exists", "why": "preserve backward compatibility."}, {"INSTEAD": "(not specified)", "dont": "special-case only the observed large directories", "why": "the constructor/setter contract must cover every generator-backed benchmark path found in w1."}, {"INSTEAD": "(not specified)", "dont": "treat deleting worktree-local `benchmark_runs` as the fix", "why": "the generation lifecycle will recreate the duplicate data."}, {"INSTEAD": "(not specified)", "dont": "write tests that only inspect `benchmark.output_dir`", "why": "assert nested generator output paths and absence of `cwd/benchmark_runs` references."}], "approach": "Start from the construction boundary, not from cleanup. Resolve the output\nroot using existing helpers before benchmark instances are created, then pass\nthe benchmark-specific datagen directory into constructors that support it.\nFor compatibility paths that must mutate `benchmark.output_dir` after\nconstruction, make the setter or local synchronization helper update nested\ngenerator state. Finish by adding regression tests that fail when an explicit\noutput root is ignored and `cwd/benchmark_runs` is touched.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-08T00:00:00Z", "completed_pr": null, "created_at": "2026-06-01T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox wrote duplicate generated benchmark data under worktree-local\n`benchmark_runs/` directories even when UAT was invoked with\n`BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs`.\n\nObserved impact from the 2026-06-01 investigation:\n\n- `/Users/joe/Developer/BenchBox/benchmark_runs` contained about 6.2G of\n generated artifacts, mostly `datagen/`, with large SF1 benchmark data.\n- `/Users/joe/Developer/BenchBox.pool-01/benchmark_runs` contained about\n 4.2G of similar generated artifacts written on 2026-06-01 while running\n the ClickHouse server corpus from branch `verify/clickhouse-server-corpus`.\n- The ClickHouse corpus command did set\n `BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs`, so the failure is not\n just missing environment configuration.\n\nThe root cause is lifecycle ordering. The CLI orchestrator constructs a\nbenchmark before the runner resolves the final output root. `BaseBenchmark`\ndefaults to `Path.cwd() / \"benchmark_runs\" / \"datagen\" / ...` when no\n`output_dir` is supplied, and several benchmark classes construct nested\n`data_generator` instances in `__init__` from that initial path. Later,\n`_resolve_output_dir_handler` mutates only `benchmark.output_dir`; nested\ngenerator paths can remain pointed at the worktree-local default. The MCP\nbenchmark path has the same pattern: it computes data paths after constructing\nbenchmark instances without an explicit output root.\n\nCorrect this at the architecture boundary so explicit CLI `--output` and\n`BENCHBOX_OUTPUT_DIR` are resolved before benchmark construction wherever\npossible, and ensure any post-construction output-dir mutation updates nested\ngenerators for benchmark families that support generation.", "id": "benchmark-runs-output-root-propagation", "preserves": ["CLI `--output` remains the highest-precedence output root.", "Existing benchmark-specific datagen subdirectory names and manifests remain compatible with reuse.", "No live cloud or Docker UAT is run as part of the unit-level fix without explicit approval.", "When neither CLI `--output` nor `BENCHBOX_OUTPUT_DIR` is set, ordinary local runs may still default to `Path.cwd()/benchmark_runs`.", "`BENCHBOX_OUTPUT_DIR` remains the shared-root mechanism for UAT and scripted runs."], "prior_art": [{"concept": "benchbox/cli/config.py:get_datagen_path \u2014 reuse existing per-benchmark datagen path conventions instead of inventing a parallel root", "decision": "reuse", "path": "benchbox/cli/config.py:get_datagen_path"}, {"concept": "benchbox/core/read_primitives/benchmark.py, benchbox/core/transactional/benchmark_base.py, benchbox/core/tpcds/benchmark/runner.py \u2014 reuse existing generator/output-dir synchronization patterns where present", "decision": "reuse", "path": "benchbox/core/read_primitives/benchmark.py"}, {"concept": "benchbox/core/runtime_paths.py:RuntimePaths.from_config \u2014 reuse the explicit > env > default precedence model", "decision": "reuse", "path": "benchbox/core/runtime_paths.py:RuntimePaths.from_config"}, {"concept": "benchbox/core/write_primitives/benchmark.py:output_dir setter behavior covered by tests/unit/core/write_primitives/test_write_primitives_benchmark_coverage.py \u2014 extend this synchronization contract to affected generated-data benchmarks", "decision": "extend", "path": "benchbox/core/write_primitives/benchmark.py:output_dir"}, {"concept": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir \u2014 reuse as the canonical environment-aware benchmark-runs root resolver", "decision": "reuse", "path": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir"}], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "docs/operations/uat-framework.md"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/*/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/*/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "only_modify", "path_glob": "benchbox/core/runtime_paths.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/path_utils.py"}, {"kind": "only_modify", "path_glob": "tests/uat/runner.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}, {"kind": "only_modify", "path_glob": "tests/unit/utils/"}], "state": "done", "title": "Propagate benchmark output roots before data generators are constructed", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_runner_helpers.py tests/unit/cli/test_orchestrator_data_sharing.py tests/unit/utils/test_path_utils.py tests/unit/utils/test_output_path_normalization.py -q", "description": "Focused path-resolution and runner/orchestrator tests pass", "expected": "all tests pass", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py tests/unit/mcp/test_benchmark_output_root_propagation.py -q", "description": "New regression tests prove nested generators honor explicit/env output roots", "expected": "representative generator-backed benchmarks use the configured root and do not reference cwd/benchmark_runs", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite still passes before PR", "expected": "all tests pass", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 3}, {"command": "make pr-preflight", "description": "PR preflight passes", "expected": "all preflight checks pass", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": [], "notes": "Capture a compact, committed summary in the PR body or TODO notes:\nchecked SHA, command or unit probe, whether an explicit output root still\nleaves a nested generator pointed at `Path.cwd()/benchmark_runs`, and the\naffected benchmark classes. Raw stdout belongs in `/tmp` or the configured\noutput root, not in git.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the current failure mode on a clean branch before editing", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w0"], "notes": "Include at least CLI orchestrator, runner helper resolution, MCP benchmark\ntools, UAT runner, and direct benchmark constructors used in tests. Record\nwhich paths can pass `output_dir` at construction time and which need a\nsynchronized setter for backward compatibility.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Map every benchmark construction path that can call generate_data", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w1"], "notes": "Prefer resolving the output root once at the call boundary and passing the\nbenchmark-specific datagen directory into the benchmark constructor.\nPreserve CLI `--output` precedence over `BENCHBOX_OUTPUT_DIR`, and preserve\nthe current cwd-local default only when neither explicit option nor env is\nset.\n\nResidual gap: explicit CLI `--output` override does not yet propagate through\nthe post-construction setter for tpcds and tpcdi (lazy/derived layout\nbenchmarks). This gap is tracked and scoped in the follow-up TODO\n`benchmark-output-root-explicit-override-propagation`; it does not block\nclosing this item because the env-root path (BENCHBOX_OUTPUT_DIR) is fully\ncorrect and the explicit-override gap has a dedicated tracking item.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pass the resolved output/datagen root into benchmark constructors before nested generators are created", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w1"], "notes": "Cover H2ODB, SSB, Vector Search, ClickBench, Nyctaxi, TPCH variants, and\nany other generator-backed benchmark found in w1. Use a local helper or\nbase-class pattern only if it reduces real duplication without flattening\nbenchmark-specific behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add or standardize output_dir synchronization for benchmark classes with nested data_generator state", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w1", "w2"], "notes": "`benchbox/mcp/tools/benchmark.py` must not instantiate generator-backed\nbenchmarks with stale cwd-local defaults before computing the data root.\nAdd focused tests or extend existing MCP coverage where practical.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the MCP benchmark path so generated data also honors explicit/env output roots", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w2", "w3", "w4"], "notes": "Include both constructor-time and post-construction paths. At minimum,\nprove that H2ODB, SSB, Vector Search, and one TPCH-family benchmark do not\ncreate or reference `cwd/benchmark_runs` when `BENCHBOX_OUTPUT_DIR` or\nan explicit CLI output root is set.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for explicit/env output roots across representative benchmark families", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "make MCP or explorer invent their own benchmark support tiers", "why": "they must consume registry metadata or documented helper output."}, {"INSTEAD": "(not specified)", "dont": "mark every benchmark stable just because it is runnable", "why": "support status should reflect docs, validation, data contract, and maintenance commitment."}, {"INSTEAD": "(not specified)", "dont": "use support_status as a replacement for surface", "why": "internal smoke benchmarks can remain runnable by explicit ID without becoming public discovery entries."}], "approach": "Follow the platform support-status pattern, but keep the benchmark-specific\npolicy honest: benchmark support depends on product guarantees, dataset\ncontract, validation coverage, and public discovery promises, not just on\nwhether the benchmark class imports.\n", "blocked_reason": null, "category": "Benchmark Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-and-discovery-policy", "id": 356, "reason": "Explorer UX changes should follow the metadata contract and public discovery decision.", "resolution": "promoted", "resolved_item": "use-benchmark-support-status-to-redesign-the-explorer-benchm", "resolved_reason": null, "summary": "Use benchmark support status to redesign the explorer benchmark browser"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-and-discovery-policy", "id": 357, "reason": "Support status is not a visibility or deprecation decision by itself.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: The item deliberately decoupled support-status from visibility (L108-110: do not hide/remove a public benchmark solely on support status; visibility changes need explicit per-benchmark surface/deprecation decision). No standing action; it's a per-case decision made at deprecation time.", "summary": "Remove or hide public benchmark facades based on support status"}], "deps": ["architecture-contract-map-and-support-taxonomy", "benchmark-api-and-core-boundary-cleanup"], "description": "The public contract map defines `support_status` for platforms and benchmarks.\nPlatform metadata already carried support status; this item closed the\nbenchmark half by adding benchmark-level support status to the benchmark\nmetadata source of truth, defining how it interacts with `surface` and\n`supports_dataframe`, and wiring the count/status claims through tested\ndiscovery paths.", "id": "benchmark-support-status-and-discovery-policy", "preserves": ["Do not create a second benchmark catalogue outside benchbox/core/benchmark_registry.py.", "Do not equate benchmark category with support_status without an explicit product decision.", "Do not hide, remove, or deprecate existing public benchmarks solely as a side effect of adding metadata.", "Do not treat supports_dataframe as support_status; it is a capability flag."], "prior_art": [{"concept": "benchbox/core/benchmark_registry.py: current benchmark metadata and discovery source of truth -- extend in place.", "decision": "extend", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "benchbox/core/platform_registry.py: platform support_status validation and summary helpers -- reuse the pattern, not the platform-specific metadata.", "decision": "reuse", "path": "benchbox/core/platform_registry.py"}, {"concept": "docs/reference/public-contracts.md: public support taxonomy and drift-check ownership -- extend rather than creating separate status language.", "decision": "extend", "path": "docs/reference/public-contracts.md"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: existing wrapper/loader/registry count drift tests -- extend with support-status assertions.", "decision": "extend", "path": "tests/unit/core/test_benchmark_api_contract.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/contract.py"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/schemas.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/discovery.py"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/"}, {"kind": "only_modify", "path_glob": "docs/reference/mcp.md"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_discovery_tools.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_tool_handlers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_resources.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_run_surface_contract.py"}], "state": "done", "title": "Architecture hardening: add benchmark support status and discovery policy", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/mcp -k 'benchmark or discovery or support_status' --tb=short", "description": "Run benchmark API and discovery contract tests after metadata migration.", "expected": "Benchmark support-status, count, and discovery tests pass or skip only for missing optional MCP SDK.", "item_id": "benchmark-support-status-and-discovery-policy", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline -k 'benchmark or read_model' --tb=short", "description": "Run explorer pipeline tests if benchmark status is exported to explorer inputs.", "expected": "Explorer benchmark label/read-model tests pass.", "item_id": "benchmark-support-status-and-discovery-policy", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "Validate TODO graph and schema after adding or completing this item.", "expected": "TODO validation and graph checks pass.", "item_id": "benchmark-support-status-and-discovery-policy", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": [], "notes": "Completed 2026-05-21 at checked SHA\nd8102e8fd7c6a216f5af18fd1d0c1c55db1a4c69.\n\nRevalidation summary:\n - `benchbox/core/benchmark_registry.py` had 23 metadata entries, 23 loader\n IDs, 22 public discovery IDs, 21 top-level facades, and one internal\n benchmark (`joinorder_synthetic`).\n - MCP discovery and resources consumed registry metadata but did not expose\n support status.\n - Explorer pipeline code did not maintain an independent benchmark support\n catalogue; no explorer code change was needed to prevent classification\n drift.\n - `docs/reference/public-contracts.md` still described benchmark\n `support_status` as pending.\n\nCapture a compact summary of checked SHA, current benchmark counts,\ndiscovery surfaces, docs claims, and any line-number drift. Do not commit\nraw command transcripts.\n\nMinimum evidence to re-check:\n - `benchbox/core/benchmark_registry.py` metadata, `surface`, and list helpers.\n - `benchbox/__init__.py` wrapper facade exports.\n - `benchbox/core/benchmark_loader.py` loader reachability.\n - MCP benchmark discovery tools and docs.\n - Explorer pipeline benchmark labels and any \"supported benchmark\" copy.\n - `docs/reference/public-contracts.md` count and support taxonomy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate benchmark discovery and support-claim evidence before editing", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w0"], "notes": "Completed with the shared taxonomy values: stable, beta, experimental,\nrepo_only, deprecated, and document_only.\n\nCurrent product decision:\n - stable: tpch, tpcds, ssb, clickbench, joinorder.\n - beta: tpcdi, h2odb, amplab, read_primitives, write_primitives,\n metadata_primitives, transaction_primitives, coffeeshop, tsbs_devops,\n nyctaxi, flightdata, vector_search.\n - experimental: tpcds_obt, ai_primitives, tpchavoc, tpch_skew, datavault.\n - repo_only: joinorder_synthetic.\n - deprecated/document_only: none.\n\nReuse the public taxonomy values: stable, beta, experimental, repo_only,\ndeprecated, and document_only. Spell out how they apply to benchmark\nfamilies, data availability, official-vs-development scale factors, and\nDataFrame support.\n\nKeep `support_status` distinct from:\n - `surface`, which controls discovery visibility.\n - `supports_dataframe`, which is a capability flag.\n - dataset availability or local optional dependency availability.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define benchmark-specific support_status semantics", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w1"], "notes": "Completed by adding `support_status` to every `BENCHMARK_METADATA` entry,\nvalidating at import time, and adding registry helpers:\n`get_benchmark_support_status()`, `get_benchmarks_by_support_status()`, and\n`get_benchmark_registry_summary()`.\n\nEvery `BENCHMARK_METADATA` entry should have exactly one valid\n`support_status`. Add helper accessors or summary functions only where\ndiscovery/docs/tests need them; avoid introducing a second benchmark\ncatalogue.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add support_status metadata and validation to benchmark_registry", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w2"], "notes": "Completed for MCP discovery/resources and docs. `list_available`,\n`get_benchmark_info`, benchmark resources, and query-detail benchmark info\nnow project registry `support_status`. CLI display was left unchanged because\nit does not make support-level claims, and explorer code was left unchanged\nbecause it has no separate benchmark support catalogue.\n\nUpdate CLI, MCP, explorer, and docs only where they make support-level\nclaims. Do not hide or remove an existing public benchmark solely because\nits support status is clarified; visibility changes need an explicit\nsurface/deprecation decision.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Propagate benchmark support status to discovery surfaces", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w2"], "notes": "Completed by extending benchmark API contract tests and MCP discovery/resource\ntests. The contract map and benchmark docs now state support-status counts\nfrom registry metadata instead of treating benchmark status as pending.\n\nExtend benchmark API contract coverage so registry count, wrapper count,\nloader count, public discovery count, and support-status summary claims\ncannot drift silently. Exact user-facing count tables should derive from\nbenchmark registry metadata or be marked editorial.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add drift checks for benchmark status and count claims", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "file follow-up TODOs for capability gaps.", "dont": "change DataFrame capability flags as part of criteria documentation", "why": "capability and support status are separate contracts"}, {"INSTEAD": "document only status rationale and evidence pointers.", "dont": "duplicate the entire BENCHMARK_METADATA table in docs", "why": "that creates a second catalog"}, {"INSTEAD": "require evidence rows.", "dont": "mark a benchmark stable only because it imports or runs at smoke scale", "why": "stable implies docs, validation, and maintenance commitment"}, {"INSTEAD": "record criteria evidence instead.", "dont": "use benchmark category as status rationale", "why": "category is workload taxonomy, not product support"}], "approach": "Start from the current public contract taxonomy, then add benchmark-specific\nevidence dimensions. Use the registry to enumerate benchmark IDs and statuses.\nCross-check docs and result-integrity spec coverage, then write a compact\nmatrix that explains current status values without becoming a second runtime\ncatalog. Add a drift check only after choosing a matrix format that can be\nparsed reliably.\n", "blocked_reason": null, "category": "Benchmark Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-25T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-criteria-matrix", "id": 358, "reason": "It is the only beta whose sole integrity-dimension gap is a missing result-integrity spec; adding one (plus a DataFrame routing decision) is the concrete blocker recorded in docs/benchmarks/support-status.md.", "resolution": "promoted", "resolved_item": "promote-vector-search-beta-stable-once-it-has-a-benchmarkspe", "resolved_reason": null, "summary": "Promote vector_search beta -> stable once it has a BenchmarkSpec"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-criteria-matrix", "id": 359, "reason": "Their status is partly coupled to the benchbox.core.base_benchmark.BaseBenchmark migration; revisit when that base is gone.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Verified benchbox/core/base_benchmark.py still exists; deprecated base has not been removed, so the reassessment trigger has not occurred.", "summary": "Reassess tpcds_obt and datavault experimental status after the deprecated core base is removed"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-criteria-matrix", "id": 360, "reason": "External-dataset betas are gated on a pinned source contract, not on query coverage.", "resolution": "promoted", "resolved_item": "pin-reproducible-external-source-contracts-for-nyctaxi-and-f", "resolved_reason": null, "summary": "Pin reproducible external-source contracts for nyctaxi and flightdata before any stable promotion"}], "deps": ["benchmark-support-status-and-discovery-policy"], "description": "PR #502 added registry-owned `support_status` for every benchmark, but the\nfirst assignment is still a product decision encoded as metadata rather than\nan auditable criteria matrix. The current counts are test-protected, but the\nrepository does not yet explain why each benchmark is stable, beta,\nexperimental, repo_only, deprecated, or document_only.\n\nBuild a benchmark support-status criteria matrix that makes future promotions\nand demotions reviewable. The matrix should evaluate each benchmark against\nconcrete product criteria: user docs coverage, result-integrity spec coverage,\ndataset/source contract, validation coverage, public discovery behavior,\nDataFrame coverage where claimed, known unsupported platform gaps, and\nmaintenance commitment. The matrix should not duplicate the registry as a\nsecond source of truth; it should document the rationale and acceptance\nthresholds for changing the registry-owned status field.", "id": "benchmark-support-status-criteria-matrix", "preserves": ["Benchmark surface visibility remains independent of support_status.", "Current public benchmark discovery count remains unchanged unless a separate surface decision changes it.", "Runtime support_status values remain registry-owned in benchbox/core/benchmark_registry.py.", "supports_dataframe remains a capability flag, not a proxy for support status."], "prior_art": [{"concept": "benchbox/core/benchmark_registry.py: BENCHMARK_METADATA support_status values - reuse as runtime source of truth.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "docs/development/result-integrity-validation.md: integrity spec coverage list - use as one evidence dimension, not as the whole support decision.", "decision": "reuse", "path": "docs/development/result-integrity-validation.md"}, {"concept": "docs/platforms/support-status.md: platform support-status rationale style - reuse the explanatory pattern, but adapt criteria to benchmark-specific evidence.", "decision": "reuse", "path": "docs/platforms/support-status.md"}, {"concept": "docs/reference/public-contracts.md: support taxonomy and count policy - extend criteria language rather than creating new status terms.", "decision": "extend", "path": "docs/reference/public-contracts.md"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: benchmark count and support-status drift tests - extend for matrix row coverage.", "decision": "extend", "path": "tests/unit/core/test_benchmark_api_contract.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/"}, {"kind": "only_modify", "path_glob": "docs/development/result-integrity-validation.md"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}], "state": "done", "title": "Define auditable benchmark support-status promotion criteria", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py -q", "description": "Benchmark registry support-status contract remains green.", "expected": "All benchmark API contract tests pass.", "item_id": "benchmark-support-status-criteria-matrix", "last_result": null, "last_run": null, "seq": 1}, {"command": "make docs-validate", "description": "Docs validate after adding the criteria matrix.", "expected": "Documentation validation passes.", "item_id": "benchmark-support-status-criteria-matrix", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph and schema remain valid if follow-up TODOs are added.", "expected": "TODO validation and graph checks pass.", "item_id": "benchmark-support-status-criteria-matrix", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": [], "notes": "Capture the checked SHA, current status counts, benchmark IDs, public IDs,\nand result-integrity spec coverage in the TODO or implementation PR body.\nDo not rely on the PR #502 review text alone; the criteria matrix must be\nbased on the current tree at implementation time.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-read the benchmark registry, public contract map, benchmark docs, and result-integrity spec list before proposing criteria", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w0"], "notes": "Write criteria for stable, beta, experimental, repo_only, deprecated, and\ndocument_only. Include benchmark-specific dimensions that platform status\ndid not need: official-vs-development scale factors, generated-vs-external\ndatasets, canonical query coverage, result validation availability, docs\ndepth, and whether DataFrame support is claimed for that benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define status-specific acceptance criteria for benchmark families", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w1"], "notes": "Produce one row per benchmark with the current registry status, rationale,\nevidence pointers, and any promotion/demotion blockers. The matrix may live\nin a developer/reference doc, but the runtime status remains in\n`benchbox/core/benchmark_registry.py`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Score every registry benchmark against the criteria and record status rationale", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w2"], "notes": "Add a focused test that fails when a benchmark status is added, removed, or\nchanged without a corresponding criteria-matrix row or explicit rationale\nupdate. Prefer a simple parseable markdown table or YAML/JSON summary if\nthe existing docs test patterns can check it cleanly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests or docs checks that keep the matrix synchronized with registry status values", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w2"], "notes": "If the matrix shows that a beta benchmark is close to stable or that an\nexperimental benchmark has unclear criteria, record follow-up TODOs or\ndeferred entries with exact blockers. Do not change support_status values\nin this TODO unless the criteria evidence makes the change unambiguous.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record concrete promotion candidates and blockers as deferred work, not as silent comments", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "assert labels and fields.", "dont": "build tests that depend on rich color codes", "why": "styling is incidental"}, {"INSTEAD": "omit or filter claims on explicit-ID surfaces.", "dont": "expose repo_only or document_only support claims over MCP discovery", "why": "the public taxonomy says those are not MCP-exposed"}, {"INSTEAD": "use explicit surface decisions for hiding.", "dont": "hide all experimental benchmarks by changing support_status behavior alone", "why": "visibility is controlled by surface"}, {"INSTEAD": "assert it separately.", "dont": "use supports_dataframe to infer support_status", "why": "DataFrame support is a capability dimension"}], "approach": "Treat `surface` as visibility, `support_status` as product support level, and\n`supports_dataframe` as capability. First write the policy table so tests have\na clear target. Then add metadata-only fixture tests for future statuses and\nfocused real-registry tests for current public beta/experimental and internal\nrepo_only benchmarks. Keep explicit-ID behavior stable unless the policy table\nexplicitly changes it.\n", "blocked_reason": null, "category": "Benchmark Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-25T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [], "deps": ["benchmark-support-status-and-discovery-policy"], "description": "PR #502 added benchmark support status and exposed it on MCP discovery. The\nimmediate review follow-up fixed CLI labeling for public benchmark listings\nand prevented MCP query-detail support-status leakage for internal/repo-only\nbenchmarks. The broader invariant still needs a durable test matrix: future\nrepo_only, document_only, deprecated, beta, and experimental benchmark entries\nshould fail loudly if a public surface exposes or hides them contrary to the\nsupport taxonomy and `surface` policy.\n\nAdd cross-surface invariant tests and policy documentation that prove\n`support_status`, `surface`, and `supports_dataframe` remain separate. The\ngoal is not to change visibility for current benchmarks. The goal is to make\nfuture status additions safe and reviewable across CLI, MCP discovery, MCP\nquery details, docs, and result/explorer inputs where applicable.", "id": "benchmark-support-status-visibility-invariants", "preserves": ["MCP discovery continues to expose support_status for public benchmarks.", "Public beta and experimental benchmarks remain visible unless a separate surface/deprecation decision changes them.", "joinorder_synthetic remains hidden from public discovery lists.", "joinorder_synthetic remains runnable by explicit ID where it was already supported.", "supports_dataframe continues to control DataFrame routing independently of support_status."], "prior_art": [{"concept": "benchbox/cli/benchmarks.py: BenchmarkManager public benchmark filtering - extend tests without changing explicit-ID run behavior.", "decision": "extend", "path": "benchbox/cli/benchmarks.py"}, {"concept": "benchbox/core/benchmark_registry.py:get_benchmark_support_status and get_benchmark_registry_summary - reuse for status assertions.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py:get_benchmark_support_status"}, {"concept": "benchbox/core/benchmark_registry.py:get_benchmark_surface and list_public_benchmark_ids - reuse for discovery visibility decisions.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py:get_benchmark_surface"}, {"concept": "benchbox/mcp/tools/benchmark.py: get_query_details benchmark_info - preserve explicit-ID compatibility while preventing unsupported support-status claims.", "decision": "reuse", "path": "benchbox/mcp/tools/benchmark.py"}, {"concept": "benchbox/mcp/tools/discovery.py: public benchmark discovery filtering - reuse as MCP listing contract.", "decision": "reuse", "path": "benchbox/mcp/tools/discovery.py"}, {"concept": "tests/unit/core/test_registry_surface_field.py: patch.dict surface tests - reuse fixture pattern for synthetic status/surface combinations.", "decision": "reuse", "path": "tests/unit/core/test_registry_surface_field.py"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/benchmarks.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/discovery.py"}, {"kind": "only_modify", "path_glob": "docs/reference/mcp.md"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_manager_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_selection.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_registry_surface_field.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_discovery_tools.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_tool_handlers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_resources.py"}], "state": "done", "title": "Harden benchmark support-status visibility invariants across public surfaces", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/core/test_registry_surface_field.py -q", "description": "Benchmark registry and surface invariants pass.", "expected": "All benchmark registry and surface tests pass.", "item_id": "benchmark-support-status-visibility-invariants", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/test_benchmark_manager_behavioral.py tests/unit/cli/test_benchmark_selection.py -q", "description": "CLI benchmark manager support-status labeling tests pass.", "expected": "All selected CLI benchmark manager tests pass.", "item_id": "benchmark-support-status-visibility-invariants", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/mcp/test_discovery_tools.py tests/unit/mcp/test_resources.py tests/unit/mcp/test_mcp_tool_handlers.py -q", "description": "MCP discovery/resource/query-detail support-status tests pass.", "expected": "All selected MCP tests pass.", "item_id": "benchmark-support-status-visibility-invariants", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": [], "notes": "Re-check CLI interactive listing, CLI explicit benchmark execution, MCP\nlist_available, MCP get_benchmark_info, MCP get_query_details, resources,\nresult publisher discovery, explorer ingestion, and result-integrity docs.\nRecord which surfaces are discovery surfaces and which are explicit-ID\nexecution or inspection surfaces.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory current public, internal, and explicit-ID benchmark surfaces", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w0"], "notes": "Specify expected behavior for stable, beta, experimental, repo_only,\ndeprecated, and document_only when `surface` is public or internal. Include\nwhether each should be listed by default, labeled, hidden, runnable by\nexplicit ID, exposed over MCP discovery, exposed over MCP query details,\nand eligible for DataFrame routing if `supports_dataframe` is true.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write a visibility policy table for support_status x surface x capability", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w1"], "notes": "Use patch.dict or small helper fixtures to create temporary benchmark\nmetadata entries for repo_only, document_only, deprecated, beta, and\nexperimental statuses. Assert discovery hiding, public labeling, and\nexplicit-ID behavior according to the policy. Avoid importing heavy\nbenchmark classes when metadata-only behavior is sufficient.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add synthetic invariant tests for future repo_only and document_only benchmark metadata", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w1"], "notes": "Keep the immediate CLI tests from the review fix, then broaden them to\ncover public beta and experimental cases in list trees, selection tables,\npreview panels, MCP list_available, MCP get_benchmark_info, and benchmark\nresources. Tests should check contract shape, not rich color styling.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend CLI and MCP tests so non-stable public benchmarks are visibly labeled", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w1", "w2"], "notes": "If explicit-ID surfaces can inspect internal benchmarks, document what\nmetadata may be returned and which support claims must be omitted. Keep the\ndocumentation aligned with the public contract map and MCP docs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document explicit-ID exceptions and support-status exposure policy", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w2", "w3", "w4"], "notes": "Run the narrowest tests for benchmark registry, CLI benchmark manager, MCP\ndiscovery/resources/tool handlers, and any result/explorer path touched by\nthe invariant work. Escalate to fast tests before PR submission.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run focused contract and MCP/CLI tests after invariant coverage lands", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add individual CLI flags per benchmark parameter (--taxi-types, --year, --end-year)", "instead": "use the generic --benchmark-option K=V mechanism instead", "why": "this creates N*M option explosion across 21 benchmarks and 30+ existing CLI options"}, {"dont": "merge benchmark-option parsing into PlatformHookRegistry", "instead": "create a parallel BenchmarkHookRegistry", "why": "platform and benchmark options have different lifecycles (platform options go to DatabaseConfig, benchmark options go to constructor kwargs)"}, {"dont": "pass raw BenchmarkConfig.options dict as **kwargs to constructors", "instead": "only forward keys that have registered BenchmarkOptionSpecs", "why": "options contains orchestrator-level settings (tuning_enabled, validation flags, estimated_time_range) that would cause TypeError on benchmark constructors"}, {"dont": "use Python inspect/reflection to auto-discover constructor params", "instead": "use explicit spec registration per benchmark (same as platform-option)", "why": "it's fragile across **kwargs, defaults, and type annotations"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-04T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-option-cli-support", "id": 351, "reason": "Tab-completion for --benchmark-option keys and choices would improve DX but requires Click shell completion integration. Not needed for initial release.\n", "resolution": "promoted", "resolved_item": "add-benchmark-option-shell-completion", "resolved_reason": null, "summary": "Add benchmark-option shell completion"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-option-cli-support", "id": 352, "reason": "Could catch mismatches between registered spec names and actual constructor param names. Low risk since specs are co-located with benchmark code and tested. Adds import-time overhead.\n", "resolution": "promoted", "resolved_item": "validate-benchmark-options-against-constructor-signature-at", "resolved_reason": null, "summary": "Validate benchmark options against constructor signature at registration time"}], "deps": [], "description": "Benchmark constructors accept parameters that are unreachable from the CLI. The orchestrator (_get_benchmark_instance) only forwards a fixed set of kwargs (scale_factor, compress_data, compression_type, compression_level, verbose, quiet, parallel). Benchmark-specific parameters like taxi_types, year, months, end_year, num_hosts, skew_preset, dimensions, hash_algorithm, seed, and force_regenerate never reach the constructor.\nThis affects 10 of 21 benchmarks. The fix is a --benchmark-option K=V mechanism mirroring the existing --platform-option K=V pattern. Each benchmark registers typed option specs; the orchestrator merges parsed options into constructor kwargs.", "id": "benchmark-option-cli-support", "preserves": ["--platform-option continues working independently", "All existing benchmark default behavior unchanged when no --benchmark-option provided", "BenchmarkConfig.options continues serving its existing purposes (tuning config, validation flags, power run settings)", "Benchmarks with only standard params (tpch, tpcds, ssb, clickbench, h2odb, etc.) unaffected", "seed and force_regenerate continue working in their current RunConfig/orchestrator paths - benchmark-option adds a parallel path to the constructor"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/*/downloader.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/*/queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/*/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/benchmark_hooks.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/datavault/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/flightdata/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds_obt/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch_skew/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tsbs_devops/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/vector_search/benchmark.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_hooks.py"}], "state": "done", "title": "Add --benchmark-option K=V CLI support for benchmark-specific parameters", "verifications": [{"command": "benchbox run --platform duckdb --benchmark nyctaxi --scale 0.01 --benchmark-option taxi_types=yellow,green,hvfhv --dry-run\n", "description": "NYC Taxi benchmark accepts taxi_types via CLI", "expected": "dry run shows 3 taxi types, 39 queries", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 1}, {"command": "benchbox run --platform duckdb --benchmark tpch --scale 0.01 --benchmark-option nonexistent=value --dry-run\n", "description": "Unknown benchmark option rejected with clear error", "expected": "error message listing available options (or none)", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -m fast -q", "description": "All existing tests pass (no regression)", "expected": "all pass, 0 failures", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/cli/test_benchmark_hooks.py -v", "description": "Benchmark hook registry unit tests pass", "expected": "all pass", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 4}, {"command": "benchbox run --help-topic benchmarks", "description": "Help-topic shows registered benchmark options", "expected": "output includes taxi_types, year, months for nyctaxi and skew_preset for tpch_skew", "item_id": "benchmark-option-cli-support", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "benchmark-option-cli-support", "needs": [], "notes": "Mirror platform_hooks.py structure: - BenchmarkOptionError (ValueError subclass) - BenchmarkOptionSpec dataclass (name, parser, default, help, choices, aliases) - List/enum parser functions: parse_int_list, parse_str_list, parse_enum_list factory - BenchmarkHookRegistry class: register_option_specs(benchmark, *specs),\n parse_options(benchmark, provided), describe_options(benchmark), get_default_options(benchmark)\n- Reuse existing parse_bool and _identity from platform_hooks or extract to shared utils\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create BenchmarkHookRegistry and BenchmarkOptionSpec in benchbox/cli/benchmark_hooks.py", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w1"], "notes": "In benchbox/cli/commands/run.py: - Add BenchmarkOptionParamType (same pattern as PlatformOptionParamType) - Add @click.option(\"--benchmark-option\", ..., type=BenchmarkOptionParamType(),\n multiple=True, hidden=True) - parallel to --platform-option\n- Parse benchmark options via BenchmarkHookRegistry.parse_options(benchmark, pairs) - Store parsed options in BenchmarkConfig.options under a namespaced key\n (e.g., options[\"benchmark_options\"] = {...}) to avoid collision with existing options\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --benchmark-option Click option to run.py and wire parsing", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w2"], "notes": "In benchbox/cli/orchestrator.py _get_benchmark_instance(): - Extract benchmark_options from config.options.get(\"benchmark_options\", {}) - Merge into kwargs dict before calling benchmark_class(**kwargs) - seed and force_regenerate: these already live in BenchmarkConfig.options\n but are NOT forwarded to constructors today. The orchestrator should check\n if the current benchmark has registered BenchmarkOptionSpecs for \"seed\" or\n \"force_regenerate\" (via BenchmarkHookRegistry.list_option_specs(benchmark)),\n and if so, include them in kwargs from config.options. Benchmarks that don't\n register these specs simply never receive them (no change). This avoids a\n dual-path where seed flows through both config.options and --benchmark-option.\n- Preserve existing try/except TypeError fallback for parallel param\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire orchestrator to forward benchmark options to constructor kwargs", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w1"], "notes": "These benchmarks require the custom parser functions built in w1 (parse_enum_list, parse_int_list, parse_datetime). This is the highest-risk work unit - validate parser edge cases (empty lists, single values, whitespace) here before w5.\nNYC Taxi (benchbox/core/nyctaxi/benchmark.py): - taxi_types: parse_enum_list(TaxiType), default=None, choices description - year: int parser, default=2019, choices=range(2019,2026) - months: parse_int_list, default=None - seed: int parser, default=None\nTSBS DevOps (benchbox/core/tsbs_devops/benchmark.py): - num_hosts: int parser, default=None - duration_days: int parser, default=None - interval_seconds: int parser, default=10 - start_time: parse_datetime, default=None\nRegistration via module-level BenchmarkHookRegistry.register_option_specs() calls (same pattern as platform option registration in benchbox/platforms/__init__.py).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register option specs for high-complexity benchmarks (NYC Taxi, TSBS DevOps - enum lists, datetime, int lists)", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w1"], "notes": "These benchmarks use only str, int, parse_bool, parse_str_list, and choices-constrained str parsers - all straightforward. Lower risk than w4.\nFlightData: end_year (int, default=LAST_AVAILABLE_YEAR) DataVault: hash_algorithm (str, choices=[\"md5\",\"sha256\",...]), record_source (str) TPC-DS OBT: tpcds_source_dir (str), dimension_mode (str, choices), channels (parse_str_list), output_format (str, choices) TPC-H Skew: skew_preset (str, choices=[\"none\",\"light\",\"moderate\",\"heavy\",\"extreme\",\"realistic\"]) Vector Search: dimensions (int, default=128) Join Order: queries_dir (str) TPC-DI: enable_parallel (parse_bool, default=False), max_workers (int)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register option specs for simple-type benchmarks (FlightData, DataVault, TPC-DS OBT, TPC-H Skew, Vector Search, Join Order, TPC-DI)", "wid": "w5"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w3", "w4"], "notes": "tests/unit/cli/test_benchmark_hooks.py: - Registry: register, parse, describe, defaults, unknown key rejection, duplicate rejection - Parsers: parse_int_list, parse_str_list, parse_enum_list (TaxiType), parse_datetime - Integration: BenchmarkOptionParamType Click type, end-to-end from CLI string to constructor kwarg - Regression: benchmarks without registered specs still construct normally\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for BenchmarkHookRegistry, parsers, and end-to-end wiring", "wid": "w6"}, {"evidence": null, "item_id": "benchmark-option-cli-support", "needs": ["w4", "w5"], "notes": "Extend --help-topic system (benchbox/cli/help.py) to show available benchmark options per benchmark. Example output for --help-topic benchmarks or --help-topic nyctaxi:\n nyctaxi:\n taxi_types - Taxi data types to load (yellow,green,hvfhv) [default: yellow]\n year - Year of TLC data (2019-2025) [default: 2019]\n months - Months to include, comma-separated (1-12) [default: all]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --help-topic documentation for benchmark options", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "blanket-sync every generator attribute", "instead": "only sync generators that should follow the OBT/primary output root.", "why": "tpcds_obt/datavault intentionally diverge"}, {"dont": "duplicate ETL path math in the tpcdi setter", "instead": "delegate to the config helpers.", "why": "TPCDIConfig already owns it"}], "approach": "Prefer reusing GeneratorOutputDirMixin where the generator is a plain instance\nattribute (tpcds). For tpcdi, whose layout is config-driven and richer than a\nsingle generator, add a focused property setter that re-derives the ETL\ndirectories rather than shoehorning it into the mixin. Land tests first that\nfail on the current tree.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-11T00:00:00Z", "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [], "deps": ["benchmark-runs-output-root-propagation"], "description": "PR #759 made benchmark datagen roots honor BENCHBOX_OUTPUT_DIR at construction\nand added GeneratorOutputDirMixin so a post-construction output_dir\nreassignment propagates to nested generators. Several generator-backed\nbenchmarks (h2odb, ssb, tpch, nyctaxi, etc.) opted into the mixin; a few did\nnot because their generators are lazy or intentionally point elsewhere.\n\nTwo benchmarks build a generator (or derive directories) in __init__ but have\nno synchronizing output_dir setter, so an explicit CLI --output root that\ndiffers from the construction default does NOT reach them:\n\n- tpcds: builds its data_generator in __init__ and has no output_dir setter, so\n a post-construction reassignment (runner _resolve_output_dir_handler) leaves\n the generator pointed at the construction-time path.\n- tpcdi: derives source_dir/staging_dir/warehouse_dir and builds its\n data_generator in __init__ from self.config.output_dir; a later output_dir\n reassignment updates neither the ETL subdirs nor the generator.\n\nThis is a pre-existing residual gap, NOT a regression. The reported failure\nmode (BENCHBOX_OUTPUT_DIR ignored) is already fixed for both: their\nconstruction-time defaults resolve via the env-aware helper (tpcds via its\ndefault path, tpcdi via TPCDIConfig.get_benchmark_runs_datagen_path), and in\nthe managed/env flow the runner's reassignment target equals the construction\ndefault, so nothing diverges. Only an explicit `--output /custom` that differs\nfrom the default exposes the gap.\n\nClose it so an explicit override propagates correctly for these two\nbenchmarks, matching the contract the mixin already provides elsewhere. NOTE:\nbenchmark-output-root-resolve-at-construction-boundary proposes a deeper fix\n(resolve the root before construction) that would subsume this item; this is\nthe tactical fix if that refactor is deferred.", "id": "benchmark-output-root-explicit-override-propagation", "preserves": ["Env/managed-flow behavior for tpcds and tpcdi remains unchanged (construction default already env-aware).", "tpcdi data reuse: existing source/staging/warehouse layout names remain compatible.", "tpcds_obt and datavault lazy generators that intentionally write to a separate source dir are NOT force-synced."], "prior_art": [{"concept": "benchbox/base.py:GeneratorOutputDirMixin \u2014 reuse/extend; tpcds can adopt it directly (its data_generator is a plain instance attribute)", "decision": "extend", "path": "benchbox/base.py:GeneratorOutputDirMixin"}, {"concept": "benchbox/core/read_primitives/benchmark.py + benchbox/core/transactional/benchmark_base.py \u2014 reuse the existing output_dir setter pattern that re-derives nested state", "decision": "reuse", "path": "benchbox/core/read_primitives/benchmark.py"}, {"concept": "benchbox/core/runner/runner.py:_resolve_output_dir_handler \u2014 the post-construction assignment site this fix must satisfy", "decision": "reuse", "path": "benchbox/core/runner/runner.py:_resolve_output_dir_handler"}, {"concept": "benchbox/core/tpcdi/config.py:TPCDIConfig (output_dir + create_directories) \u2014 reuse for re-deriving source/staging/warehouse on reassignment instead of duplicating path math", "decision": "reuse", "path": "benchbox/core/tpcdi/config.py:TPCDIConfig"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/datavault/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds_obt/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}], "state": "done", "title": "Propagate explicit --output override to nested generators for tpcds and tpcdi", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py -q", "description": "Explicit override now propagates for tpcds and tpcdi", "expected": "tpcds and tpcdi override cases pass (previously xfail)", "item_id": "benchmark-output-root-explicit-override-propagation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpcds tests/unit/core/tpcdi -q", "description": "tpcds and tpcdi unit suites still pass", "expected": "all tests pass", "item_id": "benchmark-output-root-explicit-override-propagation", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": [], "notes": "Construct each, then assign a distinct explicit output_dir and assert the\nnested generator / ETL subdirs do NOT follow today. Capture the probe to\n/tmp/override-gap-w0.log (do not commit) and record results in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm the override gap reproduces for tpcds and tpcdi on current develop", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": ["w0"], "notes": "tpcds builds data_generator as a plain attribute in __init__, so the mixin\napplies cleanly. Verify the tpcds_obt source-dir behavior is unaffected\n(its generator is lazy and intentionally writes to a separate dir).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Adopt GeneratorOutputDirMixin for tpcds so its data_generator tracks output_dir", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": ["w0"], "notes": "On reassignment, update source_dir/staging_dir/warehouse_dir and\ndata_generator.output_dir, keeping self.config.output_dir consistent. Guard\nagainst running before config exists during __init__.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a tpcdi output_dir setter that re-derives ETL subdirs and syncs the generator", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-output-root-explicit-override-propagation", "needs": ["w1", "w2"], "notes": "Coordinate with benchmark-output-root-regression-guard, which marks these\ntwo xfail; this work removes the xfail once propagation is fixed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Flip the tpcds/tpcdi xfail markers in the regression guard to passing", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "assert only benchmark.output_dir", "instead": "assert the nested generator/downloader paths too.", "why": "the original bug lived in nested generators"}, {"dont": "hand-list benchmark ids", "instead": "enumerate via get_all_benchmarks() instead.", "why": "the point is to auto-cover future benchmarks"}], "approach": "Build the guard on top of the registry so it auto-covers new benchmarks. Reuse\nthe helper functions already in the propagation test for resolving nested\ngenerators. Prefer xfail with an explicit reason over skipping for the known\ntpcds/tpcdi override gap, so the tracker shows the residual work.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-11T00:00:00Z", "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-output-root-regression-guard", "id": 353, "reason": "Cleanup of existing on-disk artifacts belongs to benchmark-runs-duplicate-artifact-cleanup, not this regression-guard item.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Cleanup of orphaned non-canonical datagen dirs is explicitly assigned to benchmark-runs-duplicate-artifact-cleanup, which is open.", "summary": "Clean up orphaned non-canonical datagen dirs (e.g. nyctaxi_1.0) left by direct library users before PR #759 canonicalized the names"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-output-root-regression-guard", "id": 354, "reason": "That filesystem-at-runtime assertion is owned by benchmark-runs-local-artifact-guardrails; this item is the unit-test + static-source counterpart (construction contract and hardcoded-default lint).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Runtime guardrail explicitly owned by benchmark-runs-local-artifact-guardrails, which is already DONE (_project/DONE/main/planning/); no open work remains here.", "summary": "Runtime/UAT guardrail that fails a run when worktree-local benchmark_runs writes appear despite an external output root"}], "deps": ["benchmark-runs-output-root-propagation"], "description": "PR #759 fixed the lifecycle-ordering bug where generator-backed benchmarks\nignored BENCHBOX_OUTPUT_DIR and wrote datagen output under a worktree-local\nbenchmark_runs/. The fix made BaseBenchmark's default datagen root env-aware\nand added an opt-in GeneratorOutputDirMixin that propagates a resolved root to\nnested generators. The accompanying tests\n(tests/unit/core/test_benchmark_output_root_propagation.py,\ntests/unit/mcp/test_benchmark_output_root_propagation.py) only assert a\nhand-picked subset of benchmark families.\n\nThe footgun is structural: a new generator-backed benchmark that builds its\ngenerator in __init__ and forgets the mixin (or hardcodes\nPath.cwd()/\"benchmark_runs\") silently reintroduces the original bug, and no\ntest or lint catches it. A one-off manual probe across all 24 registered\nbenchmarks confirmed the contract currently holds, but that evidence is not\ndurable.\n\nConvert that probe into a registry-driven regression guard so the contract is\nenforced for every current and future benchmark, and add a cheap static guard\nthat fails when a benchmark constructor hardcodes a cwd-local benchmark_runs\ndefault instead of the shared resolver.", "id": "benchmark-output-root-regression-guard", "preserves": ["The existing per-family assertions in test_benchmark_output_root_propagation.py keep passing (extend, do not delete).", "tests remain in the fast lane and write nothing under the active worktree."], "prior_art": [{"concept": "_project/TODO/main/planning/benchmark-runs-local-artifact-guardrails.yaml \u2014 align with the runtime guardrail effort; this item is the test/lint counterpart", "decision": "reuse", "path": "_project/TODO/main/planning/benchmark-runs-local-artifact-guardrails.yaml"}, {"concept": "benchbox/base.py:GeneratorOutputDirMixin (OUTPUT_DIR_GENERATOR_ATTRS) \u2014 reuse to discover the nested-generator attributes to assert against", "decision": "reuse", "path": "benchbox/base.py:GeneratorOutputDirMixin"}, {"concept": "benchbox/core/benchmark_registry.py:get_all_benchmarks / get_public_benchmark_class \u2014 reuse to enumerate every benchmark under test", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py:get_all_benchmarks"}, {"concept": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path / resolve_benchmark_runs_dir \u2014 reuse as the canonical env-aware root for assertions", "decision": "reuse", "path": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path"}, {"concept": "tests/unit/core/test_benchmark_output_root_propagation.py \u2014 extend the per-family assertions into a registry-parametrized form rather than adding a parallel test file", "decision": "extend", "path": "tests/unit/core/test_benchmark_output_root_propagation.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/base.py"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": ".github/workflows/lint.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}], "state": "done", "title": "Registry-wide regression guard for benchmark output-root propagation", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py tests/unit/mcp/test_benchmark_output_root_propagation.py -q", "description": "New registry-wide propagation tests pass", "expected": "all tests pass; every registered benchmark is parametrized", "item_id": "benchmark-output-root-regression-guard", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_guard.py -q", "description": "Static guard fails on a deliberately hardcoded cwd default", "expected": "guard passes on current tree and fails when a benchmark hardcodes Path.cwd()/benchmark_runs", "item_id": "benchmark-output-root-regression-guard", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite still passes", "expected": "no new failures attributable to this change", "item_id": "benchmark-output-root-regression-guard", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": [], "notes": "Construct each id from get_all_benchmarks() with BENCHBOX_OUTPUT_DIR set to\na tmp path and assert output_dir plus any OUTPUT_DIR_GENERATOR_ATTRS\ngenerator resolves under it. Capture stdout to /tmp/output-root-guard-w0.log\n(do not commit) and record the pass count and any exceptions in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate that every registered benchmark currently honors the env output root", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w0"], "notes": "Parametrize over get_all_benchmarks(). For each, construct via the public\nclass (mirroring the MCP/library entry point) with BENCHBOX_OUTPUT_DIR set;\nassert output_dir and each present nested generator/downloader resolve under\nthe root and that Path.cwd() is not a parent. Resolve nested generators via\n_impl when the public wrapper delegates.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a registry-parametrized test asserting env-root honoring at construction for all benchmarks", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w1"], "notes": "For benchmarks that build a generator in __init__, assign a fresh explicit\nroot after construction and assert the nested generator follows. Mark the\nknown non-mixin benchmarks (tpcds, tpcdi) xfail and reference\nbenchmark-output-root-explicit-override-propagation so the gap is tracked,\nnot hidden.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a test asserting post-construction output_dir reassignment propagates to nested generators", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w0"], "notes": "A unit test (or ruff/grep-based check wired into ci-lint) that scans\nregistered benchmark class source files for Path.cwd()/\"benchmark_runs\"\ndefaults and fails with guidance to use get_benchmark_runs_datagen_path\ninstead. Allow a documented exception list only for intentional non-datagen\npaths.\n\nScan target must cover ALL registered benchmark sources, not only files named\nbenchmark.py. For example, TPCDSBenchmark lives in\nbenchbox/core/tpcds/benchmark/runner.py, not benchmark.py. Derive scan\ntargets from the registered class __module__ paths (via get_all_benchmarks())\nrather than a glob like benchbox/core/**/benchmark.py so future benchmarks\nadded in non-standard file names are automatically covered.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a static guard that fails on hardcoded cwd-local benchmark_runs defaults in constructors", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-output-root-regression-guard", "needs": ["w1"], "notes": "Assert that a CloudStagingPath/DatabricksPath assigned as the output root\npropagates to the nested generator without being re-wrapped or losing cloud\nsemantics. Use a mocked/staging path so no live cloud access is required.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend coverage to a cloud/dbfs output root through at least one generator-backed benchmark", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add a third resolution helper", "instead": "resolve through it at the boundary.", "why": "get_benchmark_runs_datagen_path already exists"}, {"dont": "remove the output_dir setter compatibility path outright", "instead": "keep a thin shim.", "why": "external/library callers assign it post-construction"}], "approach": "Treat this as an architecture-altitude change: prefer generalizing the\nresolve-once-at-boundary mechanism over layering more per-class setters. Land\nit behind the regression guard from benchmark-output-root-regression-guard so\nany behavior drift is caught. Sequence: prove the data-source root is\nresolvable pre-construction, inject at the boundary, then simplify the now-\nredundant mutation/setters.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-14T00:00:00Z", "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-output-root-resolve-at-construction-boundary", "id": 355, "reason": "Cloud roots resolve later via credentials in _apply_default_cloud_output_dir; needs its own analysis once the local-path boundary is settled.", "resolution": "promoted", "resolved_item": "cloud-staging-path-cloudstagingpath-databrickspath-construct", "resolved_reason": null, "summary": "Cloud staging path (CloudStagingPath/DatabricksPath) construction-time injection edge cases"}], "deps": ["benchmark-output-root-regression-guard"], "description": "PR #759 fixed the output-root propagation bug with two cooperating mechanisms:\nan env-aware default in BaseBenchmark, and an opt-in GeneratorOutputDirMixin\nwhose setter re-syncs nested generators when output_dir is reassigned after\nconstruction. This works, but it leaves the original architectural smell that\nthe TODO's w2 explicitly preferred to fix: the orchestrator still constructs\nthe benchmark BEFORE the final output root is resolved, then mutates\nbenchmark.output_dir afterward (orchestrator data-sharing redirect, runner\n_resolve_output_dir_handler).\n\nConsequences of keeping post-construction mutation as the contract:\n\n- Footgun: any new generator-backed benchmark that builds its generator in\n __init__ and forgets the mixin silently reintroduces the bug. The mixin is\n opt-in, so correctness depends on author memory.\n- Duplication: GeneratorOutputDirMixin now overlaps with the bespoke output_dir\n setters in read_primitives/benchmark.py and transactional/benchmark_base.py.\n- Residual gaps: benchmarks with lazy/derived layouts (tpcds, tpcdi) need the\n separate explicit-override fix tracked in\n benchmark-output-root-explicit-override-propagation.\n\nInvestigate resolving the benchmark-specific datagen directory once at the call\nboundary and injecting it into the constructor, so generators capture the\ncorrect root immediately and post-construction mutation (and the mixin) can be\nretired or reduced to a thin compatibility shim. If adopted, this would subsume\nthe explicit-override item and let the duplicated setters collapse onto one\nmechanism.", "id": "benchmark-output-root-resolve-at-construction-boundary", "preserves": ["BENCHBOX_OUTPUT_DIR remains the shared-root mechanism; cwd-local default preserved when neither is set.", "CLI --output remains the highest-precedence output root.", "Data-sharing benchmarks (read/write/transaction primitives) keep reusing the source benchmark's datagen dir.", "External callers that still assign benchmark.output_dir after construction continue to work (compatibility shim)."], "prior_art": [{"concept": "benchbox/base.py:GeneratorOutputDirMixin and benchbox/core/read_primitives + transactional output_dir setters \u2014 candidates to dedupe once mutation is removed", "decision": "reuse", "path": "benchbox/base.py:GeneratorOutputDirMixin"}, {"concept": "benchbox/cli/orchestrator.py:_get_benchmark_instance / _resolve_output_root / _resolve_custom_output_root \u2014 the construction + resolution sites to reorder", "decision": "reuse", "path": "benchbox/cli/orchestrator.py:_get_benchmark_instance"}, {"concept": "benchbox/core/benchmark_loader.py:instantiate_benchmark_class / constructor_accepts_argument \u2014 reuse to inject output_dir only where the constructor accepts it", "decision": "reuse", "path": "benchbox/core/benchmark_loader.py:instantiate_benchmark_class"}, {"concept": "benchbox/core/runner/runner.py:_resolve_output_dir_handler \u2014 the post-construction mutation this item aims to retire or shrink", "decision": "reuse", "path": "benchbox/core/runner/runner.py:_resolve_output_dir_handler"}, {"concept": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path \u2014 reuse as the single resolver at the boundary", "decision": "reuse", "path": "benchbox/utils/path_utils.py:get_benchmark_runs_datagen_path"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "only_modify", "path_glob": "benchbox/core/transactional/benchmark_base.py"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Resolve benchmark datagen root at the construction boundary to remove post-construction mutation", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py tests/unit/mcp/test_benchmark_output_root_propagation.py -q", "description": "Output-root regression guard passes after the refactor", "expected": "all propagation tests pass with no xfails remaining for generator-backed benchmarks", "item_id": "benchmark-output-root-resolve-at-construction-boundary", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/test_orchestrator_data_sharing.py tests/unit/core/test_runner_helpers.py -q", "description": "Orchestrator/runner and data-sharing suites pass", "expected": "all tests pass", "item_id": "benchmark-output-root-resolve-at-construction-boundary", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite passes", "expected": "no new failures", "item_id": "benchmark-output-root-resolve-at-construction-boundary", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": [], "notes": "Enumerate orchestrator, runner, MCP, UAT runner, and direct test\nconstructors. For each, record whether the final root is knowable before\nconstruction and what currently blocks passing it in (e.g. data-sharing\nsource resolution needing an instance).\nDONE: the two mutation sites were orchestrator._get_benchmark_instance\n(data-sharing redirect) and runner._resolve_output_dir_handler. The final\nlocal root is knowable before construction once the data-source name is a\nclass attribute; only cloud --output roots stay deferred (need platform\nconfig), handled by the post-construction shim.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Map every site that constructs a benchmark then mutates output_dir", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": ["w0"], "notes": "get_data_source_benchmark() is currently an instance method, which forces\nconstruct-then-redirect for read/write/transaction primitives. Evaluate a\nclass-level/registry lookup so the shared datagen dir is resolvable before\nconstruction.\nDONE: added BaseBenchmark.DATA_SOURCE_BENCHMARK class attribute. The\ninstance method now falls back to it, and read/transaction/write\nprimitives set it instead of overriding the method, so the shared root is\nresolvable from the class alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Resolve the data-source root without a constructed instance", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": ["w1"], "notes": "Pass output_dir through instantiate_benchmark_class where accepted. Preserve\nCLI --output precedence over BENCHBOX_OUTPUT_DIR and the cwd fallback when\nneither is set.\nDONE: orchestrator._resolve_construction_output_dir resolves the root\n(CLI local --output > shared data-source root > managed per-benchmark\npath) and injects it via optional_kwargs. Cloud --output returns None and\ndefers to the shim. BENCHBOX_OUTPUT_DIR / cwd fallback still apply via the\nBaseBenchmark default when no root is injected.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inject the resolved datagen dir into constructors at the orchestrator/runner boundary", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-output-root-resolve-at-construction-boundary", "needs": ["w2"], "notes": "Once construction receives the final root, reduce _resolve_output_dir_handler\nto a no-op/compat shim and collapse GeneratorOutputDirMixin and the\nread_primitives/transactional setters onto a single mechanism, keeping a\ncompatibility path for external callers that still assign output_dir.\nDONE: _resolve_output_dir_handler is now a shim that only assigns when the\nresolved handler differs from the live one (no redundant re-wrap on the\nconstruction-injected path; still applies cloud handlers and external\noutput_root). read_primitives and transactional benchmark_base dropped\ntheir bespoke setters in favor of GeneratorOutputDirMixin; the mixin\nremains the single propagation mechanism and the public output_dir setter\nstays as the compatibility path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire or shrink post-construction mutation and dedupe the output_dir setters", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "clean before the output-root bug is fixed or guarded", "instead": "(not specified)", "why": "that only frees space temporarily."}, {"dont": "delete result bundles or manifests just because they live under a duplicate root", "instead": "(not specified)", "why": "classify and preserve unique evidence first."}, {"dont": "run `rm -rf` from a parent directory or with a glob", "instead": "(not specified)", "why": "cleanup must list exact approved paths."}, {"dont": "use Docker/system prune as part of this task", "instead": "(not specified)", "why": "the target is BenchBox-generated filesystem data only."}], "approach": "Treat cleanup as a controlled operations task. First prove the root cause is\nfixed or guarded. Then inventory all repo and pool worktree local\n`benchmark_runs` trees, classify datagen-only duplicates versus durable\nresult evidence, compare against the shared root, and ask for explicit\napproval with exact paths and byte counts before deletion. Finish with a\nrepresentative external-root smoke to prove cleanup does not immediately\nregress.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-01T00:00:00Z", "deferrals": [], "deps": ["benchmark-runs-local-artifact-guardrails", "benchmark-runs-output-root-propagation"], "description": "Once output-root propagation and artifact guardrails are in place, audit and\nclean the already-written duplicate generated data from repo and pool\nworktrees.\n\nKnown duplicate/generated locations from the 2026-06-01 investigation:\n\n- `/Users/joe/Developer/BenchBox/benchmark_runs`: about 6.2G, mostly\n `datagen/`, containing SF1 data for TPCH variants, Vector Search, H2ODB,\n SSB, Nyctaxi, ClickBench, and related benchmarks.\n- `/Users/joe/Developer/BenchBox.pool-01/benchmark_runs`: about 4.2G,\n mostly `datagen/`, with large files including `tpch_skew_sf1/lineitem.tbl`,\n `vector_search_sf1/vectors.csv.zst`, `h2odb_sf1/trips.tbl.zst`,\n `ssb_sf1/lineorder.tbl.zst`, `nyctaxi_1.0/trips.csv.zst`, and\n `clickbench_sf1/hits.csv.zst`.\n\nCleanup must happen only after the root cause is fixed or protected, because\notherwise the next UAT/corpus run can recreate the same duplicate data.\nCleanup must also preserve any unique result bundles, manifests, or evidence\nthat are not redundant with the shared root under `~/Developer/benchmark_runs`.\n", "id": "benchmark-runs-duplicate-artifact-cleanup", "preserves": ["Cleanup commands are path-specific; no global filesystem prune or unbounded recursive deletion.", "Git-tracked files and pool worktree metadata are untouched.", "No deletion happens until `benchmark-runs-output-root-propagation` and `benchmark-runs-local-artifact-guardrails` are merged or explicitly confirmed active.", "Unique result JSON, manifests, lifecycle logs, and submission evidence are preserved before any datagen cleanup."], "prior_art": [{"concept": "benchmark_runs/ repository ignore conventions \u2014 preserve generated data out of git", "decision": "reuse", "path": "benchmark_runs/"}, {"concept": "docs/operations/uat-framework.md:Long-Running UAT \u2014 reuse shared-root conventions for durable UAT evidence", "decision": "reuse", "path": "docs/operations/uat-framework.md:Long-Running"}, {"concept": "tests/uat validation/submission tooling \u2014 use existing manifest/result validators before discarding any non-datagen evidence", "decision": "reuse", "path": "tests/uat"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "docs/operations/uat-framework.md"}], "state": "planning", "title": "Audit and clean duplicate worktree-local benchmark_runs artifacts", "verifications": [{"command": "find ~/Developer -maxdepth 2 -type d -name benchmark_runs -path '*BenchBox*' -print", "description": "Inventory report covers every BenchBox clone/pool worktree with a local benchmark_runs tree", "expected": "all candidate local benchmark_runs paths are listed and classified", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": null, "description": "Cleanup approval names exact paths and byte counts", "expected": "user approval explicitly covers each destructive path", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 2}, {"command": "du -sh /Users/joe/Developer/BenchBox/benchmark_runs /Users/joe/Developer/BenchBox.pool-*/benchmark_runs 2>/dev/null || true", "description": "Post-cleanup audit shows approved duplicate paths removed or reduced as expected", "expected": "approved duplicate datagen paths are absent or no longer contain redundant generated data", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 3}, {"command": "BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --queries Q1 --phases generate --non-interactive", "description": "External-root smoke does not recreate worktree-local generated data", "expected": "shared root receives expected output; active worktree-local benchmark_runs remains absent or unchanged", "item_id": "benchmark-runs-duplicate-artifact-cleanup", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": [], "notes": "Record the merged PR numbers or checked SHA(s). If either prerequisite is\nnot available, stop after producing an audit report and do not delete data.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Confirm root-cause fix and guardrails are merged or active before deleting anything", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w0"], "notes": "Produce a compact inventory with path, size, newest mtime, file count,\ntop large files, and whether each directory is datagen-only or contains\nresult bundles/manifests. Raw command output stays in `/tmp`; the durable\nsummary can go in a PR body, TODO note, or small markdown report.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Inventory repo and pool worktree `benchmark_runs` directories", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w1"], "notes": "Identify which generated data is redundant with the shared root and which\nevidence, if any, is unique. Preserve unique run results, manifests,\nlifecycle logs, and submission evidence before cleanup.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Compare worktree-local artifacts against `~/Developer/benchmark_runs`", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w2"], "notes": "Present the specific paths, byte counts, and preservation plan. Do not\ninfer approval from this TODO alone because deletion is intentionally\ndestructive.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Request explicit approval for the exact cleanup operation", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w3"], "notes": "Prefer removing redundant datagen-only directories. If unique evidence is\nfound, move it to the approved shared root or an agreed archive location\nbefore deleting the duplicate worktree-local tree.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Remove or archive only approved duplicate generated data", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-runs-duplicate-artifact-cleanup", "needs": ["w4"], "notes": "Run a lightweight external-root smoke after cleanup. Confirm the cleaned\nworktree-local path remains absent or unchanged and the shared root\nreceives any expected generated artifacts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify no new worktree-local artifacts appear after a representative external-root smoke", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "auto-delete unexpected artifacts", "instead": "(not specified)", "why": "detection and reporting come first; cleanup is handled separately."}, {"dont": "ban every `benchmark_runs/` directory globally", "instead": "(not specified)", "why": "BenchBox still supports cwd-local output for explicit local/default runs."}, {"dont": "make a broad disk scan mandatory for every unit test", "instead": "(not specified)", "why": "keep expensive size checks in UAT/preflight paths."}, {"dont": "rely on documentation alone", "instead": "(not specified)", "why": "add executable checks that fail when the invariant is broken."}], "approach": "Add detection after the root-cause fix, not before. Start with a small\nregression test that proves external-root runs leave the worktree clean.\nThen add a UAT hygiene check that snapshots the worktree-local\n`benchmark_runs` path before a run and compares after each run or at teardown.\nDocument the invariant and wire enforcement only where an external root is\nconfigured, so default local usage remains compatible.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-01T00:00:00Z", "deferrals": [], "deps": ["benchmark-runs-output-root-propagation"], "description": "After `benchmark-runs-output-root-propagation` fixes the lifecycle bug,\nBenchBox needs regression guardrails so long UAT and corpus runs cannot\nsilently fill pool worktrees with duplicate generated data again.\n\nThe 2026-06-01 evidence showed the problematic shape clearly:\n\n- The corpus was launched from `/Users/joe/Developer/BenchBox.pool-01` with\n `BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs`.\n- Despite that, `/Users/joe/Developer/BenchBox.pool-01/benchmark_runs`\n accumulated about 4.2G of generated artifacts, mostly `datagen/`.\n- A similar stale/generated tree already existed in the primary clone under\n `/Users/joe/Developer/BenchBox/benchmark_runs`.\n\nThis TODO adds test and operational guardrails that treat unexpected\nworktree-local `benchmark_runs` writes as a failure when an external output\nroot is configured. The guardrails must be precise enough to avoid breaking\nordinary ad hoc runs that intentionally use the cwd-local default.", "id": "benchmark-runs-local-artifact-guardrails", "preserves": ["Guardrail failures are actionable: they print the unexpected local path and expected configured root.", "Long-running UAT continues to use `BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs` as the canonical shared root.", "No guardrail deletes or moves generated data automatically.", "Ordinary ad hoc runs with no `BENCHBOX_OUTPUT_DIR` and no explicit `--output` can still use cwd-local `benchmark_runs`."], "prior_art": [{"concept": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir \u2014 reuse the environment-aware root decision", "decision": "reuse", "path": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir"}, {"concept": "docs/operations/uat-framework.md:Long-Running UAT \u2014 extend the existing shared-root requirement instead of adding a competing run-root policy", "decision": "extend", "path": "docs/operations/uat-framework.md:Long-Running"}, {"concept": "make agent-write-preflight and make pr-preflight \u2014 reuse existing guard-command flow for repository hygiene checks", "decision": "reuse", "path": "make"}, {"concept": "tests/uat/runner.py \u2014 extend the runner that already sets `BENCHBOX_OUTPUT_DIR` and passes `--output`", "decision": "extend", "path": "tests/uat/runner.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/*/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "docs/operations/uat-framework.md"}, {"kind": "only_modify", "path_glob": "tests/uat/"}, {"kind": "only_modify", "path_glob": "tests/uat/runner.py"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Add guardrails that fail UAT when worktree-local benchmark_runs artifacts appear", "verifications": [{"command": "uv run -- python -m pytest tests/uat/test_output_root_hygiene.py -q", "description": "External-root artifact hygiene regression passes", "expected": "test fails on unexpected cwd/benchmark_runs writes and passes when no local artifacts are created", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/uat/test_runner.py -q", "description": "UAT runner tests still pass", "expected": "all tests pass", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/test_runner_helpers.py tests/unit/cli/test_orchestrator_data_sharing.py tests/unit/utils/test_path_utils.py -q", "description": "Focused path tests from the root-cause TODO still pass", "expected": "all tests pass", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 3}, {"command": "make pr-preflight", "description": "PR preflight passes", "expected": "all preflight checks pass", "item_id": "benchmark-runs-local-artifact-guardrails", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": [], "notes": "Confirm the fixed code no longer creates worktree-local datagen artifacts\nduring a representative external-root run. Record checked SHA, command,\nrun root, and before/after worktree-local path state in the PR body or\nTODO notes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the guardrail target after output-root propagation lands", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w0"], "notes": "The test should create an isolated cwd with a configured external root,\nrun a lightweight representative generation path, and fail if\n`cwd/benchmark_runs` exists or any nested generator path points there.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a targeted unit or integration guard for explicit/env output roots", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w1"], "notes": "When `BENCHBOX_OUTPUT_DIR` or UAT `--output` points outside the current\nworktree, capture the initial state of `cwd/benchmark_runs` and fail the\nrun if new files or size growth appear there. The failure message must\nname both the expected root and the unexpected local path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a UAT-level pre/post artifact hygiene check for external-root runs", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w2"], "notes": "Update operations docs with the invariant: long-running UAT writes under\n`~/Developer/benchmark_runs`, never under a clone or pool worktree. Include\ncompact audit commands for `du`, recent writes, and large-file summaries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the external-root invariant and emergency triage command", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-runs-local-artifact-guardrails", "needs": ["w2"], "notes": "Prefer UAT/preflight-scoped enforcement over a global prohibition on\n`benchmark_runs/`. If adding to `make pr-preflight`, gate only unexpected\ngrowth or explicit external-root scenarios so developers can still run\nsmall local ad hoc benchmarks.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the hygiene check into the appropriate local gate without making ordinary local runs noisy", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add another output-root environment variable", "instead": "(not specified)", "why": "BenchBox already has `BENCHBOX_OUTPUT_DIR` and path resolver helpers."}, {"dont": "globally remove the cwd-local default for ordinary ad hoc runs unless no explicit/env output root exists", "instead": "(not specified)", "why": "preserve backward compatibility."}, {"dont": "special-case only the observed large directories", "instead": "(not specified)", "why": "the constructor/setter contract must cover every generator-backed benchmark path found in w1."}, {"dont": "treat deleting worktree-local `benchmark_runs` as the fix", "instead": "(not specified)", "why": "the generation lifecycle will recreate the duplicate data."}, {"dont": "write tests that only inspect `benchmark.output_dir`", "instead": "(not specified)", "why": "assert nested generator output paths and absence of `cwd/benchmark_runs` references."}], "approach": "Start from the construction boundary, not from cleanup. Resolve the output\nroot using existing helpers before benchmark instances are created, then pass\nthe benchmark-specific datagen directory into constructors that support it.\nFor compatibility paths that must mutate `benchmark.output_dir` after\nconstruction, make the setter or local synchronization helper update nested\ngenerator state. Finish by adding regression tests that fail when an explicit\noutput root is ignored and `cwd/benchmark_runs` is touched.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-08T00:00:00Z", "completed_pr": null, "created_at": "2026-06-01T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox wrote duplicate generated benchmark data under worktree-local\n`benchmark_runs/` directories even when UAT was invoked with\n`BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs`.\n\nObserved impact from the 2026-06-01 investigation:\n\n- `/Users/joe/Developer/BenchBox/benchmark_runs` contained about 6.2G of\n generated artifacts, mostly `datagen/`, with large SF1 benchmark data.\n- `/Users/joe/Developer/BenchBox.pool-01/benchmark_runs` contained about\n 4.2G of similar generated artifacts written on 2026-06-01 while running\n the ClickHouse server corpus from branch `verify/clickhouse-server-corpus`.\n- The ClickHouse corpus command did set\n `BENCHBOX_OUTPUT_DIR=~/Developer/benchmark_runs`, so the failure is not\n just missing environment configuration.\n\nThe root cause is lifecycle ordering. The CLI orchestrator constructs a\nbenchmark before the runner resolves the final output root. `BaseBenchmark`\ndefaults to `Path.cwd() / \"benchmark_runs\" / \"datagen\" / ...` when no\n`output_dir` is supplied, and several benchmark classes construct nested\n`data_generator` instances in `__init__` from that initial path. Later,\n`_resolve_output_dir_handler` mutates only `benchmark.output_dir`; nested\ngenerator paths can remain pointed at the worktree-local default. The MCP\nbenchmark path has the same pattern: it computes data paths after constructing\nbenchmark instances without an explicit output root.\n\nCorrect this at the architecture boundary so explicit CLI `--output` and\n`BENCHBOX_OUTPUT_DIR` are resolved before benchmark construction wherever\npossible, and ensure any post-construction output-dir mutation updates nested\ngenerators for benchmark families that support generation.", "id": "benchmark-runs-output-root-propagation", "preserves": ["CLI `--output` remains the highest-precedence output root.", "Existing benchmark-specific datagen subdirectory names and manifests remain compatible with reuse.", "No live cloud or Docker UAT is run as part of the unit-level fix without explicit approval.", "When neither CLI `--output` nor `BENCHBOX_OUTPUT_DIR` is set, ordinary local runs may still default to `Path.cwd()/benchmark_runs`.", "`BENCHBOX_OUTPUT_DIR` remains the shared-root mechanism for UAT and scripted runs."], "prior_art": [{"concept": "benchbox/cli/config.py:get_datagen_path \u2014 reuse existing per-benchmark datagen path conventions instead of inventing a parallel root", "decision": "reuse", "path": "benchbox/cli/config.py:get_datagen_path"}, {"concept": "benchbox/core/read_primitives/benchmark.py, benchbox/core/transactional/benchmark_base.py, benchbox/core/tpcds/benchmark/runner.py \u2014 reuse existing generator/output-dir synchronization patterns where present", "decision": "reuse", "path": "benchbox/core/read_primitives/benchmark.py"}, {"concept": "benchbox/core/runtime_paths.py:RuntimePaths.from_config \u2014 reuse the explicit > env > default precedence model", "decision": "reuse", "path": "benchbox/core/runtime_paths.py:RuntimePaths.from_config"}, {"concept": "benchbox/core/write_primitives/benchmark.py:output_dir setter behavior covered by tests/unit/core/write_primitives/test_write_primitives_benchmark_coverage.py \u2014 extend this synchronization contract to affected generated-data benchmarks", "decision": "extend", "path": "benchbox/core/write_primitives/benchmark.py:output_dir"}, {"concept": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir \u2014 reuse as the canonical environment-aware benchmark-runs root resolver", "decision": "reuse", "path": "benchbox/utils/path_utils.py:resolve_benchmark_runs_dir"}], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "docs/operations/uat-framework.md"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/*/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/*/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "only_modify", "path_glob": "benchbox/core/runtime_paths.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/path_utils.py"}, {"kind": "only_modify", "path_glob": "tests/uat/runner.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}, {"kind": "only_modify", "path_glob": "tests/unit/utils/"}], "state": "done", "title": "Propagate benchmark output roots before data generators are constructed", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_runner_helpers.py tests/unit/cli/test_orchestrator_data_sharing.py tests/unit/utils/test_path_utils.py tests/unit/utils/test_output_path_normalization.py -q", "description": "Focused path-resolution and runner/orchestrator tests pass", "expected": "all tests pass", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_output_root_propagation.py tests/unit/mcp/test_benchmark_output_root_propagation.py -q", "description": "New regression tests prove nested generators honor explicit/env output roots", "expected": "representative generator-backed benchmarks use the configured root and do not reference cwd/benchmark_runs", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite still passes before PR", "expected": "all tests pass", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 3}, {"command": "make pr-preflight", "description": "PR preflight passes", "expected": "all preflight checks pass", "item_id": "benchmark-runs-output-root-propagation", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": [], "notes": "Capture a compact, committed summary in the PR body or TODO notes:\nchecked SHA, command or unit probe, whether an explicit output root still\nleaves a nested generator pointed at `Path.cwd()/benchmark_runs`, and the\naffected benchmark classes. Raw stdout belongs in `/tmp` or the configured\noutput root, not in git.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the current failure mode on a clean branch before editing", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w0"], "notes": "Include at least CLI orchestrator, runner helper resolution, MCP benchmark\ntools, UAT runner, and direct benchmark constructors used in tests. Record\nwhich paths can pass `output_dir` at construction time and which need a\nsynchronized setter for backward compatibility.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Map every benchmark construction path that can call generate_data", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w1"], "notes": "Prefer resolving the output root once at the call boundary and passing the\nbenchmark-specific datagen directory into the benchmark constructor.\nPreserve CLI `--output` precedence over `BENCHBOX_OUTPUT_DIR`, and preserve\nthe current cwd-local default only when neither explicit option nor env is\nset.\n\nResidual gap: explicit CLI `--output` override does not yet propagate through\nthe post-construction setter for tpcds and tpcdi (lazy/derived layout\nbenchmarks). This gap is tracked and scoped in the follow-up TODO\n`benchmark-output-root-explicit-override-propagation`; it does not block\nclosing this item because the env-root path (BENCHBOX_OUTPUT_DIR) is fully\ncorrect and the explicit-override gap has a dedicated tracking item.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pass the resolved output/datagen root into benchmark constructors before nested generators are created", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w1"], "notes": "Cover H2ODB, SSB, Vector Search, ClickBench, Nyctaxi, TPCH variants, and\nany other generator-backed benchmark found in w1. Use a local helper or\nbase-class pattern only if it reduces real duplication without flattening\nbenchmark-specific behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add or standardize output_dir synchronization for benchmark classes with nested data_generator state", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w1", "w2"], "notes": "`benchbox/mcp/tools/benchmark.py` must not instantiate generator-backed\nbenchmarks with stale cwd-local defaults before computing the data root.\nAdd focused tests or extend existing MCP coverage where practical.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the MCP benchmark path so generated data also honors explicit/env output roots", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-runs-output-root-propagation", "needs": ["w2", "w3", "w4"], "notes": "Include both constructor-time and post-construction paths. At minimum,\nprove that H2ODB, SSB, Vector Search, and one TPCH-family benchmark do not\ncreate or reference `cwd/benchmark_runs` when `BENCHBOX_OUTPUT_DIR` or\nan explicit CLI output root is set.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for explicit/env output roots across representative benchmark families", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "make MCP or explorer invent their own benchmark support tiers", "instead": "(not specified)", "why": "they must consume registry metadata or documented helper output."}, {"dont": "mark every benchmark stable just because it is runnable", "instead": "(not specified)", "why": "support status should reflect docs, validation, data contract, and maintenance commitment."}, {"dont": "use support_status as a replacement for surface", "instead": "(not specified)", "why": "internal smoke benchmarks can remain runnable by explicit ID without becoming public discovery entries."}], "approach": "Follow the platform support-status pattern, but keep the benchmark-specific\npolicy honest: benchmark support depends on product guarantees, dataset\ncontract, validation coverage, and public discovery promises, not just on\nwhether the benchmark class imports.\n", "blocked_reason": null, "category": "Benchmark Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-and-discovery-policy", "id": 356, "reason": "Explorer UX changes should follow the metadata contract and public discovery decision.", "resolution": "promoted", "resolved_item": "use-benchmark-support-status-to-redesign-the-explorer-benchm", "resolved_reason": null, "summary": "Use benchmark support status to redesign the explorer benchmark browser"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-and-discovery-policy", "id": 357, "reason": "Support status is not a visibility or deprecation decision by itself.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: The item deliberately decoupled support-status from visibility (L108-110: do not hide/remove a public benchmark solely on support status; visibility changes need explicit per-benchmark surface/deprecation decision). No standing action; it's a per-case decision made at deprecation time.", "summary": "Remove or hide public benchmark facades based on support status"}], "deps": ["architecture-contract-map-and-support-taxonomy", "benchmark-api-and-core-boundary-cleanup"], "description": "The public contract map defines `support_status` for platforms and benchmarks.\nPlatform metadata already carried support status; this item closed the\nbenchmark half by adding benchmark-level support status to the benchmark\nmetadata source of truth, defining how it interacts with `surface` and\n`supports_dataframe`, and wiring the count/status claims through tested\ndiscovery paths.", "id": "benchmark-support-status-and-discovery-policy", "preserves": ["Do not create a second benchmark catalogue outside benchbox/core/benchmark_registry.py.", "Do not equate benchmark category with support_status without an explicit product decision.", "Do not hide, remove, or deprecate existing public benchmarks solely as a side effect of adding metadata.", "Do not treat supports_dataframe as support_status; it is a capability flag."], "prior_art": [{"concept": "benchbox/core/benchmark_registry.py: current benchmark metadata and discovery source of truth -- extend in place.", "decision": "extend", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "benchbox/core/platform_registry.py: platform support_status validation and summary helpers -- reuse the pattern, not the platform-specific metadata.", "decision": "reuse", "path": "benchbox/core/platform_registry.py"}, {"concept": "docs/reference/public-contracts.md: public support taxonomy and drift-check ownership -- extend rather than creating separate status language.", "decision": "extend", "path": "docs/reference/public-contracts.md"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: existing wrapper/loader/registry count drift tests -- extend with support-status assertions.", "decision": "extend", "path": "tests/unit/core/test_benchmark_api_contract.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/contract.py"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/schemas.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/discovery.py"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/"}, {"kind": "only_modify", "path_glob": "docs/reference/mcp.md"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_discovery_tools.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_tool_handlers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_resources.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_run_surface_contract.py"}], "state": "done", "title": "Architecture hardening: add benchmark support status and discovery policy", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/mcp -k 'benchmark or discovery or support_status' --tb=short", "description": "Run benchmark API and discovery contract tests after metadata migration.", "expected": "Benchmark support-status, count, and discovery tests pass or skip only for missing optional MCP SDK.", "item_id": "benchmark-support-status-and-discovery-policy", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline -k 'benchmark or read_model' --tb=short", "description": "Run explorer pipeline tests if benchmark status is exported to explorer inputs.", "expected": "Explorer benchmark label/read-model tests pass.", "item_id": "benchmark-support-status-and-discovery-policy", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "Validate TODO graph and schema after adding or completing this item.", "expected": "TODO validation and graph checks pass.", "item_id": "benchmark-support-status-and-discovery-policy", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": [], "notes": "Completed 2026-05-21 at checked SHA\nd8102e8fd7c6a216f5af18fd1d0c1c55db1a4c69.\n\nRevalidation summary:\n - `benchbox/core/benchmark_registry.py` had 23 metadata entries, 23 loader\n IDs, 22 public discovery IDs, 21 top-level facades, and one internal\n benchmark (`joinorder_synthetic`).\n - MCP discovery and resources consumed registry metadata but did not expose\n support status.\n - Explorer pipeline code did not maintain an independent benchmark support\n catalogue; no explorer code change was needed to prevent classification\n drift.\n - `docs/reference/public-contracts.md` still described benchmark\n `support_status` as pending.\n\nCapture a compact summary of checked SHA, current benchmark counts,\ndiscovery surfaces, docs claims, and any line-number drift. Do not commit\nraw command transcripts.\n\nMinimum evidence to re-check:\n - `benchbox/core/benchmark_registry.py` metadata, `surface`, and list helpers.\n - `benchbox/__init__.py` wrapper facade exports.\n - `benchbox/core/benchmark_loader.py` loader reachability.\n - MCP benchmark discovery tools and docs.\n - Explorer pipeline benchmark labels and any \"supported benchmark\" copy.\n - `docs/reference/public-contracts.md` count and support taxonomy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate benchmark discovery and support-claim evidence before editing", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w0"], "notes": "Completed with the shared taxonomy values: stable, beta, experimental,\nrepo_only, deprecated, and document_only.\n\nCurrent product decision:\n - stable: tpch, tpcds, ssb, clickbench, joinorder.\n - beta: tpcdi, h2odb, amplab, read_primitives, write_primitives,\n metadata_primitives, transaction_primitives, coffeeshop, tsbs_devops,\n nyctaxi, flightdata, vector_search.\n - experimental: tpcds_obt, ai_primitives, tpchavoc, tpch_skew, datavault.\n - repo_only: joinorder_synthetic.\n - deprecated/document_only: none.\n\nReuse the public taxonomy values: stable, beta, experimental, repo_only,\ndeprecated, and document_only. Spell out how they apply to benchmark\nfamilies, data availability, official-vs-development scale factors, and\nDataFrame support.\n\nKeep `support_status` distinct from:\n - `surface`, which controls discovery visibility.\n - `supports_dataframe`, which is a capability flag.\n - dataset availability or local optional dependency availability.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define benchmark-specific support_status semantics", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w1"], "notes": "Completed by adding `support_status` to every `BENCHMARK_METADATA` entry,\nvalidating at import time, and adding registry helpers:\n`get_benchmark_support_status()`, `get_benchmarks_by_support_status()`, and\n`get_benchmark_registry_summary()`.\n\nEvery `BENCHMARK_METADATA` entry should have exactly one valid\n`support_status`. Add helper accessors or summary functions only where\ndiscovery/docs/tests need them; avoid introducing a second benchmark\ncatalogue.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add support_status metadata and validation to benchmark_registry", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w2"], "notes": "Completed for MCP discovery/resources and docs. `list_available`,\n`get_benchmark_info`, benchmark resources, and query-detail benchmark info\nnow project registry `support_status`. CLI display was left unchanged because\nit does not make support-level claims, and explorer code was left unchanged\nbecause it has no separate benchmark support catalogue.\n\nUpdate CLI, MCP, explorer, and docs only where they make support-level\nclaims. Do not hide or remove an existing public benchmark solely because\nits support status is clarified; visibility changes need an explicit\nsurface/deprecation decision.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Propagate benchmark support status to discovery surfaces", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-support-status-and-discovery-policy", "needs": ["w2"], "notes": "Completed by extending benchmark API contract tests and MCP discovery/resource\ntests. The contract map and benchmark docs now state support-status counts\nfrom registry metadata instead of treating benchmark status as pending.\n\nExtend benchmark API contract coverage so registry count, wrapper count,\nloader count, public discovery count, and support-status summary claims\ncannot drift silently. Exact user-facing count tables should derive from\nbenchmark registry metadata or be marked editorial.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add drift checks for benchmark status and count claims", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change DataFrame capability flags as part of criteria documentation", "instead": "file follow-up TODOs for capability gaps.", "why": "capability and support status are separate contracts"}, {"dont": "duplicate the entire BENCHMARK_METADATA table in docs", "instead": "document only status rationale and evidence pointers.", "why": "that creates a second catalog"}, {"dont": "mark a benchmark stable only because it imports or runs at smoke scale", "instead": "require evidence rows.", "why": "stable implies docs, validation, and maintenance commitment"}, {"dont": "use benchmark category as status rationale", "instead": "record criteria evidence instead.", "why": "category is workload taxonomy, not product support"}], "approach": "Start from the current public contract taxonomy, then add benchmark-specific\nevidence dimensions. Use the registry to enumerate benchmark IDs and statuses.\nCross-check docs and result-integrity spec coverage, then write a compact\nmatrix that explains current status values without becoming a second runtime\ncatalog. Add a drift check only after choosing a matrix format that can be\nparsed reliably.\n", "blocked_reason": null, "category": "Benchmark Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-25T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-criteria-matrix", "id": 358, "reason": "It is the only beta whose sole integrity-dimension gap is a missing result-integrity spec; adding one (plus a DataFrame routing decision) is the concrete blocker recorded in docs/benchmarks/support-status.md.", "resolution": "promoted", "resolved_item": "promote-vector-search-beta-stable-once-it-has-a-benchmarkspe", "resolved_reason": null, "summary": "Promote vector_search beta -> stable once it has a BenchmarkSpec"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-criteria-matrix", "id": 359, "reason": "Their status is partly coupled to the benchbox.core.base_benchmark.BaseBenchmark migration; revisit when that base is gone.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Verified benchbox/core/base_benchmark.py still exists; deprecated base has not been removed, so the reassessment trigger has not occurred.", "summary": "Reassess tpcds_obt and datavault experimental status after the deprecated core base is removed"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "benchmark-support-status-criteria-matrix", "id": 360, "reason": "External-dataset betas are gated on a pinned source contract, not on query coverage.", "resolution": "promoted", "resolved_item": "pin-reproducible-external-source-contracts-for-nyctaxi-and-f", "resolved_reason": null, "summary": "Pin reproducible external-source contracts for nyctaxi and flightdata before any stable promotion"}], "deps": ["benchmark-support-status-and-discovery-policy"], "description": "PR #502 added registry-owned `support_status` for every benchmark, but the\nfirst assignment is still a product decision encoded as metadata rather than\nan auditable criteria matrix. The current counts are test-protected, but the\nrepository does not yet explain why each benchmark is stable, beta,\nexperimental, repo_only, deprecated, or document_only.\n\nBuild a benchmark support-status criteria matrix that makes future promotions\nand demotions reviewable. The matrix should evaluate each benchmark against\nconcrete product criteria: user docs coverage, result-integrity spec coverage,\ndataset/source contract, validation coverage, public discovery behavior,\nDataFrame coverage where claimed, known unsupported platform gaps, and\nmaintenance commitment. The matrix should not duplicate the registry as a\nsecond source of truth; it should document the rationale and acceptance\nthresholds for changing the registry-owned status field.", "id": "benchmark-support-status-criteria-matrix", "preserves": ["Benchmark surface visibility remains independent of support_status.", "Current public benchmark discovery count remains unchanged unless a separate surface decision changes it.", "Runtime support_status values remain registry-owned in benchbox/core/benchmark_registry.py.", "supports_dataframe remains a capability flag, not a proxy for support status."], "prior_art": [{"concept": "benchbox/core/benchmark_registry.py: BENCHMARK_METADATA support_status values - reuse as runtime source of truth.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "docs/development/result-integrity-validation.md: integrity spec coverage list - use as one evidence dimension, not as the whole support decision.", "decision": "reuse", "path": "docs/development/result-integrity-validation.md"}, {"concept": "docs/platforms/support-status.md: platform support-status rationale style - reuse the explanatory pattern, but adapt criteria to benchmark-specific evidence.", "decision": "reuse", "path": "docs/platforms/support-status.md"}, {"concept": "docs/reference/public-contracts.md: support taxonomy and count policy - extend criteria language rather than creating new status terms.", "decision": "extend", "path": "docs/reference/public-contracts.md"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: benchmark count and support-status drift tests - extend for matrix row coverage.", "decision": "extend", "path": "tests/unit/core/test_benchmark_api_contract.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/"}, {"kind": "only_modify", "path_glob": "docs/development/result-integrity-validation.md"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}], "state": "done", "title": "Define auditable benchmark support-status promotion criteria", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py -q", "description": "Benchmark registry support-status contract remains green.", "expected": "All benchmark API contract tests pass.", "item_id": "benchmark-support-status-criteria-matrix", "last_result": null, "last_run": null, "seq": 1}, {"command": "make docs-validate", "description": "Docs validate after adding the criteria matrix.", "expected": "Documentation validation passes.", "item_id": "benchmark-support-status-criteria-matrix", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph and schema remain valid if follow-up TODOs are added.", "expected": "TODO validation and graph checks pass.", "item_id": "benchmark-support-status-criteria-matrix", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": [], "notes": "Capture the checked SHA, current status counts, benchmark IDs, public IDs,\nand result-integrity spec coverage in the TODO or implementation PR body.\nDo not rely on the PR #502 review text alone; the criteria matrix must be\nbased on the current tree at implementation time.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-read the benchmark registry, public contract map, benchmark docs, and result-integrity spec list before proposing criteria", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w0"], "notes": "Write criteria for stable, beta, experimental, repo_only, deprecated, and\ndocument_only. Include benchmark-specific dimensions that platform status\ndid not need: official-vs-development scale factors, generated-vs-external\ndatasets, canonical query coverage, result validation availability, docs\ndepth, and whether DataFrame support is claimed for that benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define status-specific acceptance criteria for benchmark families", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w1"], "notes": "Produce one row per benchmark with the current registry status, rationale,\nevidence pointers, and any promotion/demotion blockers. The matrix may live\nin a developer/reference doc, but the runtime status remains in\n`benchbox/core/benchmark_registry.py`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Score every registry benchmark against the criteria and record status rationale", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w2"], "notes": "Add a focused test that fails when a benchmark status is added, removed, or\nchanged without a corresponding criteria-matrix row or explicit rationale\nupdate. Prefer a simple parseable markdown table or YAML/JSON summary if\nthe existing docs test patterns can check it cleanly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests or docs checks that keep the matrix synchronized with registry status values", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-support-status-criteria-matrix", "needs": ["w2"], "notes": "If the matrix shows that a beta benchmark is close to stable or that an\nexperimental benchmark has unclear criteria, record follow-up TODOs or\ndeferred entries with exact blockers. Do not change support_status values\nin this TODO unless the criteria evidence makes the change unambiguous.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record concrete promotion candidates and blockers as deferred work, not as silent comments", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "build tests that depend on rich color codes", "instead": "assert labels and fields.", "why": "styling is incidental"}, {"dont": "expose repo_only or document_only support claims over MCP discovery", "instead": "omit or filter claims on explicit-ID surfaces.", "why": "the public taxonomy says those are not MCP-exposed"}, {"dont": "hide all experimental benchmarks by changing support_status behavior alone", "instead": "use explicit surface decisions for hiding.", "why": "visibility is controlled by surface"}, {"dont": "use supports_dataframe to infer support_status", "instead": "assert it separately.", "why": "DataFrame support is a capability dimension"}], "approach": "Treat `surface` as visibility, `support_status` as product support level, and\n`supports_dataframe` as capability. First write the policy table so tests have\na clear target. Then add metadata-only fixture tests for future statuses and\nfocused real-registry tests for current public beta/experimental and internal\nrepo_only benchmarks. Keep explicit-ID behavior stable unless the policy table\nexplicitly changes it.\n", "blocked_reason": null, "category": "Benchmark Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-25T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [], "deps": ["benchmark-support-status-and-discovery-policy"], "description": "PR #502 added benchmark support status and exposed it on MCP discovery. The\nimmediate review follow-up fixed CLI labeling for public benchmark listings\nand prevented MCP query-detail support-status leakage for internal/repo-only\nbenchmarks. The broader invariant still needs a durable test matrix: future\nrepo_only, document_only, deprecated, beta, and experimental benchmark entries\nshould fail loudly if a public surface exposes or hides them contrary to the\nsupport taxonomy and `surface` policy.\n\nAdd cross-surface invariant tests and policy documentation that prove\n`support_status`, `surface`, and `supports_dataframe` remain separate. The\ngoal is not to change visibility for current benchmarks. The goal is to make\nfuture status additions safe and reviewable across CLI, MCP discovery, MCP\nquery details, docs, and result/explorer inputs where applicable.", "id": "benchmark-support-status-visibility-invariants", "preserves": ["MCP discovery continues to expose support_status for public benchmarks.", "Public beta and experimental benchmarks remain visible unless a separate surface/deprecation decision changes them.", "joinorder_synthetic remains hidden from public discovery lists.", "joinorder_synthetic remains runnable by explicit ID where it was already supported.", "supports_dataframe continues to control DataFrame routing independently of support_status."], "prior_art": [{"concept": "benchbox/cli/benchmarks.py: BenchmarkManager public benchmark filtering - extend tests without changing explicit-ID run behavior.", "decision": "extend", "path": "benchbox/cli/benchmarks.py"}, {"concept": "benchbox/core/benchmark_registry.py:get_benchmark_support_status and get_benchmark_registry_summary - reuse for status assertions.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py:get_benchmark_support_status"}, {"concept": "benchbox/core/benchmark_registry.py:get_benchmark_surface and list_public_benchmark_ids - reuse for discovery visibility decisions.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py:get_benchmark_surface"}, {"concept": "benchbox/mcp/tools/benchmark.py: get_query_details benchmark_info - preserve explicit-ID compatibility while preventing unsupported support-status claims.", "decision": "reuse", "path": "benchbox/mcp/tools/benchmark.py"}, {"concept": "benchbox/mcp/tools/discovery.py: public benchmark discovery filtering - reuse as MCP listing contract.", "decision": "reuse", "path": "benchbox/mcp/tools/discovery.py"}, {"concept": "tests/unit/core/test_registry_surface_field.py: patch.dict surface tests - reuse fixture pattern for synthetic status/surface combinations.", "decision": "reuse", "path": "tests/unit/core/test_registry_surface_field.py"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/benchmarks.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/discovery.py"}, {"kind": "only_modify", "path_glob": "docs/reference/mcp.md"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_manager_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_benchmark_selection.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_registry_surface_field.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_discovery_tools.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_tool_handlers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_resources.py"}], "state": "done", "title": "Harden benchmark support-status visibility invariants across public surfaces", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py tests/unit/core/test_registry_surface_field.py -q", "description": "Benchmark registry and surface invariants pass.", "expected": "All benchmark registry and surface tests pass.", "item_id": "benchmark-support-status-visibility-invariants", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/test_benchmark_manager_behavioral.py tests/unit/cli/test_benchmark_selection.py -q", "description": "CLI benchmark manager support-status labeling tests pass.", "expected": "All selected CLI benchmark manager tests pass.", "item_id": "benchmark-support-status-visibility-invariants", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/mcp/test_discovery_tools.py tests/unit/mcp/test_resources.py tests/unit/mcp/test_mcp_tool_handlers.py -q", "description": "MCP discovery/resource/query-detail support-status tests pass.", "expected": "All selected MCP tests pass.", "item_id": "benchmark-support-status-visibility-invariants", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": [], "notes": "Re-check CLI interactive listing, CLI explicit benchmark execution, MCP\nlist_available, MCP get_benchmark_info, MCP get_query_details, resources,\nresult publisher discovery, explorer ingestion, and result-integrity docs.\nRecord which surfaces are discovery surfaces and which are explicit-ID\nexecution or inspection surfaces.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory current public, internal, and explicit-ID benchmark surfaces", "wid": "w0"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w0"], "notes": "Specify expected behavior for stable, beta, experimental, repo_only,\ndeprecated, and document_only when `surface` is public or internal. Include\nwhether each should be listed by default, labeled, hidden, runnable by\nexplicit ID, exposed over MCP discovery, exposed over MCP query details,\nand eligible for DataFrame routing if `supports_dataframe` is true.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write a visibility policy table for support_status x surface x capability", "wid": "w1"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w1"], "notes": "Use patch.dict or small helper fixtures to create temporary benchmark\nmetadata entries for repo_only, document_only, deprecated, beta, and\nexperimental statuses. Assert discovery hiding, public labeling, and\nexplicit-ID behavior according to the policy. Avoid importing heavy\nbenchmark classes when metadata-only behavior is sufficient.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add synthetic invariant tests for future repo_only and document_only benchmark metadata", "wid": "w2"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w1"], "notes": "Keep the immediate CLI tests from the review fix, then broaden them to\ncover public beta and experimental cases in list trees, selection tables,\npreview panels, MCP list_available, MCP get_benchmark_info, and benchmark\nresources. Tests should check contract shape, not rich color styling.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend CLI and MCP tests so non-stable public benchmarks are visibly labeled", "wid": "w3"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w1", "w2"], "notes": "If explicit-ID surfaces can inspect internal benchmarks, document what\nmetadata may be returned and which support claims must be omitted. Keep the\ndocumentation aligned with the public contract map and MCP docs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document explicit-ID exceptions and support-status exposure policy", "wid": "w4"}, {"evidence": null, "item_id": "benchmark-support-status-visibility-invariants", "needs": ["w2", "w3", "w4"], "notes": "Run the narrowest tests for benchmark registry, CLI benchmark manager, MCP\ndiscovery/resources/tool handlers, and any result/explorer path touched by\nthe invariant work. Escalate to fast tests before PR submission.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run focused contract and MCP/CLI tests after invariant coverage lands", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-18T00:00:00Z", "completed_pr": null, "created_at": "2026-01-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement bidirectional sync between private (BenchBox) and public (BenchBox-public)\nrepositories. This is a redesign after the previous implementation failed due to:\n- Building separate filtering logic that ignored DOCS_DIR_EXCLUDES, CLAUDE_DIR_EXCLUDES\n- Copying files without committing them\n- No awareness of which version was newer\n- Silently overwriting public repo changes\n\nThe new design reuses existing prepare_release.py infrastructure and always commits\nafter sync operations.", "id": "bidirectional-repo-sync-v2", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Bidirectional Repository Sync (Redesigned)", "verifications": [], "work": [], "worktree": "release-sync"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-18T00:00:00Z", "completed_pr": null, "created_at": "2026-01-17T00:00:00Z", "deferrals": [], "deps": [], "description": "Add bidirectional sync support between private and public repositories, with a\n\"true up\" mode for incremental releases that only push missing or updated files.\n\nCurrent release automation creates an ephemeral public tree fresh each release,\nwith no tracking of what's in the public repo and no sync capability. This makes\nit difficult to:\n- Push bug fixes without full release (true-up)\n- Import external contributions from public repo\n- Track what files differ between repos\n\nRequirements clarified:\n1. Public repo is on GitHub - Already exists as remote repository\n2. Git-based diff - Use git to determine what changed\n3. Bidirectional sync - Support both private\u2192public and public\u2192private changes\n\nPlan file: /Users/joe/.claude/plans/synthetic-dazzling-pudding.md", "id": "bidirectional-sync-and-true-up-releases", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Bidirectional Sync and True-Up Releases for Public/Private Repos", "verifications": [], "work": [], "worktree": "release-sync"} -{"anti_patterns": [{"INSTEAD": "label them explicitly as external manual observations.", "dont": "present APPLY CHANGES INTO timings as BenchBox harness results", "why": "they come from a Databricks pipeline run the harness cannot drive"}, {"INSTEAD": "log dated checks first.", "dont": "publish without the research/verification file", "why": "the blog evidence standard requires every claim to be replayable"}, {"INSTEAD": "reconstruct and count it.", "dont": "repeat the '200 lines' claim uncritically", "why": "the real equivalent MERGE for a basic Type 2 is far shorter"}], "approach": "1. Research pass: read Databricks APPLY CHANGES INTO / AUTO CDC docs; capture\n exact syntax, SCD Type 2 semantics, SEQUENCE BY, APPLY AS DELETE, and the\n pipeline-context requirement. Log each verified fact (dated) in the\n research file.\n2. Reconstruct the equivalent standard MERGE (reuse the SCD2 op from\n write-primitives-scd-type2-coverage) and do an honest line/edge-case count\n vs the declarative form.\n3. Measure the MERGE form via BenchBox across engines; if a Databricks\n workspace is available, run the APPLY CHANGES INTO pipeline on the same\n data and record it as a clearly-labeled external observation.\n4. Outline -> draft following the platform-deep-dive structure. Land the\n convenience-vs-portability trade-off with the side-by-side code and the\n numbers we actually have.\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-27T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "blog-apply-changes-into-vs-standard-sql", "id": 303, "reason": "Requires a pipeline-execution adapter the harness does not have; the post treats Databricks pipeline timings as external observations instead. Tied to the same constraint in write-primitives-scd-type2-coverage.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Blocked on a pipeline-execution adapter BenchBox does not have; the blog post already works around it via external observations.", "summary": "Benchmarking the DLT pipeline path inside the BenchBox harness"}], "deps": ["write-primitives-scd-type2-coverage"], "description": "Write a platform-deep-dive blog post that explicitly evaluates Databricks'\ndeclarative APPLY CHANGES INTO (DLT / Lakeflow Declarative Pipelines) against\nhand-written standard SQL MERGE for Slowly Changing Dimension (SCD) Type 2\nmaintenance.\n\nThe hook (from a widely shared Databricks/Delta Lake infographic): APPLY\nCHANGES INTO replaces \"~200 lines of MERGE logic\" with a few declarative lines\nthat handle inserts/updates/deletes, history management, and record\nopen/close automatically. The post should test that marketing claim with a\nfair, reproducible, BenchBox-grounded evaluation rather than restating it.\n\nThe post must cover BOTH axes honestly:\n - Ergonomics / maintainability: lines of code, edge cases handled for you\n (out-of-order events via SEQUENCE BY, schema evolution, deletes), and\n where the abstraction leaks.\n - Portability / lock-in: APPLY CHANGES INTO only runs inside a Databricks\n declarative pipeline (not a plain SQL warehouse, not any other engine),\n whereas the MERGE form runs unchanged across DuckDB, PostgreSQL,\n Snowflake, BigQuery, and ClickHouse.\n - Performance / cost: where measurable, compare the two on the same SCD2\n workload (see dependency below) - net new versions written, runtime, and\n $ where a cost model exists - and clearly label what is NOT measurable\n inside BenchBox today (the DLT pipeline path) versus what is.\n\nThis is a companion to the write-primitives-scd-type2-coverage TODO: that TODO\nadds the portable MERGE-based SCD2 operation to the benchmark; this post uses\nit (plus a Databricks pipeline run, if feasible) to produce the evaluation.", "id": "blog-apply-changes-into-vs-standard-sql", "preserves": ["BenchBox harness results and manually-run Databricks pipeline observations are never conflated", "Every performance / behavior claim is backed by a dated entry in the research/verification file (house evidence standard)"], "prior_art": [{"concept": "_blog/STYLE_GUIDE.md and _blog/VOICE_REFERENCE.md \u2014 match house voice and evidence standards", "decision": "reuse", "path": "_blog/STYLE_GUIDE.md"}, {"concept": "_blog/platform-deep-dives/drafts/duckdb-architecture.md \u2014 reuse the platform-deep-dive structure and depth", "decision": "reuse", "path": "_blog/platform-deep-dives/drafts/duckdb-architecture.md"}, {"concept": "_blog/platform-deep-dives/research/duckdb-architecture-verification-2026-02-20.md \u2014 reuse the research/verification-log convention (every claim backed by a dated, replayable check)", "decision": "reuse", "path": "_blog/platform-deep-dives/research/duckdb-architecture-verification-2026-02-20.md"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "_blog/platform-deep-dives/"}], "state": "done", "title": "Blog - Evaluate Databricks APPLY CHANGES INTO vs Standard SQL MERGE for SCD Type 2", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_todo_executable_docs.py -q", "description": "Research/verification file exists with dated, sourced claims", "expected": "all tests pass (no broken doc references introduced)", "item_id": "blog-apply-changes-into-vs-standard-sql", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox import WritePrimitives; print(any(k.startswith('merge_scd_type2') for k in WritePrimitives(0.01).get_all_operations()))\"", "description": "Draft references a real, runnable BenchBox SCD2 operation", "expected": "True", "item_id": "blog-apply-changes-into-vs-standard-sql", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": [], "notes": "Databricks docs: APPLY CHANGES INTO / AUTO CDC, SCD TYPE 2, SEQUENCE BY,\nAPPLY AS DELETE, COLUMNS/EXCEPT, the DLT/Lakeflow pipeline requirement.\nLog each verified fact (dated) in the research file.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research APPLY CHANGES INTO semantics and pipeline-context constraint", "wid": "w1"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w1"], "notes": "Reuse the merge_scd_type2 op. Count real lines and enumerate which edge\ncases (ordering, deletes, schema evolution) each form handles.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconstruct equivalent standard MERGE and do honest LOC/edge-case count", "wid": "w2"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w2"], "notes": "MERGE form across engines via BenchBox (runtime, rows versioned, $ where\na cost model exists). APPLY CHANGES INTO only if a Databricks workspace is\navailable; record as a clearly-labeled external observation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gather measurements - MERGE via BenchBox; APPLY CHANGES INTO externally if feasible", "wid": "w3"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write outline", "wid": "w4"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write draft with side-by-side code and the measurements we have", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use hyphens.", "dont": "introduce em/en-dashes", "why": "house rule for _blog/ content"}, {"INSTEAD": "re-run first.", "dont": "reuse the stale audit SF1 number without re-measuring", "why": "the op changed"}], "approach": "Small factual addition. Re-measure SF1 rather than reuse the audit's ~13.3 ms,\nthen extend the existing table and soften the \"~1.7x\" sentence to acknowledge the\nnon-constant per-decade rate. Draft is unpublished, so this is low urgency.\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up deferred from scd2-docs-and-count-hygiene-followup (audit finding N12)\nand re-deferred from scd2-basic-idempotency-and-cleanup-scoping w4, which updated\nthe blog line counts (18 -> 20) but did not add the third scale point.\n\nThe SCD2 blog draft\n(_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md) presents\nthe \"change-batch-bound, not dimension-bound\" scale claim on only two points\n(SF0.01 and SF0.1, ~1.7x). Adding an SF1 row makes the growth curve honest: the\naudit re-derivation measured SF1.0 basic median ~13.3 ms against a 150,000-row\ndimension, i.e. 100x data costs ~6.4x, and the per-decade rate is NOT constant\n(~1.74x then ~3.66x). The two-point presentation reads flatter than the curve.\n\nScope: blog draft only (the research note already records the SF1 figure in its\nscale-independence section). Re-measure SF1 before publishing rather than\ntrusting the stale audit number, since the idempotency guard changed basic's SQL.\n", "id": "blog-scd2-sf1-scale-row", "preserves": ["No em-dashes or en-dashes in blog content (house rule).", "SF0.01 / SF0.1 rows and their sources are unchanged unless re-measured.", "The change-batch-bound framing stays accurate; do not overstate flatness."], "prior_art": [{"concept": "_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md:196-213 - the SF0.01 / SF0.1 scale tables and the ~1.7x observation to extend", "decision": "extend", "path": "_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md:196-213"}, {"concept": "_project/DONE/scd2-audit-followup/scd2-basic-idempotency-and-cleanup-scoping.yaml w4 - where the SF1 fold-in was re-deferred", "decision": "reuse", "path": "_project/DONE/scd2-audit-followup/scd2-basic-idempotency-and-cleanup-scoping.yaml"}, {"concept": "_project/DONE/scd2-audit-followup/scd2-docs-and-count-hygiene-followup.yaml deferred[] - N12 origin", "decision": "reuse", "path": "_project/DONE/scd2-audit-followup/scd2-docs-and-count-hygiene-followup.yaml"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md"}], "state": "planning", "title": "Add the SF1 basic row to the SCD2 blog scale table for balance", "verifications": [{"command": "grep -nP \"[\\x{2014}\\x{2013}]\" _blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md", "description": "No em/en-dash in the edited draft.", "expected": "no matches (exit 1)", "item_id": "blog-scd2-sf1-scale-row", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "blog-scd2-sf1-scale-row", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-measure basic at SF1.0 (post-idempotency-guard) with the audit protocol; capture median/min/p90.", "wid": "w1"}, {"evidence": null, "item_id": "blog-scd2-sf1-scale-row", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add the SF1 row to the blog scale table and adjust the growth-rate sentence (per-decade rate is not constant; ~6.4x for 100x data).", "wid": "w2"}], "worktree": "scd2-audit-followup"} -{"anti_patterns": [{"INSTEAD": "handle JIT compilation optionally", "dont": "add @bodo.jit decorators directly in adapter methods", "why": "the adapter must work with or without JIT"}, {"INSTEAD": "use conditional imports with BODO_AVAILABLE flag", "dont": "add bodo as a required project dependency", "why": "it is an optional platform"}, {"INSTEAD": "keep all Bodo-specific code in bodo_df.py", "dont": "modify PandasFamilyAdapter", "why": "Bodo is just another consumer of the base class"}, {"INSTEAD": "mock bodo imports in tests", "dont": "require Bodo installation for unit tests", "why": "Bodo has heavy dependencies (MPI, Numba) not available in CI"}], "approach": "Follow the pattern established by ModinDataFrameAdapter in\nbenchbox/platforms/dataframe/modin_df.py (~500 lines). These are thin\nadapters that:\n1. Extend PandasFamilyAdapter with platform-specific I/O overrides\n2. Use try/except import for availability detection (BODO_AVAILABLE flag)\n3. Override read_csv, read_parquet, to_datetime, and timedelta_days\n4. Inherit all query execution, aggregation, and validation from the base class\n\nStart with modin_df.py as the closest reference since Modin also provides\na Pandas-compatible API with a different execution backend.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-09T00:00:00Z", "completed_pr": null, "created_at": "2026-02-09T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:10Z", "from_item": "bodo-dataframe-platform", "id": 578, "reason": "JIT compilation is Bodo's main value proposition but adds significant complexity. Start with Bodo's Pandas-compatible I/O only; add JIT support as a follow-up once the basic adapter is validated.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Exact match: open item bodo-jit-and-mpi-support explicitly covers @bodo.jit compilation follow-up.", "summary": "@bodo.jit compilation of query functions"}, {"created_at": "2026-07-18T19:50:10Z", "from_item": "bodo-dataframe-platform", "id": 579, "reason": "Multi-node execution requires mpiexec launch and cluster setup. Initial adapter targets single-node multi-core parallelism only.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Bodo MPI multi-node execution is exactly the scope of open item bodo-jit-and-mpi-support.", "summary": "MPI multi-node distributed execution"}], "deps": ["implement-dataframe-benchmarks"], "description": "Implement Bodo as a new DataFrame platform adapter in the Pandas family.\nBodo is a high-performance computing engine that provides automatic\nparallelization and optimization of Python/Pandas code using Numba JIT\ncompilation. It enables distributed execution of Pandas workloads across\nmultiple cores and nodes without requiring code changes.\n\nThe adapter should extend PandasFamilyAdapter, following the same pattern\nused by Modin, Dask, and cuDF adapters. Since Bodo provides a Pandas-compatible\nAPI, the majority of functionality is inherited from the base class, with\noverrides needed primarily for I/O operations and distributed initialization.", "id": "bodo-dataframe-platform", "preserves": ["Existing Pandas-family adapters (pandas_df, modin_df, dask_df, cudf_df) continue to pass their test suites", "PandasFamilyAdapter base class interface and all existing method signatures unchanged", "Platform registry backward compatible with existing platform names", "adapter_factory.py mapping for existing platforms returns same adapter classes"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/expression_family.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/modin_df.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/pandas_df.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/pandas_family.py"}, {"kind": "only_modify", "path_glob": "_project/specs/platforms/bodo.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/adapter_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/bodo_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/platform_checker.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_bodo_df.py"}], "state": "done", "title": "Bodo DataFrame Platform Adapter", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.dataframe.bodo_df import BodoDataFrameAdapter; print('import OK')\"", "description": "Bodo adapter imports correctly when bodo is available", "expected": "import OK (or ImportError if bodo not installed)", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.platforms import is_dataframe_platform; print(is_dataframe_platform('bodo-df'))\"", "description": "Bodo recognized as a DataFrame platform", "expected": "True", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.platforms import list_available_dataframe_platforms; print('bodo-df' in list_available_dataframe_platforms())\"", "description": "Bodo appears in available platform list", "expected": "True", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/test_bodo_df.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/ -v --tb=short", "description": "Existing dataframe tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "bodo-dataframe-platform", "needs": [], "notes": "Extend PandasFamilyAdapter. Override read_csv and read_parquet with Bodo-compatible implementations. Implement platform_name, get_platform_info, and BODO_AVAILABLE flag. Handle Bodo-specific type conversions and limitations. Use modin_df.py as the closest reference implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create BodoDataFrameAdapter in benchbox/platforms/dataframe/bodo_df.py", "wid": "w1"}, {"evidence": null, "item_id": "bodo-dataframe-platform", "needs": ["w1"], "notes": "Add to dataframe __init__.py exports, adapter_factory.py mapping, platform_registry.py metadata, and platform_checker.py DATAFRAME_PLATFORMS.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register Bodo in adapter_factory, platform_registry, and platform_checker", "wid": "w2"}, {"evidence": null, "item_id": "bodo-dataframe-platform", "needs": ["w1"], "notes": "Test import availability check, adapter instantiation (mocked), data loading methods (mocked), platform info and metadata. Verify existing dataframe tests still pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create unit tests in tests/unit/platforms/dataframe/test_bodo_df.py", "wid": "w3"}, {"evidence": null, "item_id": "bodo-dataframe-platform", "needs": ["w1", "w2", "w3"], "notes": "Document Bodo-specific requirements, limitations, and usage.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create platform spec documentation in _project/specs/platforms/bodo.md", "wid": "w4"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "present APPLY CHANGES INTO timings as BenchBox harness results", "instead": "label them explicitly as external manual observations.", "why": "they come from a Databricks pipeline run the harness cannot drive"}, {"dont": "publish without the research/verification file", "instead": "log dated checks first.", "why": "the blog evidence standard requires every claim to be replayable"}, {"dont": "repeat the '200 lines' claim uncritically", "instead": "reconstruct and count it.", "why": "the real equivalent MERGE for a basic Type 2 is far shorter"}], "approach": "1. Research pass: read Databricks APPLY CHANGES INTO / AUTO CDC docs; capture\n exact syntax, SCD Type 2 semantics, SEQUENCE BY, APPLY AS DELETE, and the\n pipeline-context requirement. Log each verified fact (dated) in the\n research file.\n2. Reconstruct the equivalent standard MERGE (reuse the SCD2 op from\n write-primitives-scd-type2-coverage) and do an honest line/edge-case count\n vs the declarative form.\n3. Measure the MERGE form via BenchBox across engines; if a Databricks\n workspace is available, run the APPLY CHANGES INTO pipeline on the same\n data and record it as a clearly-labeled external observation.\n4. Outline -> draft following the platform-deep-dive structure. Land the\n convenience-vs-portability trade-off with the side-by-side code and the\n numbers we actually have.\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-27T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "blog-apply-changes-into-vs-standard-sql", "id": 303, "reason": "Requires a pipeline-execution adapter the harness does not have; the post treats Databricks pipeline timings as external observations instead. Tied to the same constraint in write-primitives-scd-type2-coverage.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Blocked on a pipeline-execution adapter BenchBox does not have; the blog post already works around it via external observations.", "summary": "Benchmarking the DLT pipeline path inside the BenchBox harness"}], "deps": ["write-primitives-scd-type2-coverage"], "description": "Write a platform-deep-dive blog post that explicitly evaluates Databricks'\ndeclarative APPLY CHANGES INTO (DLT / Lakeflow Declarative Pipelines) against\nhand-written standard SQL MERGE for Slowly Changing Dimension (SCD) Type 2\nmaintenance.\n\nThe hook (from a widely shared Databricks/Delta Lake infographic): APPLY\nCHANGES INTO replaces \"~200 lines of MERGE logic\" with a few declarative lines\nthat handle inserts/updates/deletes, history management, and record\nopen/close automatically. The post should test that marketing claim with a\nfair, reproducible, BenchBox-grounded evaluation rather than restating it.\n\nThe post must cover BOTH axes honestly:\n - Ergonomics / maintainability: lines of code, edge cases handled for you\n (out-of-order events via SEQUENCE BY, schema evolution, deletes), and\n where the abstraction leaks.\n - Portability / lock-in: APPLY CHANGES INTO only runs inside a Databricks\n declarative pipeline (not a plain SQL warehouse, not any other engine),\n whereas the MERGE form runs unchanged across DuckDB, PostgreSQL,\n Snowflake, BigQuery, and ClickHouse.\n - Performance / cost: where measurable, compare the two on the same SCD2\n workload (see dependency below) - net new versions written, runtime, and\n $ where a cost model exists - and clearly label what is NOT measurable\n inside BenchBox today (the DLT pipeline path) versus what is.\n\nThis is a companion to the write-primitives-scd-type2-coverage TODO: that TODO\nadds the portable MERGE-based SCD2 operation to the benchmark; this post uses\nit (plus a Databricks pipeline run, if feasible) to produce the evaluation.", "id": "blog-apply-changes-into-vs-standard-sql", "preserves": ["BenchBox harness results and manually-run Databricks pipeline observations are never conflated", "Every performance / behavior claim is backed by a dated entry in the research/verification file (house evidence standard)"], "prior_art": [{"concept": "_blog/STYLE_GUIDE.md and _blog/VOICE_REFERENCE.md \u2014 match house voice and evidence standards", "decision": "reuse", "path": "_blog/STYLE_GUIDE.md"}, {"concept": "_blog/platform-deep-dives/drafts/duckdb-architecture.md \u2014 reuse the platform-deep-dive structure and depth", "decision": "reuse", "path": "_blog/platform-deep-dives/drafts/duckdb-architecture.md"}, {"concept": "_blog/platform-deep-dives/research/duckdb-architecture-verification-2026-02-20.md \u2014 reuse the research/verification-log convention (every claim backed by a dated, replayable check)", "decision": "reuse", "path": "_blog/platform-deep-dives/research/duckdb-architecture-verification-2026-02-20.md"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "_blog/platform-deep-dives/"}], "state": "done", "title": "Blog - Evaluate Databricks APPLY CHANGES INTO vs Standard SQL MERGE for SCD Type 2", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_todo_executable_docs.py -q", "description": "Research/verification file exists with dated, sourced claims", "expected": "all tests pass (no broken doc references introduced)", "item_id": "blog-apply-changes-into-vs-standard-sql", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox import WritePrimitives; print(any(k.startswith('merge_scd_type2') for k in WritePrimitives(0.01).get_all_operations()))\"", "description": "Draft references a real, runnable BenchBox SCD2 operation", "expected": "True", "item_id": "blog-apply-changes-into-vs-standard-sql", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": [], "notes": "Databricks docs: APPLY CHANGES INTO / AUTO CDC, SCD TYPE 2, SEQUENCE BY,\nAPPLY AS DELETE, COLUMNS/EXCEPT, the DLT/Lakeflow pipeline requirement.\nLog each verified fact (dated) in the research file.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research APPLY CHANGES INTO semantics and pipeline-context constraint", "wid": "w1"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w1"], "notes": "Reuse the merge_scd_type2 op. Count real lines and enumerate which edge\ncases (ordering, deletes, schema evolution) each form handles.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconstruct equivalent standard MERGE and do honest LOC/edge-case count", "wid": "w2"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w2"], "notes": "MERGE form across engines via BenchBox (runtime, rows versioned, $ where\na cost model exists). APPLY CHANGES INTO only if a Databricks workspace is\navailable; record as a clearly-labeled external observation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gather measurements - MERGE via BenchBox; APPLY CHANGES INTO externally if feasible", "wid": "w3"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write outline", "wid": "w4"}, {"evidence": null, "item_id": "blog-apply-changes-into-vs-standard-sql", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write draft with side-by-side code and the measurements we have", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "introduce em/en-dashes", "instead": "use hyphens.", "why": "house rule for _blog/ content"}, {"dont": "reuse the stale audit SF1 number without re-measuring", "instead": "re-run first.", "why": "the op changed"}], "approach": "Small factual addition. Re-measure SF1 rather than reuse the audit's ~13.3 ms,\nthen extend the existing table and soften the \"~1.7x\" sentence to acknowledge the\nnon-constant per-decade rate. Draft is unpublished, so this is low urgency.\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up deferred from scd2-docs-and-count-hygiene-followup (audit finding N12)\nand re-deferred from scd2-basic-idempotency-and-cleanup-scoping w4, which updated\nthe blog line counts (18 -> 20) but did not add the third scale point.\n\nThe SCD2 blog draft\n(_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md) presents\nthe \"change-batch-bound, not dimension-bound\" scale claim on only two points\n(SF0.01 and SF0.1, ~1.7x). Adding an SF1 row makes the growth curve honest: the\naudit re-derivation measured SF1.0 basic median ~13.3 ms against a 150,000-row\ndimension, i.e. 100x data costs ~6.4x, and the per-decade rate is NOT constant\n(~1.74x then ~3.66x). The two-point presentation reads flatter than the curve.\n\nScope: blog draft only (the research note already records the SF1 figure in its\nscale-independence section). Re-measure SF1 before publishing rather than\ntrusting the stale audit number, since the idempotency guard changed basic's SQL.\n", "id": "blog-scd2-sf1-scale-row", "preserves": ["No em-dashes or en-dashes in blog content (house rule).", "SF0.01 / SF0.1 rows and their sources are unchanged unless re-measured.", "The change-batch-bound framing stays accurate; do not overstate flatness."], "prior_art": [{"concept": "_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md:196-213 - the SF0.01 / SF0.1 scale tables and the ~1.7x observation to extend", "decision": "extend", "path": "_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md:196-213"}, {"concept": "_project/DONE/scd2-audit-followup/scd2-basic-idempotency-and-cleanup-scoping.yaml w4 - where the SF1 fold-in was re-deferred", "decision": "reuse", "path": "_project/DONE/scd2-audit-followup/scd2-basic-idempotency-and-cleanup-scoping.yaml"}, {"concept": "_project/DONE/scd2-audit-followup/scd2-docs-and-count-hygiene-followup.yaml deferred[] - N12 origin", "decision": "reuse", "path": "_project/DONE/scd2-audit-followup/scd2-docs-and-count-hygiene-followup.yaml"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "_blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md"}], "state": "planning", "title": "Add the SF1 basic row to the SCD2 blog scale table for balance", "verifications": [{"command": "grep -nP \"[\\x{2014}\\x{2013}]\" _blog/platform-deep-dives/drafts/apply-changes-into-vs-standard-sql.md", "description": "No em/en-dash in the edited draft.", "expected": "no matches (exit 1)", "item_id": "blog-scd2-sf1-scale-row", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "blog-scd2-sf1-scale-row", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-measure basic at SF1.0 (post-idempotency-guard) with the audit protocol; capture median/min/p90.", "wid": "w1"}, {"evidence": null, "item_id": "blog-scd2-sf1-scale-row", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add the SF1 row to the blog scale table and adjust the growth-rate sentence (per-decade rate is not constant; ~6.4x for 100x data).", "wid": "w2"}], "worktree": "scd2-audit-followup"} +{"anti_patterns": [{"dont": "add @bodo.jit decorators directly in adapter methods", "instead": "handle JIT compilation optionally", "why": "the adapter must work with or without JIT"}, {"dont": "add bodo as a required project dependency", "instead": "use conditional imports with BODO_AVAILABLE flag", "why": "it is an optional platform"}, {"dont": "modify PandasFamilyAdapter", "instead": "keep all Bodo-specific code in bodo_df.py", "why": "Bodo is just another consumer of the base class"}, {"dont": "require Bodo installation for unit tests", "instead": "mock bodo imports in tests", "why": "Bodo has heavy dependencies (MPI, Numba) not available in CI"}], "approach": "Follow the pattern established by ModinDataFrameAdapter in\nbenchbox/platforms/dataframe/modin_df.py (~500 lines). These are thin\nadapters that:\n1. Extend PandasFamilyAdapter with platform-specific I/O overrides\n2. Use try/except import for availability detection (BODO_AVAILABLE flag)\n3. Override read_csv, read_parquet, to_datetime, and timedelta_days\n4. Inherit all query execution, aggregation, and validation from the base class\n\nStart with modin_df.py as the closest reference since Modin also provides\na Pandas-compatible API with a different execution backend.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-09T00:00:00Z", "completed_pr": null, "created_at": "2026-02-09T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:10Z", "from_item": "bodo-dataframe-platform", "id": 578, "reason": "JIT compilation is Bodo's main value proposition but adds significant complexity. Start with Bodo's Pandas-compatible I/O only; add JIT support as a follow-up once the basic adapter is validated.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Exact match: open item bodo-jit-and-mpi-support explicitly covers @bodo.jit compilation follow-up.", "summary": "@bodo.jit compilation of query functions"}, {"created_at": "2026-07-18T19:50:10Z", "from_item": "bodo-dataframe-platform", "id": 579, "reason": "Multi-node execution requires mpiexec launch and cluster setup. Initial adapter targets single-node multi-core parallelism only.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Bodo MPI multi-node execution is exactly the scope of open item bodo-jit-and-mpi-support.", "summary": "MPI multi-node distributed execution"}], "deps": ["implement-dataframe-benchmarks"], "description": "Implement Bodo as a new DataFrame platform adapter in the Pandas family.\nBodo is a high-performance computing engine that provides automatic\nparallelization and optimization of Python/Pandas code using Numba JIT\ncompilation. It enables distributed execution of Pandas workloads across\nmultiple cores and nodes without requiring code changes.\n\nThe adapter should extend PandasFamilyAdapter, following the same pattern\nused by Modin, Dask, and cuDF adapters. Since Bodo provides a Pandas-compatible\nAPI, the majority of functionality is inherited from the base class, with\noverrides needed primarily for I/O operations and distributed initialization.", "id": "bodo-dataframe-platform", "preserves": ["Existing Pandas-family adapters (pandas_df, modin_df, dask_df, cudf_df) continue to pass their test suites", "PandasFamilyAdapter base class interface and all existing method signatures unchanged", "Platform registry backward compatible with existing platform names", "adapter_factory.py mapping for existing platforms returns same adapter classes"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/expression_family.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/modin_df.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/pandas_df.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/pandas_family.py"}, {"kind": "only_modify", "path_glob": "_project/specs/platforms/bodo.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/adapter_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/bodo_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/platform_checker.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_bodo_df.py"}], "state": "done", "title": "Bodo DataFrame Platform Adapter", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.dataframe.bodo_df import BodoDataFrameAdapter; print('import OK')\"", "description": "Bodo adapter imports correctly when bodo is available", "expected": "import OK (or ImportError if bodo not installed)", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.platforms import is_dataframe_platform; print(is_dataframe_platform('bodo-df'))\"", "description": "Bodo recognized as a DataFrame platform", "expected": "True", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.platforms import list_available_dataframe_platforms; print('bodo-df' in list_available_dataframe_platforms())\"", "description": "Bodo appears in available platform list", "expected": "True", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/test_bodo_df.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/ -v --tb=short", "description": "Existing dataframe tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "bodo-dataframe-platform", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "bodo-dataframe-platform", "needs": [], "notes": "Extend PandasFamilyAdapter. Override read_csv and read_parquet with Bodo-compatible implementations. Implement platform_name, get_platform_info, and BODO_AVAILABLE flag. Handle Bodo-specific type conversions and limitations. Use modin_df.py as the closest reference implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create BodoDataFrameAdapter in benchbox/platforms/dataframe/bodo_df.py", "wid": "w1"}, {"evidence": null, "item_id": "bodo-dataframe-platform", "needs": ["w1"], "notes": "Add to dataframe __init__.py exports, adapter_factory.py mapping, platform_registry.py metadata, and platform_checker.py DATAFRAME_PLATFORMS.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register Bodo in adapter_factory, platform_registry, and platform_checker", "wid": "w2"}, {"evidence": null, "item_id": "bodo-dataframe-platform", "needs": ["w1"], "notes": "Test import availability check, adapter instantiation (mocked), data loading methods (mocked), platform info and metadata. Verify existing dataframe tests still pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create unit tests in tests/unit/platforms/dataframe/test_bodo_df.py", "wid": "w3"}, {"evidence": null, "item_id": "bodo-dataframe-platform", "needs": ["w1", "w2", "w3"], "notes": "Document Bodo-specific requirements, limitations, and usage.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create platform spec documentation in _project/specs/platforms/bodo.md", "wid": "w4"}], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "The Bodo DataFrame adapter currently uses Pandas-compatible I/O only.\n@bodo.jit is Bodo's core value proposition - auto-parallelization of Pandas\ncode via JIT compilation. MPI multi-node execution is the next tier.\n\nDeferred from bodo-dataframe-platform (completed 2026-02-09). The initial\nadapter established the Pandas-compatible baseline; these items add the\nperformance-differentiating features.\n\nNote: the Bodo adapter source file does not currently exist in the codebase\n(only __pycache__ artifacts remain). The adapter may need to be recreated\nor the DONE item may have been completed as research-only.\n", "id": "bodo-jit-and-mpi-support", "preserves": [], "prior_art": [], "priority": "low", "scope": [], "state": "planning", "title": "Add @bodo.jit Compilation and MPI Distributed Execution to Bodo Adapter", "verifications": [], "work": [{"evidence": null, "item_id": "bodo-jit-and-mpi-support", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Verify Bodo adapter exists or recreate from DONE item - establish Pandas-compatible baseline", "wid": "w1"}, {"evidence": null, "item_id": "bodo-jit-and-mpi-support", "needs": ["w1"], "notes": "Start with simple TPC-H queries (Q1, Q6). JIT compilation adds warmup\noverhead on first run; subsequent runs use cached compilation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add @bodo.jit compilation mode - decorate query functions, handle warmup, implement compilation caching", "wid": "w2"}, {"evidence": null, "item_id": "bodo-jit-and-mpi-support", "needs": ["w2"], "notes": "Only meaningful after JIT compilation works on single node. Requires\nmpiexec and cluster configuration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add MPI multi-node distributed execution support - mpiexec launch, hostfile config, data partitioning", "wid": "w3"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "always assert on the SQL string", "dont": "assert mock.called for SQL generation tests", "why": "called=True doesn't verify the SQL is correct"}, {"INSTEAD": "use real inputs and assert on real output", "dont": "mock internal helper functions to test a dispatcher", "why": "you then test the mock, not the code"}, {"INSTEAD": "test every exit(1) branch", "dont": "test only the happy path for CLI wrappers", "why": "error validation is where bugs live"}, {"INSTEAD": "assert the actual value", "dont": "test that a config field 'is not None'", "why": "that can never fail once the field is populated"}, {"INSTEAD": "assert the specific preset names", "dont": "write a test that only calls get_available_presets() and asserts len > 0", "why": "that can't catch a bug where a preset is removed"}], "approach": null, "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-26T00:00:00Z", "completed_pr": null, "created_at": "2026-03-26T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bottom-25-module-coverage", "id": 304, "reason": "Both require Spark session setup (SparkContext or SparkSession). PySpark tests already have a 1046-line test file at 23% because most tests skip when PySpark is not installed. Meaningful coverage requires a PySpark fixture or the separate dataframe platform CI environment. Cover in the dataframe-platform testing initiative, not here.", "resolution": "promoted", "resolved_item": "platforms-databricks-dataframe-adapter-18-120-miss-and-platf", "resolved_reason": null, "summary": "platforms/databricks/dataframe_adapter (18%, 120 miss) and platforms/dataframe/pyspark_df (23%, 200 miss)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bottom-25-module-coverage", "id": 305, "reason": "Azure Spark utilities require mocking Synapse/Fabric Spark session internals. Low ROI (only 30 missing lines) and fragile mock setup. Cover alongside other Azure Spark work when a real Spark session fixture is available.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly deferred until a real Spark session fixture exists; no such fixture is tracked as available yet.", "summary": "platforms/azure/spark_execution_utils (27%, 30 miss)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bottom-25-module-coverage", "id": 306, "reason": "These are in the omit list of .coveragerc_core and were explicitly deferred in non-dataframe-test-coverage. Still deferred: high effort, niche benchmark, requires complex CDC/SCD fixtures. Cover after 80% overall target is sustained.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Condition was 'after 80% overall coverage target sustained'; Makefile still sets suite-wide threshold at 70%, so condition unmet.", "summary": "TPC-DI ETL modules (core/tpcdi/etl/batch.py 882 miss, transformations.py 1053 miss)"}], "deps": ["non-dataframe-test-coverage"], "description": "After completing non-dataframe-test-coverage, 25 modules remain below 42%\ncoverage and were either not targeted by the previous TODO or only partially\naddressed. This plan brings each one individually to 80%+.\n\nModules by current coverage (March 24 baseline, may be slightly higher by now):\n- 0%: core/pg_extension_comparison (68 miss), platforms/azure/config_utils (22),\n platforms/credentials/shared (42)\n- 16%: utils/dependency_validation (83 miss) - w14 targeted 70% but tests are thin\n- 18%: platforms/databricks/dataframe_adapter (120 miss)\n- 19%: core/tuning/generators/doris (93 miss), core/coffeeshop/benchmark (88 miss)\n- 19%: core/tuning/generators/questdb (61 miss)\n- 20%: core/coffeeshop/schema (29 miss)\n- 22%: cli/commands/datagen (32 miss)\n- 23%: platforms/dataframe/pyspark_df (200 miss)\n- 27%: platforms/azure/spark_execution_utils (30 miss)\n- 29%: platforms/databend/__init__ (12 miss)\n- 31%: platforms/motherduck (99 miss)\n- 33%: cli/commands/run_official (33 miss), core/tpcdi/schema_extensions (22 miss)\n- 34%: platforms/bigquery (514 miss) - w21 gained stmts but not to 80%\n- 35%: platforms/databricks/adapter (618 miss) - same\n- 35%: mcp/__init__ (14 miss)\n- 36%: cli/system (17 miss)\n- 37%: platforms/redshift (639 miss)\n- 39%: tpch_skew (27 miss)\n- 40%: cli/commands/tuning_group (133 miss), platforms/snowflake (405 miss)\n- 41%: platforms/athena (360 miss)\n\nTesting philosophy: every test must assert on the actual output value or behavior\n(SQL string content, config field value, error message). No import-only tests,\nno assert-not-None, no mock.called checks in lieu of output assertions.\n\nCloud platform SQL generation tests: always assert the generated SQL string\ncontains specific clauses - COPY FROM S3, DISTRIBUTED BY, PARTITION BY, etc.", "id": "bottom-25-module-coverage", "preserves": ["All existing tests must continue to pass - new tests are additive only", "Cloud adapter tests must not require real credentials - mock at the connection boundary only", "No assert-not-None, assert True, or hasattr-only assertions - every assertion must fail if the code under test is broken", "SQL-generation assertions must check the actual SQL string content, not just that a method was called", "Test file names must describe the behavior being tested, not contain 'coverage'"], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Raise bottom 25 modules (all <42% coverage) to 80%", "verifications": [], "work": [{"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test pg_extension_comparison pure functions (0% \u2192 100%)", "wid": "w1"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test cli/system SystemProfiler + mcp/__init__ lazy exports (35%, 35% \u2192 80%)", "wid": "w10"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test MotherDuck adapter init, config, and mocked connection (31% \u2192 80%)", "wid": "w11"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test credentials/shared interactive prompt with mocked Rich (0% \u2192 80%)", "wid": "w12"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test tuning_group CLI commands via CliRunner (40% \u2192 80%)", "wid": "w13"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Redshift SQL generation - uncover remaining branches (37% \u2192 80%)", "wid": "w14"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Snowflake SQL generation - uncover remaining branches (40% \u2192 80%)", "wid": "w15"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test BigQuery SQL generation - uncover remaining branches (34% \u2192 80%)", "wid": "w16"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Databricks adapter SQL generation - uncover remaining branches (35% \u2192 80%)", "wid": "w17"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Athena SQL generation - uncover remaining branches (41% \u2192 80%)", "wid": "w18"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Azure Synapse SQL generation - uncover remaining branches (54% \u2192 80%)", "wid": "w19"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test config builder pure functions: azure/config_utils + databend/__init__ (0%, 29% \u2192 80%)", "wid": "w2"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test schema constant modules: coffeeshop/schema + tpcdi/schema_extensions (20%, 33% \u2192 80%)", "wid": "w3"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test dependency_validation unit functions with synthetic data (16% \u2192 80%)", "wid": "w4"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CoffeeShop benchmark end-to-end with DuckDB (19% \u2192 80%)", "wid": "w5"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Doris DDL generator SQL output (19% \u2192 80%)", "wid": "w6"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test QuestDB DDL generator SQL output (19% \u2192 80%)", "wid": "w7"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CLI wrapper commands via CliRunner: datagen + run_official (22%, 33% \u2192 80%)", "wid": "w8"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPCHSkew facade validation and property delegation (39% \u2192 80%)", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "hash the full, order-normalized result set.", "dont": "compute the digest over only the row count or a single column", "why": "that reproduces the value-blindness"}, {"INSTEAD": "gate it behind a flag the correctness gate sets.", "dont": "emit the digest on every `benchbox run`", "why": "it adds cost to normal benchmarking and changes the payload for all consumers"}, {"INSTEAD": "require non-SKIP under strict mode.", "dont": "exempt the digest check from strict arming", "why": "a missing digest would then silently disarm the value oracle exactly like the old scale-gated strict block did"}, {"INSTEAD": "reuse _calculate_checksum (digest) and validate_results_checksum (comparison).", "dont": "fork ResultValidator for the bounded gate", "why": "TPC-Havoc depends on it"}, {"INSTEAD": "only add the value axis.", "dont": "re-implement the no-skip/strict/ratchet work from #830", "why": "it is done"}], "approach": "Reuse the existing value-level comparator (validate_results_checksum) and the\npinned reference seed rather than building new machinery. Sequence: (w1) make\nthe runner emit a deterministic digest behind a gate-only flag; (w2) store the\n18 SF=1 reference digests in the expected-results registry and assert them under\nthe same strict-arming as the row-count check; (w3) prove sensitivity with\nplanted cardinality-preserving mutations (shared with cross-surface w10); (w4)\nfix the docs/naming and the CI-scope ratchet; (w5) settle the TPC-DS loose band.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-23T00:00:00Z", "completed_pr": null, "created_at": "2026-06-23T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bounded-correctness-gate-value-oracle", "id": 361, "reason": "Out of scope; breadth is owned by benchmark-correctness-oracle-coverage-map (w1/w2) and the cross-surface gate. This item fixes the value-blindness of the EXISTING bounded tpch/tpcds gate at its existing SF=1 cell.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Breadth of value-digest coverage beyond tpch/tpcds SF1 is explicitly assigned to the open oracle-coverage-map item's mandate. (tracked by benchmark-correctness-oracle-coverage-map)", "summary": "Backfill stored value digests for benchmarks beyond tpch/tpcds, or for scales above SF=1."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bounded-correctness-gate-value-oracle", "id": 362, "reason": "Currently unreachable - pytest sets suite skip attributes consistently with children, and the report is generated in-process to a mktemp file with no external control. Noted as a latent hardening only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Deferral text itself says the gap is currently unreachable; pure latent-hardening note with no concrete trigger.", "summary": "Harden the no-skip JUnit guard to count child elements instead of trusting suite-level attributes."}], "deps": [], "description": "An adversarial review of the bounded correctness gate (#830, reviewed on\ndevelop) reproduced that the gate named \"correctness\" only validates ROW\nCOUNTS, never result VALUES. The hardening in #830 (no-skip JUnit guard,\nstrict-arming hoist, discrimination ratchet) is real and survives attack, but\nit ratchets on cardinality SHAPE, not on detection of a wrong answer.\n\nReproduced facts (all verified by reading code / running the gate on develop):\n\n- `_validate_against_expected_results`\n (tests/integration/test_local_platform_benchmark_matrix.py) calls\n `validator.validate_query_result(actual_row_count=int(query[\"rows\"]), ...)`,\n and that method (benchbox/core/validation/query_validation.py:99) compares only\n counts. All 18 gate-configured TPC-H queries are EXACT-mode, but EXACT means\n \"exact row count\", not \"exact values\"\n (benchbox/core/expected_results/models.py ValidationMode).\n- The result payload the gate consumes carries ONLY a per-query row count\n (`{\"id\", \"ms\", \"rows\"}` - benchbox/core/results/schema.py). It contains no\n values, checksums, or digests, so value validation is not merely absent: it\n is impossible without extending the result contract.\n- Consequence: a value-mutating bug that preserves cardinality ships GREEN. The\n canonical example is TPC-H Q1 (`sum(l_extendedprice*(1-l_discount))`), which\n returns 4 grouped rows no matter how wrong the revenue arithmetic is; the gate\n asserts rows==4 and passes. Wrong rounding, a swapped column, an off-by-one in\n an aggregate, or a wrong but same-cardinality join all pass.\n- Scale: TPC-H/TPC-DS expected results exist ONLY at SF=1\n (benchbox/core/expected_results/loader.py raises for any other scale), and the\n bounded gate pins SF=1 (_select_scale), so even the row-count oracle covers\n exactly one scale.\n- The reference qgen seed is already pinned for the gate (get_reference_seed in\n benchbox/core/tpch/benchmark.py), so at SF=1 the result VALUES are\n deterministic - which is precisely what makes a stored value digest feasible.\n\nThis item adds the missing value axis to the bounded gate so a wrong-but-same-\ncardinality answer is caught, and corrects the docs/naming that oversell the\ngate's current guarantee. It is the value-level complement to the cardinality\nhardening already shipped in testing-approach-oracle-hardening (#830, DONE).", "id": "bounded-correctness-gate-value-oracle", "preserves": ["TPC-Havoc gates that already use validate_results_checksum stay green - reuse, do not fork, the comparator.", "The #830 cardinality hardening is unchanged: no-skip JUnit guard, strict arming, and the discrimination ratchet keep working exactly as today.", "The existing row-count validation stays in place and authoritative; the digest check is additive, never a replacement.", "`benchbox run` default behavior, payload shape, and cost are unchanged when the digest flag is unset (digest emission is gate-only)."], "prior_art": [{"concept": "_project/DONE/main/planning/testing-approach-oracle-hardening.yaml - #830 cardinality hardening this complements; do NOT redo the no-skip/strict/ratchet work, add the value axis it explicitly deferred.", "decision": "reuse", "path": "_project/DONE/main/planning/testing-approach-oracle-hardening.yaml"}, {"concept": "_project/TODO/main/active/cross-surface-oracle-remediation.yaml:w10 - the parallel mutation-sensitivity harness for the cross-surface oracle; share the planted-mutation test approach rather than building a second one.", "decision": "reuse", "path": "_project/TODO/main/active/cross-surface-oracle-remediation.yaml:w10"}, {"concept": "benchbox/core/expected_results/registry.py - the expected-results provider registry where stored reference digests (keyed by benchmark/query_id/SF) would live alongside the existing row-count answers.", "decision": "reuse", "path": "benchbox/core/expected_results/registry.py"}, {"concept": "benchbox/core/results/schema.py / normalizer.py - the result payload contract that today carries only `rows`; a per-query result digest field would be added here (gate-only, behind a flag, to avoid runtime cost on normal runs).", "decision": "reuse", "path": "benchbox/core/results/schema.py"}, {"concept": "benchbox/core/tpch/benchmark.py:35 get_reference_seed - the pinned seed that makes SF=1 result values deterministic; the stored reference digests must be computed at this seed.", "decision": "reuse", "path": "benchbox/core/tpch/benchmark.py:35"}, {"concept": "benchbox/core/tpchavoc/validation.py:229 _calculate_checksum - the EXISTING single-result digest primitive to reuse for w1 (emit a digest); currently private, so promote/expose it rather than inventing a new hash. It already backs the TPC-Havoc gates.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py:229"}, {"concept": "benchbox/core/tpchavoc/validation.py:81 validate_results_checksum - the EXISTING pairwise (original vs variant) checksum comparator to reuse for w2 (compare emitted digest vs stored reference); not a digest emitter itself.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py:81"}, {"concept": "tests/integration/test_local_platform_benchmark_matrix.py:_validate_against_expected_results - the validation seam to extend; keep the row-count check, add the digest check.", "decision": "extend", "path": "tests/integration/test_local_platform_benchmark_matrix.py:_validate_against_expected_results"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/"}, {"kind": "only_modify", "path_glob": "benchbox/core/expected_results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Add value-level validation to the bounded TPC-H/TPC-DS correctness gate (it is row-count-only today)", "verifications": [{"command": "make test-correctness-gate", "description": "Bounded gate validates values and cardinality and is green on a correct tree", "expected": "1 passed, ran=1 skipped=0; both row-count and digest checks evaluated for all 18 queries", "item_id": "bounded-correctness-gate-value-oracle", "last_result": null, "last_run": null, "seq": 1}, {"command": "apply a Q1 aggregate-value mutation, then run the sensitivity test (w3)", "description": "A planted cardinality-preserving value mutation turns the gate RED", "expected": "the gate fails on a value-digest mismatch despite unchanged row count", "item_id": "bounded-correctness-gate-value-oracle", "last_result": null, "last_run": null, "seq": 2}, {"command": "make tpchavoc-equivalence-report && make tpchavoc-dataframe-equivalence-report", "description": "TPC-Havoc value gates remain green (comparator reuse did not regress them)", "expected": "both exit 0 with unchanged output", "item_id": "bounded-correctness-gate-value-oracle", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": [], "notes": "Extend the runner/normalizer so a benchmark run can emit a canonical,\norder-normalized digest of each stream-0 query's full result set, written\ninto the per-query result JSON next to `rows`. Reuse `_calculate_checksum`\n(tpchavoc/validation.py:229, the single-result digest primitive; promote it\nfrom private) so SQL and the gate share one digest definition. Gate it behind\nan env flag (e.g. BENCHBOX_EMIT_RESULT_DIGEST=1) set only by\n`make test-correctness-gate`, so normal `benchbox run` keeps its current\ncost and payload shape.\nCanonicalization must be value-correct, not order-correct: sort full rows\nbefore hashing (matching the existing comparator) so unordered queries are\nstable, and normalize numeric formatting/rounding so a digest is reproducible\nacross DuckDB builds at the pinned seed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit a deterministic per-query result digest into the result payload (gate-only, behind a flag)", "wid": "w1"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": ["w1"], "notes": "Compute the reference digests once at SF=1 with the pinned reference seed,\nstore them via the expected-results registry (keyed by benchmark/query_id/\nscale), and extend `_validate_against_expected_results` to compare the emitted\ndigest to the stored one IN ADDITION to the row count. The digest check must\nbe subject to the SAME strict-arming the row-count check already has (every\nconfigured query must produce a non-SKIP value validation under\nBENCHBOX_STRICT_EXPECTED_RESULTS), so a missing digest disarms RED, not green.\nKeep the change additive: a benchmark/scale without stored digests still\nvalidates row counts and (non-strict) skips the digest, exactly as today.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Store reference value digests for the 18 bounded TPC-H queries at SF=1/pinned seed and assert them in the gate", "wid": "w2"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": ["w2"], "notes": "Add a sensitivity test (coordinate with cross-surface-oracle-remediation w10's\nmutation harness) that injects a value-only error which preserves row count -\ne.g. perturb a Q1 aggregate value, swap two columns of equal cardinality, or\nchange rounding - and asserts the gate goes RED. This converts the\ndiscrimination-ratchet from \"the query set could detect cardinality changes\"\nto \"the gate provably detects a wrong value\", closing the headline review gap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Prove sensitivity: a value mutation that preserves cardinality turns the gate RED", "wid": "w3"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": ["w2"], "notes": "The review found two disclosure defects independent of the value fix:\n - the Makefile note / tests/README / #830 docs describe the gate as\n deliberately NOT proving values; once w2 lands, update them to state the\n gate proves value+cardinality at SF=1/pinned-seed, and that values are\n unguarded above SF=1.\n - the required `correctness-gate` CI job (.github/workflows/pr.yml) actually\n runs SIX gates - test-correctness-gate plus tpchavoc variant, tpchavoc\n DataFrame, and ssb/amplab/coffeeshop cross-surface (all value-level) - but\n the ratchet test (test_standardized_test_commands.py\n test_develop_pr_invokes_bounded_correctness_gate) only asserts\n `make test-correctness-gate` is present, so a reader infers the job is\n row-count-only. Add a ratchet asserting the value-level gate steps are\n present, and document the job's true composition.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct the docs/naming that oversell the gate, including the required CI job's real scope", "wid": "w4"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": [], "notes": "TPC-DS validation runs in LOOSE mode with a +/-50% default tolerance\n(benchbox/core/expected_results/models.py loose_tolerance_percent=50.0), so a\nregression that shifts TPC-DS cardinality by <50% hides. Scope it honestly:\nthis loose band lives in the broad stress matrix, NOT in the bounded\n`make test-correctness-gate` (which is tpch-duckdb only), so it is less\nload-bearing than a reader of the review might assume. Decide per query\nwhether the pinned-seed run is deterministic enough to tighten TPC-DS toward\nEXACT (or RANGE with a tight band), or document why loose is retained, and\nrecord the decision next to the TPC-DS provider.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Quantify and decide the TPC-DS loose +/-50% band", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep them separate PRs/items.", "dont": "bundle the main->release rename into this item", "why": "Decision B has real functional blast radius (verify-tag-on-main, test.yml base_ref gates) and Decision A's value plus low risk should not wait on it"}], "approach": null, "blocked_reason": null, "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-08T00:00:00Z", "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": [], "description": "BRANCHING-STRATEGY REVIEW (2026-07-06, session on PR #993). The repo already\ntreats `develop` as the integration branch \u2014 all PRs target it, the\ndevelop-squash-only ruleset and develop-post-merge auto-revert live there \u2014\nbut GitHub's default_branch is still `main` (confirmed via\n`gh api repos/joeharris76/BenchBox --jq .default_branch` -> \"main\"). This\nitem flips the default to `develop`.\n\nDecision A is deliberately separated from the main->release rename\n(Decision B, see rename-release-branch-main-to-release): A is a low-risk,\nreversible, one-call admin change and captures most of the value on its own.\n\nKEY INTERACTION \u2014 this resolves an open High/In-Progress TODO. GitHub runs\non.schedule workflows ONLY from the default branch's copy of the file.\nrelease-canary.yml lives on develop, not main, so its daily cron has never\nfired (see release-canary-scheduled-activation, whose w1 menu proposes\nlanding the file on main as options (a)/(b)/(c)). Making develop the default\ndissolves that problem: the canary \u2014 and any future scheduled workflow\nauthored on develop \u2014 registers and fires with no per-release re-landing.\nCross-link and re-scope that item when this lands.\n\nVerified safe (documented in the session's branch-rename runbook draft):\nPyPI trusted publishing is branch-independent (keyed on repo + workflow\nfilename release.yml + environment pypi, never a branch), and GitHub Pages\ndeploys via deploy-pages (source = \"GitHub Actions\", not a branch). Neither\nis affected by changing the default branch.", "id": "branch-default-switch-to-develop", "preserves": ["PyPI trusted publishing and GitHub Pages config \u2014 both branch-independent; do not touch them as part of this change.", "main stays release-only and push-restricted regardless of the default-branch setting \u2014 the default flip changes clone/PR/UI defaults and scheduled-workflow source, not the release gating."], "prior_art": [{"concept": ".github/workflows/nightly.yml \u2014 the scheduled workflow currently on main that DOES fire; after the default flips, scheduled workflows fire from develop instead, so confirm nightly still triggers as expected.", "decision": "reuse", "path": ".github/workflows/nightly.yml"}, {"concept": "_project/TODO/main/active/release-canary-scheduled-activation.yaml \u2014 the open item this resolves; its w1 option menu (a/b/c) predates and omits the develop-as-default option. Re-scope its w2/w3 once this lands.", "decision": "reuse", "path": "_project/TODO/main/active/release-canary-scheduled-activation.yaml"}, {"concept": "docs/operations/repo-admin-settings.md \u2014 the admin-state runbook; its 'Scheduled activation' section is written under the main-as-default premise and must be updated to reflect develop-as-default.", "decision": "reuse", "path": "docs/operations/repo-admin-settings.md"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/docs.yml"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml"}, {"kind": "only_modify", "path_glob": "docs/operations/repo-admin-settings.md"}], "state": "done", "title": "Make develop the GitHub default branch (Decision A of the branching-strategy review)", "verifications": [{"command": "gh api repos/joeharris76/BenchBox --jq .default_branch", "description": "Default branch is develop", "expected": "develop", "item_id": "branch-default-switch-to-develop", "last_result": null, "last_run": null, "seq": 1}, {"command": "gh api repos/joeharris76/BenchBox/actions/workflows --jq '.workflows[].path' | grep release-canary", "description": "Scheduled canary now registered", "expected": ".github/workflows/release-canary.yml present", "item_id": "branch-default-switch-to-develop", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "branch-default-switch-to-develop", "needs": [], "notes": "`gh api -X PATCH repos/joeharris76/BenchBox -f default_branch=develop`.\nAdmin-only. Reversible: re-run with default_branch=main to roll back.\nRecord the dated before/after in repo-admin-settings.md.\nDONE 2026-07-08: maintainer ran the PATCH. Verified via MCP\nsearch_repositories -> default_branch=\"develop\" (was \"main\").\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "ADMIN: set default_branch=develop", "wid": "w1"}, {"evidence": null, "item_id": "branch-default-switch-to-develop", "needs": ["w1"], "notes": "After the flip, confirm release-canary.yml is registered:\n`gh api repos/joeharris76/BenchBox/actions/workflows --jq '.workflows[].path'`\nnow includes .github/workflows/release-canary.yml, and\n`gh api .../actions/workflows/release-canary.yml/runs` no longer 404s.\nDispatch once (workflow_dispatch) to prove liveness; expected first\nresult is RED on 0.3.0's broken import (that is the canary working \u2014\nsee release-recovery-v0-3-1). Confirm nightly.yml still fires from the\nnew default.\nDONE 2026-07-08: Actions list-workflows API now returns 26 registered\nworkflows (was 19); Release Canary (id 309070628) is present and active,\nas are phase3-promotion-review and orphaned-commit-detector \u2014 all\npreviously unregistered develop-only scheduled workflows. Registration\n(the blocker) is confirmed. Immediate workflow_dispatch was NOT run to\navoid an unprompted CI run; the daily 08:00 UTC cron now fires on its\nown (expected first result RED on broken 0.3.0 \u2014 cross-link into\nrelease-recovery-v0-3-1 when it lands). Maintainer may dispatch for\ninstant proof if desired.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify scheduled workflows now register and fire from develop", "wid": "w2"}, {"evidence": null, "item_id": "branch-default-switch-to-develop", "needs": ["w1"], "notes": "The 'Scheduled activation of release-canary.yml' section and any text\nasserting 'the default branch is main' become stale. Rewrite to reflect\ndevelop-as-default; several (a)/(b)/(c) workarounds for the canary\nbecome unnecessary. Record the new live state with a date.\nDONE 2026-07-08: added a dated RESOLVED banner to the 'Scheduled\nactivation' section and corrected the 'default branch (main)' assertion\nto note develop-as-default. Conservative edit \u2014 historical analysis and\nstill-live code-path descriptions (bootstrap, RELEASE_CANARY_REF shell,\nliveness guard) preserved rather than rewritten.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update repo-admin-settings.md: the default-branch premise inverts", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "fold this into the shrink-campaign ledger", "why": "_project/goal-shrink-core-code.md scopes structural/god-class-style refactors out of that loop; this is a standalone architecture fix."}, {"INSTEAD": "(not specified)", "dont": "remove the verbosity.py try/except ImportError guard before confirming w1+w2 actually eliminate the cycle", "why": "removing it first will surface the exact ImportError the guard was added to suppress."}, {"INSTEAD": "(not specified)", "dont": "special-case version resolution per install method (editable vs wheel) with duplicated logic", "why": "reuse the existing _PYPROJECT_CACHE/tomllib fallback for the source-tree case."}], "approach": "Start from utils/version.py since it is the more clearly self-contained\nfix (only needs a version string, not general benchbox internals). Decide\nbetween importlib.metadata vs. a leaf benchbox/_version.py by checking\nwhether validate_version_consistency() needs anything else from the\npackage root; if not, importlib.metadata + tomllib fallback avoids adding\na new file. Apply the same fix to datagen_manifest.py once the pattern is\nsettled. Only then touch verbosity.py's guard and the __init__.py comment,\nsince removing them prematurely (before the cycle is actually broken) would\nreintroduce the failure the guard exists to prevent.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-03T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "break-root-import-cycle", "id": 307, "reason": "The review identified verbosity's fan-in as the multiplier, but a full SCC decomposition is a larger effort; this TODO targets the two root-importing leaf modules identified as the actual cycle cause. Re-scope separately if the cycle persists after this fix (see import-layering-enforcement, which adds tooling to detect any residual cycle).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: import-layering-enforcement (DONE) shipped .importlinter with an active ratchet-down mechanism; residual cycle edges now handled incrementally by that tool, not a big-bang SCC breakup.", "summary": "Broader breakup of the ~61-module SCC beyond the version/datagen_manifest edges"}], "deps": [], "description": "Architecture review (Work Area E, structural remediation) found that\n`benchbox/utils/version.py:27` and `benchbox/utils/datagen_manifest.py:23`\nboth do `import benchbox` at module scope (to read `__version__` and to\nresolve project-root paths, respectively). Because\n`benchbox/utils/verbosity.py:12` imports `benchbox.utils.version`, and\neffectively every subsystem imports `verbosity` for its logging mixin, this\nsingle edge pulls a large strongly-connected component (~61 modules per the\nreview) into one cycle rooted at the package `__init__`.\n\nLive symptoms already visible in the code, not just theory:\n\n- `benchbox/__init__.py:14-16` carries a load-order comment: \"`__version__`\n must be defined before `from . import platforms` because the platforms\n import chain can circle back to benchbox.cli.app which reads\n benchbox.__version__ at module level.\" That comment is a tell that the\n root module's initialization order is load-bearing.\n- `benchbox/utils/verbosity.py:11-15` wraps its `benchbox.utils.version`\n import in `try/except ImportError` with the comment \"optional dependency\n during bootstrapping\" - a defensive guard that exists only because the\n cycle can make the import fail depending on where in package\n initialization it is triggered.\n\nThis is a structural-debt item independent of the shrink campaign:\n`_project/goal-shrink-core-code.md` explicitly scopes god-class\ndecompositions and similar risk-trading refactors out of the autonomous\nshrink loop. This TODO is a targeted import-layering fix, not a shrink-loop\nitem, and should not be picked up by that loop.", "id": "break-root-import-cycle", "preserves": ["The lean-import guarantee already enforced by tests/unit/test_import_isolation.py (no heavy third-party modules like pandas/polars/pyarrow/duckdb on `import benchbox`) stays green.", "The wheel's reported version (importlib.metadata / `benchbox --version` equivalent) stays correct for both editable/dev installs and built wheels.", "validate_version_consistency() behavior in benchbox/__init__.py:30-46 (warns via UserWarning on RuntimeError, does not hard-fail import) is unchanged."], "prior_art": [{"concept": "benchbox/utils/version.py:_PYPROJECT_CACHE and PROJECT_ROOT - existing tomllib-based pyproject parsing to reuse for the source-tree fallback rather than adding a second TOML reader.", "decision": "reuse", "path": "benchbox/utils/version.py:_PYPROJECT_CACHE"}, {"concept": "tests/unit/test_import_isolation.py:_run_entrypoint - existing subprocess-isolation pattern to extend for the new w4 regression test, rather than inventing a new import-isolation harness.", "decision": "extend", "path": "tests/unit/test_import_isolation.py:_run_entrypoint"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/_version.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/datagen_manifest.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/verbosity.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/version.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_import_isolation.py"}], "state": "done", "title": "Break benchbox root-package import cycle pulled in by utils/version.py and utils/datagen_manifest.py", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_import_isolation.py -v", "description": "Package still imports cleanly and lean-import isolation stays green", "expected": "all import-isolation tests pass, 0 failures", "item_id": "break-root-import-cycle", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/test_import_isolation.py -k version_no_root_import -v", "description": "New regression test proves utils.version no longer imports the package root", "expected": "test passes", "item_id": "break-root-import-cycle", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c 'import benchbox; print(benchbox.__version__)'", "description": "Version consistency validation still runs at import time without hard-failing", "expected": "prints the current version string with no traceback", "item_id": "break-root-import-cycle", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "break-root-import-cycle", "needs": [], "notes": "Implemented via importlib.metadata.version(\"benchbox\") with a\nsource-tree pyproject.toml fallback (used only on\nPackageNotFoundError), NOT the leaf-module alternative. Reasoning:\nget_package_version() must stay independently sourced from\nget_pyproject_version() so check_version_consistency() can still\ndetect real drift between the two - test_check_version_consistency_detects_mismatch\nmonkeypatches only get_pyproject_version() and expects a mismatch to be\nreported. If get_package_version() had unconditionally preferred the\nsource-tree pyproject.toml (as literally suggested in the TODO body for\nthe \"editable/dev checkout\" case), the two sources would have collapsed\ninto the same call, making that existing test always report\n\"consistent\" - a real regression. importlib.metadata.version(\"benchbox\")\ncurrently returns \"0.2.1\", matching pyproject.toml exactly in this dev\ncheckout, so both tests\n(test_check_version_consistency_matches_pyproject and\ntest_check_version_consistency_detects_mismatch) pass unmodified.\nDid not add benchbox/_version.py - not needed once importlib.metadata\ncovers the installed/dev case, and it would have required updating\nscripts/update_version.py's regex-based `__version__ = \"...\"` target in\nbenchbox/__init__.py, which is outside this TODO's scope_limit.\nREVISED after review: the first cut mapped the \"benchbox.__init__\"\nconsistency source to get_package_version() (dist metadata), which\nsilently dropped coverage of the actual `__version__ = \"...\"` literal\nin benchbox/__init__.py - the value scripts/update_version.py bumps at\nrelease cut and the exact drift class that shipped the 0.3.0 wheel\nwith __version__ = \"0.2.1\". Fixed: added get_init_version() which\nreads the literal STATICALLY from the benchbox/__init__.py source file\n(same regex scripts/update_version.py uses; works inside wheels since\n__init__.py ships; no package-root import, so the cycle stays broken)\nand mapped the \"benchbox.__init__\" source to it. Dist metadata is now\nchecked as its own distinct \"installed-dist\" source (absent-when-not-\ninstalled is exempt from missing-source accounting in both dev and\ninstalled modes - a bare checkout is legitimate - but participates in\nmismatch detection when present). get_package_version() remains\nimportlib.metadata-first (fallback: init literal, then pyproject).\nAdded four tests to tests/unit/utils/test_utils_version_management.py\ncovering the static read, init-literal drift, installed-dist drift,\nand missing-installed-dist tolerance.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop utils/version.py from importing the benchbox package root", "wid": "w1"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w1"], "notes": "Replaced `import benchbox` + `benchbox.__version__` with\n`from benchbox.utils.version import get_package_version` +\n`get_package_version()`. `grep -n \"benchbox\\.\"` on the file post-change\nshows only the new `from benchbox.utils.version import\nget_package_version` line - no remaining package-root reference.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop utils/datagen_manifest.py from importing the benchbox package root", "wid": "w2"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w1", "w2"], "notes": "Converted benchbox/utils/verbosity.py's try/except ImportError around\nthe utils.version import to a plain top-level import (confirmed w1+w2\neliminate the failure mode first, per anti_patterns). Removed the\n\"__version__ must be defined before `from . import platforms`\" comment\nand load-order constraint text from benchbox/__init__.py (the\n`__version__ = \"0.2.1\"` literal itself is unchanged/still first in the\nfile, just without the stale comment describing a hazard that no\nlonger applies now that utils.version doesn't round-trip through the\npackage root).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the verbosity bootstrap guard and the __init__.py load-order comment", "wid": "w3"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w1", "w2", "w3"], "notes": "Deviated from the literal \"assert benchbox.platforms/deep submodule\nabsent from sys.modules\" wording: Python's import machinery always\nfully initializes a parent package before importing any dotted\nsubmodule, so a subprocess doing `import benchbox.utils.version` alone\ncan never observe \"benchbox not fully initialized\" - that part of the\noriginal test proposal is not achievable given CPython import\nsemantics, regardless of this fix. Implemented instead as an AST-based\nstatic check (test_version_no_root_import, parametrized over both\nfiles) asserting neither utils/version.py nor\nutils/datagen_manifest.py contains a module-level `import benchbox`\nstatement - the actual, checkable claim that was broken before and\nfixed by w1/w2. Added in tests/unit/test_import_isolation.py per the\nprior_art pointer (reused _run_entrypoint, generalized with an optional\nheavy_roots param for the separate NYCTaxi test in w5).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test asserting utils.version does not import the package root", "wid": "w4"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w2"], "notes": "Removed the eager `from benchbox.nyctaxi import NYCTaxi`, added\n`\"NYCTaxi\": _BenchmarkSpec(\"nyctaxi\", \"NYCTaxi\", ())` to\n_BENCHMARK_REGISTRY, and fixed the pre-existing (oddly already-present)\n`from benchbox.nyctaxi import NYCTaxi as NYCTaxi_ # noqa: F401` in the\nTYPE_CHECKING block to the plain `from benchbox.nyctaxi import\nNYCTaxi` (it was aliased to an unused underscore name, presumably\nleftover from a prior partial attempt at this same fix). Verified via\nsubprocess that `benchbox.nyctaxi`/`benchbox.core.nyctaxi` are absent\nfrom sys.modules after bare `import benchbox` and only load on first\n`benchbox.NYCTaxi` access.\nDeviation: could NOT verify \"numpy absent from sys.modules after bare\nimport benchbox\" as the TODO's w5 notes suggested - confirmed via\n`python -X importtime` that benchbox.tpch_skew's generator module\n(core/tpch_skew/generator.py) already eagerly imports numpy via a\nseparate, pre-existing eager import at benchbox/__init__.py's `from\nbenchbox.tpch_skew import TPCHSkew`, unrelated to NYCTaxi and out of\nthis TODO's scope_limit (only benchbox/__init__.py's NYCTaxi wiring is\nin scope, not core/tpch_skew/). So numpy remains on the `import\nbenchbox` cold-start path today regardless of this fix - the TODO\nbody's claim that NYCTaxi was \"the one heavy library left on the\ncold-start path\" is stale. Flagging as a follow-up candidate for the\nreviewer / a new TODO scoped to tpch_skew.\nREVISED after review: making NYCTaxi lazy exposed an import-order\nhazard in tests/unit/cli/test_benchmark_hooks.py - its per-test\nregistry snapshot/restore fixture wipes registrations performed by\nmodule imports that first happen INSIDE a test body, and sys.modules\ncaching prevents re-registration, so\nTestRealBenchmarkSpecs::test_nyctaxi_taxi_types_parser went red\n(KeyError 'taxi_types') once `import benchbox` stopped guaranteeing\nnyctaxi's registration. Fixed by adding a module-scoped autouse\nfixture in that test file importing all five benchmark modules whose\nspecs the file reads (nyctaxi, tsbs_devops, tpch_skew, vector_search,\njoinorder - the latter two were always lazy and carried the same\nlatent hazard) before the first per-test snapshot is taken. Grepped\ntests/ for other list_option_specs/_option_specs consumers: no other\nfile combines snapshot/restore with in-test registration imports, and\nall pass in isolation with -n 0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the eager NYCTaxi import in benchbox/__init__.py lazy like the other registry benchmarks", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "assert mock.called for SQL generation tests", "instead": "always assert on the SQL string", "why": "called=True doesn't verify the SQL is correct"}, {"dont": "mock internal helper functions to test a dispatcher", "instead": "use real inputs and assert on real output", "why": "you then test the mock, not the code"}, {"dont": "test only the happy path for CLI wrappers", "instead": "test every exit(1) branch", "why": "error validation is where bugs live"}, {"dont": "test that a config field 'is not None'", "instead": "assert the actual value", "why": "that can never fail once the field is populated"}, {"dont": "write a test that only calls get_available_presets() and asserts len > 0", "instead": "assert the specific preset names", "why": "that can't catch a bug where a preset is removed"}], "approach": null, "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-26T00:00:00Z", "completed_pr": null, "created_at": "2026-03-26T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bottom-25-module-coverage", "id": 304, "reason": "Both require Spark session setup (SparkContext or SparkSession). PySpark tests already have a 1046-line test file at 23% because most tests skip when PySpark is not installed. Meaningful coverage requires a PySpark fixture or the separate dataframe platform CI environment. Cover in the dataframe-platform testing initiative, not here.", "resolution": "promoted", "resolved_item": "platforms-databricks-dataframe-adapter-18-120-miss-and-platf", "resolved_reason": null, "summary": "platforms/databricks/dataframe_adapter (18%, 120 miss) and platforms/dataframe/pyspark_df (23%, 200 miss)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bottom-25-module-coverage", "id": 305, "reason": "Azure Spark utilities require mocking Synapse/Fabric Spark session internals. Low ROI (only 30 missing lines) and fragile mock setup. Cover alongside other Azure Spark work when a real Spark session fixture is available.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly deferred until a real Spark session fixture exists; no such fixture is tracked as available yet.", "summary": "platforms/azure/spark_execution_utils (27%, 30 miss)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bottom-25-module-coverage", "id": 306, "reason": "These are in the omit list of .coveragerc_core and were explicitly deferred in non-dataframe-test-coverage. Still deferred: high effort, niche benchmark, requires complex CDC/SCD fixtures. Cover after 80% overall target is sustained.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Condition was 'after 80% overall coverage target sustained'; Makefile still sets suite-wide threshold at 70%, so condition unmet.", "summary": "TPC-DI ETL modules (core/tpcdi/etl/batch.py 882 miss, transformations.py 1053 miss)"}], "deps": ["non-dataframe-test-coverage"], "description": "After completing non-dataframe-test-coverage, 25 modules remain below 42%\ncoverage and were either not targeted by the previous TODO or only partially\naddressed. This plan brings each one individually to 80%+.\n\nModules by current coverage (March 24 baseline, may be slightly higher by now):\n- 0%: core/pg_extension_comparison (68 miss), platforms/azure/config_utils (22),\n platforms/credentials/shared (42)\n- 16%: utils/dependency_validation (83 miss) - w14 targeted 70% but tests are thin\n- 18%: platforms/databricks/dataframe_adapter (120 miss)\n- 19%: core/tuning/generators/doris (93 miss), core/coffeeshop/benchmark (88 miss)\n- 19%: core/tuning/generators/questdb (61 miss)\n- 20%: core/coffeeshop/schema (29 miss)\n- 22%: cli/commands/datagen (32 miss)\n- 23%: platforms/dataframe/pyspark_df (200 miss)\n- 27%: platforms/azure/spark_execution_utils (30 miss)\n- 29%: platforms/databend/__init__ (12 miss)\n- 31%: platforms/motherduck (99 miss)\n- 33%: cli/commands/run_official (33 miss), core/tpcdi/schema_extensions (22 miss)\n- 34%: platforms/bigquery (514 miss) - w21 gained stmts but not to 80%\n- 35%: platforms/databricks/adapter (618 miss) - same\n- 35%: mcp/__init__ (14 miss)\n- 36%: cli/system (17 miss)\n- 37%: platforms/redshift (639 miss)\n- 39%: tpch_skew (27 miss)\n- 40%: cli/commands/tuning_group (133 miss), platforms/snowflake (405 miss)\n- 41%: platforms/athena (360 miss)\n\nTesting philosophy: every test must assert on the actual output value or behavior\n(SQL string content, config field value, error message). No import-only tests,\nno assert-not-None, no mock.called checks in lieu of output assertions.\n\nCloud platform SQL generation tests: always assert the generated SQL string\ncontains specific clauses - COPY FROM S3, DISTRIBUTED BY, PARTITION BY, etc.", "id": "bottom-25-module-coverage", "preserves": ["All existing tests must continue to pass - new tests are additive only", "Cloud adapter tests must not require real credentials - mock at the connection boundary only", "No assert-not-None, assert True, or hasattr-only assertions - every assertion must fail if the code under test is broken", "SQL-generation assertions must check the actual SQL string content, not just that a method was called", "Test file names must describe the behavior being tested, not contain 'coverage'"], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Raise bottom 25 modules (all <42% coverage) to 80%", "verifications": [], "work": [{"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test pg_extension_comparison pure functions (0% \u2192 100%)", "wid": "w1"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test cli/system SystemProfiler + mcp/__init__ lazy exports (35%, 35% \u2192 80%)", "wid": "w10"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test MotherDuck adapter init, config, and mocked connection (31% \u2192 80%)", "wid": "w11"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test credentials/shared interactive prompt with mocked Rich (0% \u2192 80%)", "wid": "w12"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test tuning_group CLI commands via CliRunner (40% \u2192 80%)", "wid": "w13"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Redshift SQL generation - uncover remaining branches (37% \u2192 80%)", "wid": "w14"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Snowflake SQL generation - uncover remaining branches (40% \u2192 80%)", "wid": "w15"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test BigQuery SQL generation - uncover remaining branches (34% \u2192 80%)", "wid": "w16"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Databricks adapter SQL generation - uncover remaining branches (35% \u2192 80%)", "wid": "w17"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Athena SQL generation - uncover remaining branches (41% \u2192 80%)", "wid": "w18"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Azure Synapse SQL generation - uncover remaining branches (54% \u2192 80%)", "wid": "w19"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test config builder pure functions: azure/config_utils + databend/__init__ (0%, 29% \u2192 80%)", "wid": "w2"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test schema constant modules: coffeeshop/schema + tpcdi/schema_extensions (20%, 33% \u2192 80%)", "wid": "w3"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test dependency_validation unit functions with synthetic data (16% \u2192 80%)", "wid": "w4"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CoffeeShop benchmark end-to-end with DuckDB (19% \u2192 80%)", "wid": "w5"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Doris DDL generator SQL output (19% \u2192 80%)", "wid": "w6"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test QuestDB DDL generator SQL output (19% \u2192 80%)", "wid": "w7"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CLI wrapper commands via CliRunner: datagen + run_official (22%, 33% \u2192 80%)", "wid": "w8"}, {"evidence": null, "item_id": "bottom-25-module-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPCHSkew facade validation and property delegation (39% \u2192 80%)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "compute the digest over only the row count or a single column", "instead": "hash the full, order-normalized result set.", "why": "that reproduces the value-blindness"}, {"dont": "emit the digest on every `benchbox run`", "instead": "gate it behind a flag the correctness gate sets.", "why": "it adds cost to normal benchmarking and changes the payload for all consumers"}, {"dont": "exempt the digest check from strict arming", "instead": "require non-SKIP under strict mode.", "why": "a missing digest would then silently disarm the value oracle exactly like the old scale-gated strict block did"}, {"dont": "fork ResultValidator for the bounded gate", "instead": "reuse _calculate_checksum (digest) and validate_results_checksum (comparison).", "why": "TPC-Havoc depends on it"}, {"dont": "re-implement the no-skip/strict/ratchet work from #830", "instead": "only add the value axis.", "why": "it is done"}], "approach": "Reuse the existing value-level comparator (validate_results_checksum) and the\npinned reference seed rather than building new machinery. Sequence: (w1) make\nthe runner emit a deterministic digest behind a gate-only flag; (w2) store the\n18 SF=1 reference digests in the expected-results registry and assert them under\nthe same strict-arming as the row-count check; (w3) prove sensitivity with\nplanted cardinality-preserving mutations (shared with cross-surface w10); (w4)\nfix the docs/naming and the CI-scope ratchet; (w5) settle the TPC-DS loose band.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-23T00:00:00Z", "completed_pr": null, "created_at": "2026-06-23T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bounded-correctness-gate-value-oracle", "id": 361, "reason": "Out of scope; breadth is owned by benchmark-correctness-oracle-coverage-map (w1/w2) and the cross-surface gate. This item fixes the value-blindness of the EXISTING bounded tpch/tpcds gate at its existing SF=1 cell.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Breadth of value-digest coverage beyond tpch/tpcds SF1 is explicitly assigned to the open oracle-coverage-map item's mandate. (tracked by benchmark-correctness-oracle-coverage-map)", "summary": "Backfill stored value digests for benchmarks beyond tpch/tpcds, or for scales above SF=1."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bounded-correctness-gate-value-oracle", "id": 362, "reason": "Currently unreachable - pytest sets suite skip attributes consistently with children, and the report is generated in-process to a mktemp file with no external control. Noted as a latent hardening only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Deferral text itself says the gap is currently unreachable; pure latent-hardening note with no concrete trigger.", "summary": "Harden the no-skip JUnit guard to count child elements instead of trusting suite-level attributes."}], "deps": [], "description": "An adversarial review of the bounded correctness gate (#830, reviewed on\ndevelop) reproduced that the gate named \"correctness\" only validates ROW\nCOUNTS, never result VALUES. The hardening in #830 (no-skip JUnit guard,\nstrict-arming hoist, discrimination ratchet) is real and survives attack, but\nit ratchets on cardinality SHAPE, not on detection of a wrong answer.\n\nReproduced facts (all verified by reading code / running the gate on develop):\n\n- `_validate_against_expected_results`\n (tests/integration/test_local_platform_benchmark_matrix.py) calls\n `validator.validate_query_result(actual_row_count=int(query[\"rows\"]), ...)`,\n and that method (benchbox/core/validation/query_validation.py:99) compares only\n counts. All 18 gate-configured TPC-H queries are EXACT-mode, but EXACT means\n \"exact row count\", not \"exact values\"\n (benchbox/core/expected_results/models.py ValidationMode).\n- The result payload the gate consumes carries ONLY a per-query row count\n (`{\"id\", \"ms\", \"rows\"}` - benchbox/core/results/schema.py). It contains no\n values, checksums, or digests, so value validation is not merely absent: it\n is impossible without extending the result contract.\n- Consequence: a value-mutating bug that preserves cardinality ships GREEN. The\n canonical example is TPC-H Q1 (`sum(l_extendedprice*(1-l_discount))`), which\n returns 4 grouped rows no matter how wrong the revenue arithmetic is; the gate\n asserts rows==4 and passes. Wrong rounding, a swapped column, an off-by-one in\n an aggregate, or a wrong but same-cardinality join all pass.\n- Scale: TPC-H/TPC-DS expected results exist ONLY at SF=1\n (benchbox/core/expected_results/loader.py raises for any other scale), and the\n bounded gate pins SF=1 (_select_scale), so even the row-count oracle covers\n exactly one scale.\n- The reference qgen seed is already pinned for the gate (get_reference_seed in\n benchbox/core/tpch/benchmark.py), so at SF=1 the result VALUES are\n deterministic - which is precisely what makes a stored value digest feasible.\n\nThis item adds the missing value axis to the bounded gate so a wrong-but-same-\ncardinality answer is caught, and corrects the docs/naming that oversell the\ngate's current guarantee. It is the value-level complement to the cardinality\nhardening already shipped in testing-approach-oracle-hardening (#830, DONE).", "id": "bounded-correctness-gate-value-oracle", "preserves": ["TPC-Havoc gates that already use validate_results_checksum stay green - reuse, do not fork, the comparator.", "The #830 cardinality hardening is unchanged: no-skip JUnit guard, strict arming, and the discrimination ratchet keep working exactly as today.", "The existing row-count validation stays in place and authoritative; the digest check is additive, never a replacement.", "`benchbox run` default behavior, payload shape, and cost are unchanged when the digest flag is unset (digest emission is gate-only)."], "prior_art": [{"concept": "_project/DONE/main/planning/testing-approach-oracle-hardening.yaml - #830 cardinality hardening this complements; do NOT redo the no-skip/strict/ratchet work, add the value axis it explicitly deferred.", "decision": "reuse", "path": "_project/DONE/main/planning/testing-approach-oracle-hardening.yaml"}, {"concept": "_project/TODO/main/active/cross-surface-oracle-remediation.yaml:w10 - the parallel mutation-sensitivity harness for the cross-surface oracle; share the planted-mutation test approach rather than building a second one.", "decision": "reuse", "path": "_project/TODO/main/active/cross-surface-oracle-remediation.yaml:w10"}, {"concept": "benchbox/core/expected_results/registry.py - the expected-results provider registry where stored reference digests (keyed by benchmark/query_id/SF) would live alongside the existing row-count answers.", "decision": "reuse", "path": "benchbox/core/expected_results/registry.py"}, {"concept": "benchbox/core/results/schema.py / normalizer.py - the result payload contract that today carries only `rows`; a per-query result digest field would be added here (gate-only, behind a flag, to avoid runtime cost on normal runs).", "decision": "reuse", "path": "benchbox/core/results/schema.py"}, {"concept": "benchbox/core/tpch/benchmark.py:35 get_reference_seed - the pinned seed that makes SF=1 result values deterministic; the stored reference digests must be computed at this seed.", "decision": "reuse", "path": "benchbox/core/tpch/benchmark.py:35"}, {"concept": "benchbox/core/tpchavoc/validation.py:229 _calculate_checksum - the EXISTING single-result digest primitive to reuse for w1 (emit a digest); currently private, so promote/expose it rather than inventing a new hash. It already backs the TPC-Havoc gates.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py:229"}, {"concept": "benchbox/core/tpchavoc/validation.py:81 validate_results_checksum - the EXISTING pairwise (original vs variant) checksum comparator to reuse for w2 (compare emitted digest vs stored reference); not a digest emitter itself.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py:81"}, {"concept": "tests/integration/test_local_platform_benchmark_matrix.py:_validate_against_expected_results - the validation seam to extend; keep the row-count check, add the digest check.", "decision": "extend", "path": "tests/integration/test_local_platform_benchmark_matrix.py:_validate_against_expected_results"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/"}, {"kind": "only_modify", "path_glob": "benchbox/core/expected_results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Add value-level validation to the bounded TPC-H/TPC-DS correctness gate (it is row-count-only today)", "verifications": [{"command": "make test-correctness-gate", "description": "Bounded gate validates values and cardinality and is green on a correct tree", "expected": "1 passed, ran=1 skipped=0; both row-count and digest checks evaluated for all 18 queries", "item_id": "bounded-correctness-gate-value-oracle", "last_result": null, "last_run": null, "seq": 1}, {"command": "apply a Q1 aggregate-value mutation, then run the sensitivity test (w3)", "description": "A planted cardinality-preserving value mutation turns the gate RED", "expected": "the gate fails on a value-digest mismatch despite unchanged row count", "item_id": "bounded-correctness-gate-value-oracle", "last_result": null, "last_run": null, "seq": 2}, {"command": "make tpchavoc-equivalence-report && make tpchavoc-dataframe-equivalence-report", "description": "TPC-Havoc value gates remain green (comparator reuse did not regress them)", "expected": "both exit 0 with unchanged output", "item_id": "bounded-correctness-gate-value-oracle", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": [], "notes": "Extend the runner/normalizer so a benchmark run can emit a canonical,\norder-normalized digest of each stream-0 query's full result set, written\ninto the per-query result JSON next to `rows`. Reuse `_calculate_checksum`\n(tpchavoc/validation.py:229, the single-result digest primitive; promote it\nfrom private) so SQL and the gate share one digest definition. Gate it behind\nan env flag (e.g. BENCHBOX_EMIT_RESULT_DIGEST=1) set only by\n`make test-correctness-gate`, so normal `benchbox run` keeps its current\ncost and payload shape.\nCanonicalization must be value-correct, not order-correct: sort full rows\nbefore hashing (matching the existing comparator) so unordered queries are\nstable, and normalize numeric formatting/rounding so a digest is reproducible\nacross DuckDB builds at the pinned seed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit a deterministic per-query result digest into the result payload (gate-only, behind a flag)", "wid": "w1"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": ["w1"], "notes": "Compute the reference digests once at SF=1 with the pinned reference seed,\nstore them via the expected-results registry (keyed by benchmark/query_id/\nscale), and extend `_validate_against_expected_results` to compare the emitted\ndigest to the stored one IN ADDITION to the row count. The digest check must\nbe subject to the SAME strict-arming the row-count check already has (every\nconfigured query must produce a non-SKIP value validation under\nBENCHBOX_STRICT_EXPECTED_RESULTS), so a missing digest disarms RED, not green.\nKeep the change additive: a benchmark/scale without stored digests still\nvalidates row counts and (non-strict) skips the digest, exactly as today.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Store reference value digests for the 18 bounded TPC-H queries at SF=1/pinned seed and assert them in the gate", "wid": "w2"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": ["w2"], "notes": "Add a sensitivity test (coordinate with cross-surface-oracle-remediation w10's\nmutation harness) that injects a value-only error which preserves row count -\ne.g. perturb a Q1 aggregate value, swap two columns of equal cardinality, or\nchange rounding - and asserts the gate goes RED. This converts the\ndiscrimination-ratchet from \"the query set could detect cardinality changes\"\nto \"the gate provably detects a wrong value\", closing the headline review gap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Prove sensitivity: a value mutation that preserves cardinality turns the gate RED", "wid": "w3"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": ["w2"], "notes": "The review found two disclosure defects independent of the value fix:\n - the Makefile note / tests/README / #830 docs describe the gate as\n deliberately NOT proving values; once w2 lands, update them to state the\n gate proves value+cardinality at SF=1/pinned-seed, and that values are\n unguarded above SF=1.\n - the required `correctness-gate` CI job (.github/workflows/pr.yml) actually\n runs SIX gates - test-correctness-gate plus tpchavoc variant, tpchavoc\n DataFrame, and ssb/amplab/coffeeshop cross-surface (all value-level) - but\n the ratchet test (test_standardized_test_commands.py\n test_develop_pr_invokes_bounded_correctness_gate) only asserts\n `make test-correctness-gate` is present, so a reader infers the job is\n row-count-only. Add a ratchet asserting the value-level gate steps are\n present, and document the job's true composition.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct the docs/naming that oversell the gate, including the required CI job's real scope", "wid": "w4"}, {"evidence": null, "item_id": "bounded-correctness-gate-value-oracle", "needs": [], "notes": "TPC-DS validation runs in LOOSE mode with a +/-50% default tolerance\n(benchbox/core/expected_results/models.py loose_tolerance_percent=50.0), so a\nregression that shifts TPC-DS cardinality by <50% hides. Scope it honestly:\nthis loose band lives in the broad stress matrix, NOT in the bounded\n`make test-correctness-gate` (which is tpch-duckdb only), so it is less\nload-bearing than a reader of the review might assume. Decide per query\nwhether the pinned-seed run is deterministic enough to tighten TPC-DS toward\nEXACT (or RANGE with a tight band), or document why loose is retained, and\nrecord the decision next to the TPC-DS provider.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Quantify and decide the TPC-DS loose +/-50% band", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "bundle the main->release rename into this item", "instead": "keep them separate PRs/items.", "why": "Decision B has real functional blast radius (verify-tag-on-main, test.yml base_ref gates) and Decision A's value plus low risk should not wait on it"}], "approach": null, "blocked_reason": null, "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-08T00:00:00Z", "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": [], "description": "BRANCHING-STRATEGY REVIEW (2026-07-06, session on PR #993). The repo already\ntreats `develop` as the integration branch \u2014 all PRs target it, the\ndevelop-squash-only ruleset and develop-post-merge auto-revert live there \u2014\nbut GitHub's default_branch is still `main` (confirmed via\n`gh api repos/joeharris76/BenchBox --jq .default_branch` -> \"main\"). This\nitem flips the default to `develop`.\n\nDecision A is deliberately separated from the main->release rename\n(Decision B, see rename-release-branch-main-to-release): A is a low-risk,\nreversible, one-call admin change and captures most of the value on its own.\n\nKEY INTERACTION \u2014 this resolves an open High/In-Progress TODO. GitHub runs\non.schedule workflows ONLY from the default branch's copy of the file.\nrelease-canary.yml lives on develop, not main, so its daily cron has never\nfired (see release-canary-scheduled-activation, whose w1 menu proposes\nlanding the file on main as options (a)/(b)/(c)). Making develop the default\ndissolves that problem: the canary \u2014 and any future scheduled workflow\nauthored on develop \u2014 registers and fires with no per-release re-landing.\nCross-link and re-scope that item when this lands.\n\nVerified safe (documented in the session's branch-rename runbook draft):\nPyPI trusted publishing is branch-independent (keyed on repo + workflow\nfilename release.yml + environment pypi, never a branch), and GitHub Pages\ndeploys via deploy-pages (source = \"GitHub Actions\", not a branch). Neither\nis affected by changing the default branch.", "id": "branch-default-switch-to-develop", "preserves": ["PyPI trusted publishing and GitHub Pages config \u2014 both branch-independent; do not touch them as part of this change.", "main stays release-only and push-restricted regardless of the default-branch setting \u2014 the default flip changes clone/PR/UI defaults and scheduled-workflow source, not the release gating."], "prior_art": [{"concept": ".github/workflows/nightly.yml \u2014 the scheduled workflow currently on main that DOES fire; after the default flips, scheduled workflows fire from develop instead, so confirm nightly still triggers as expected.", "decision": "reuse", "path": ".github/workflows/nightly.yml"}, {"concept": "_project/TODO/main/active/release-canary-scheduled-activation.yaml \u2014 the open item this resolves; its w1 option menu (a/b/c) predates and omits the develop-as-default option. Re-scope its w2/w3 once this lands.", "decision": "reuse", "path": "_project/TODO/main/active/release-canary-scheduled-activation.yaml"}, {"concept": "docs/operations/repo-admin-settings.md \u2014 the admin-state runbook; its 'Scheduled activation' section is written under the main-as-default premise and must be updated to reflect develop-as-default.", "decision": "reuse", "path": "docs/operations/repo-admin-settings.md"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/docs.yml"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml"}, {"kind": "only_modify", "path_glob": "docs/operations/repo-admin-settings.md"}], "state": "done", "title": "Make develop the GitHub default branch (Decision A of the branching-strategy review)", "verifications": [{"command": "gh api repos/joeharris76/BenchBox --jq .default_branch", "description": "Default branch is develop", "expected": "develop", "item_id": "branch-default-switch-to-develop", "last_result": null, "last_run": null, "seq": 1}, {"command": "gh api repos/joeharris76/BenchBox/actions/workflows --jq '.workflows[].path' | grep release-canary", "description": "Scheduled canary now registered", "expected": ".github/workflows/release-canary.yml present", "item_id": "branch-default-switch-to-develop", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "branch-default-switch-to-develop", "needs": [], "notes": "`gh api -X PATCH repos/joeharris76/BenchBox -f default_branch=develop`.\nAdmin-only. Reversible: re-run with default_branch=main to roll back.\nRecord the dated before/after in repo-admin-settings.md.\nDONE 2026-07-08: maintainer ran the PATCH. Verified via MCP\nsearch_repositories -> default_branch=\"develop\" (was \"main\").\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "ADMIN: set default_branch=develop", "wid": "w1"}, {"evidence": null, "item_id": "branch-default-switch-to-develop", "needs": ["w1"], "notes": "After the flip, confirm release-canary.yml is registered:\n`gh api repos/joeharris76/BenchBox/actions/workflows --jq '.workflows[].path'`\nnow includes .github/workflows/release-canary.yml, and\n`gh api .../actions/workflows/release-canary.yml/runs` no longer 404s.\nDispatch once (workflow_dispatch) to prove liveness; expected first\nresult is RED on 0.3.0's broken import (that is the canary working \u2014\nsee release-recovery-v0-3-1). Confirm nightly.yml still fires from the\nnew default.\nDONE 2026-07-08: Actions list-workflows API now returns 26 registered\nworkflows (was 19); Release Canary (id 309070628) is present and active,\nas are phase3-promotion-review and orphaned-commit-detector \u2014 all\npreviously unregistered develop-only scheduled workflows. Registration\n(the blocker) is confirmed. Immediate workflow_dispatch was NOT run to\navoid an unprompted CI run; the daily 08:00 UTC cron now fires on its\nown (expected first result RED on broken 0.3.0 \u2014 cross-link into\nrelease-recovery-v0-3-1 when it lands). Maintainer may dispatch for\ninstant proof if desired.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify scheduled workflows now register and fire from develop", "wid": "w2"}, {"evidence": null, "item_id": "branch-default-switch-to-develop", "needs": ["w1"], "notes": "The 'Scheduled activation of release-canary.yml' section and any text\nasserting 'the default branch is main' become stale. Rewrite to reflect\ndevelop-as-default; several (a)/(b)/(c) workarounds for the canary\nbecome unnecessary. Record the new live state with a date.\nDONE 2026-07-08: added a dated RESOLVED banner to the 'Scheduled\nactivation' section and corrected the 'default branch (main)' assertion\nto note develop-as-default. Conservative edit \u2014 historical analysis and\nstill-live code-path descriptions (bootstrap, RELEASE_CANARY_REF shell,\nliveness guard) preserved rather than rewritten.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update repo-admin-settings.md: the default-branch premise inverts", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fold this into the shrink-campaign ledger", "instead": "(not specified)", "why": "_project/goal-shrink-core-code.md scopes structural/god-class-style refactors out of that loop; this is a standalone architecture fix."}, {"dont": "remove the verbosity.py try/except ImportError guard before confirming w1+w2 actually eliminate the cycle", "instead": "(not specified)", "why": "removing it first will surface the exact ImportError the guard was added to suppress."}, {"dont": "special-case version resolution per install method (editable vs wheel) with duplicated logic", "instead": "(not specified)", "why": "reuse the existing _PYPROJECT_CACHE/tomllib fallback for the source-tree case."}], "approach": "Start from utils/version.py since it is the more clearly self-contained\nfix (only needs a version string, not general benchbox internals). Decide\nbetween importlib.metadata vs. a leaf benchbox/_version.py by checking\nwhether validate_version_consistency() needs anything else from the\npackage root; if not, importlib.metadata + tomllib fallback avoids adding\na new file. Apply the same fix to datagen_manifest.py once the pattern is\nsettled. Only then touch verbosity.py's guard and the __init__.py comment,\nsince removing them prematurely (before the cycle is actually broken) would\nreintroduce the failure the guard exists to prevent.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-03T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "break-root-import-cycle", "id": 307, "reason": "The review identified verbosity's fan-in as the multiplier, but a full SCC decomposition is a larger effort; this TODO targets the two root-importing leaf modules identified as the actual cycle cause. Re-scope separately if the cycle persists after this fix (see import-layering-enforcement, which adds tooling to detect any residual cycle).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: import-layering-enforcement (DONE) shipped .importlinter with an active ratchet-down mechanism; residual cycle edges now handled incrementally by that tool, not a big-bang SCC breakup.", "summary": "Broader breakup of the ~61-module SCC beyond the version/datagen_manifest edges"}], "deps": [], "description": "Architecture review (Work Area E, structural remediation) found that\n`benchbox/utils/version.py:27` and `benchbox/utils/datagen_manifest.py:23`\nboth do `import benchbox` at module scope (to read `__version__` and to\nresolve project-root paths, respectively). Because\n`benchbox/utils/verbosity.py:12` imports `benchbox.utils.version`, and\neffectively every subsystem imports `verbosity` for its logging mixin, this\nsingle edge pulls a large strongly-connected component (~61 modules per the\nreview) into one cycle rooted at the package `__init__`.\n\nLive symptoms already visible in the code, not just theory:\n\n- `benchbox/__init__.py:14-16` carries a load-order comment: \"`__version__`\n must be defined before `from . import platforms` because the platforms\n import chain can circle back to benchbox.cli.app which reads\n benchbox.__version__ at module level.\" That comment is a tell that the\n root module's initialization order is load-bearing.\n- `benchbox/utils/verbosity.py:11-15` wraps its `benchbox.utils.version`\n import in `try/except ImportError` with the comment \"optional dependency\n during bootstrapping\" - a defensive guard that exists only because the\n cycle can make the import fail depending on where in package\n initialization it is triggered.\n\nThis is a structural-debt item independent of the shrink campaign:\n`_project/goal-shrink-core-code.md` explicitly scopes god-class\ndecompositions and similar risk-trading refactors out of the autonomous\nshrink loop. This TODO is a targeted import-layering fix, not a shrink-loop\nitem, and should not be picked up by that loop.", "id": "break-root-import-cycle", "preserves": ["The lean-import guarantee already enforced by tests/unit/test_import_isolation.py (no heavy third-party modules like pandas/polars/pyarrow/duckdb on `import benchbox`) stays green.", "The wheel's reported version (importlib.metadata / `benchbox --version` equivalent) stays correct for both editable/dev installs and built wheels.", "validate_version_consistency() behavior in benchbox/__init__.py:30-46 (warns via UserWarning on RuntimeError, does not hard-fail import) is unchanged."], "prior_art": [{"concept": "benchbox/utils/version.py:_PYPROJECT_CACHE and PROJECT_ROOT - existing tomllib-based pyproject parsing to reuse for the source-tree fallback rather than adding a second TOML reader.", "decision": "reuse", "path": "benchbox/utils/version.py:_PYPROJECT_CACHE"}, {"concept": "tests/unit/test_import_isolation.py:_run_entrypoint - existing subprocess-isolation pattern to extend for the new w4 regression test, rather than inventing a new import-isolation harness.", "decision": "extend", "path": "tests/unit/test_import_isolation.py:_run_entrypoint"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/_version.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/datagen_manifest.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/verbosity.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/version.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_import_isolation.py"}], "state": "done", "title": "Break benchbox root-package import cycle pulled in by utils/version.py and utils/datagen_manifest.py", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_import_isolation.py -v", "description": "Package still imports cleanly and lean-import isolation stays green", "expected": "all import-isolation tests pass, 0 failures", "item_id": "break-root-import-cycle", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/test_import_isolation.py -k version_no_root_import -v", "description": "New regression test proves utils.version no longer imports the package root", "expected": "test passes", "item_id": "break-root-import-cycle", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c 'import benchbox; print(benchbox.__version__)'", "description": "Version consistency validation still runs at import time without hard-failing", "expected": "prints the current version string with no traceback", "item_id": "break-root-import-cycle", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "break-root-import-cycle", "needs": [], "notes": "Implemented via importlib.metadata.version(\"benchbox\") with a\nsource-tree pyproject.toml fallback (used only on\nPackageNotFoundError), NOT the leaf-module alternative. Reasoning:\nget_package_version() must stay independently sourced from\nget_pyproject_version() so check_version_consistency() can still\ndetect real drift between the two - test_check_version_consistency_detects_mismatch\nmonkeypatches only get_pyproject_version() and expects a mismatch to be\nreported. If get_package_version() had unconditionally preferred the\nsource-tree pyproject.toml (as literally suggested in the TODO body for\nthe \"editable/dev checkout\" case), the two sources would have collapsed\ninto the same call, making that existing test always report\n\"consistent\" - a real regression. importlib.metadata.version(\"benchbox\")\ncurrently returns \"0.2.1\", matching pyproject.toml exactly in this dev\ncheckout, so both tests\n(test_check_version_consistency_matches_pyproject and\ntest_check_version_consistency_detects_mismatch) pass unmodified.\nDid not add benchbox/_version.py - not needed once importlib.metadata\ncovers the installed/dev case, and it would have required updating\nscripts/update_version.py's regex-based `__version__ = \"...\"` target in\nbenchbox/__init__.py, which is outside this TODO's scope_limit.\nREVISED after review: the first cut mapped the \"benchbox.__init__\"\nconsistency source to get_package_version() (dist metadata), which\nsilently dropped coverage of the actual `__version__ = \"...\"` literal\nin benchbox/__init__.py - the value scripts/update_version.py bumps at\nrelease cut and the exact drift class that shipped the 0.3.0 wheel\nwith __version__ = \"0.2.1\". Fixed: added get_init_version() which\nreads the literal STATICALLY from the benchbox/__init__.py source file\n(same regex scripts/update_version.py uses; works inside wheels since\n__init__.py ships; no package-root import, so the cycle stays broken)\nand mapped the \"benchbox.__init__\" source to it. Dist metadata is now\nchecked as its own distinct \"installed-dist\" source (absent-when-not-\ninstalled is exempt from missing-source accounting in both dev and\ninstalled modes - a bare checkout is legitimate - but participates in\nmismatch detection when present). get_package_version() remains\nimportlib.metadata-first (fallback: init literal, then pyproject).\nAdded four tests to tests/unit/utils/test_utils_version_management.py\ncovering the static read, init-literal drift, installed-dist drift,\nand missing-installed-dist tolerance.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop utils/version.py from importing the benchbox package root", "wid": "w1"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w1"], "notes": "Replaced `import benchbox` + `benchbox.__version__` with\n`from benchbox.utils.version import get_package_version` +\n`get_package_version()`. `grep -n \"benchbox\\.\"` on the file post-change\nshows only the new `from benchbox.utils.version import\nget_package_version` line - no remaining package-root reference.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop utils/datagen_manifest.py from importing the benchbox package root", "wid": "w2"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w1", "w2"], "notes": "Converted benchbox/utils/verbosity.py's try/except ImportError around\nthe utils.version import to a plain top-level import (confirmed w1+w2\neliminate the failure mode first, per anti_patterns). Removed the\n\"__version__ must be defined before `from . import platforms`\" comment\nand load-order constraint text from benchbox/__init__.py (the\n`__version__ = \"0.2.1\"` literal itself is unchanged/still first in the\nfile, just without the stale comment describing a hazard that no\nlonger applies now that utils.version doesn't round-trip through the\npackage root).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the verbosity bootstrap guard and the __init__.py load-order comment", "wid": "w3"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w1", "w2", "w3"], "notes": "Deviated from the literal \"assert benchbox.platforms/deep submodule\nabsent from sys.modules\" wording: Python's import machinery always\nfully initializes a parent package before importing any dotted\nsubmodule, so a subprocess doing `import benchbox.utils.version` alone\ncan never observe \"benchbox not fully initialized\" - that part of the\noriginal test proposal is not achievable given CPython import\nsemantics, regardless of this fix. Implemented instead as an AST-based\nstatic check (test_version_no_root_import, parametrized over both\nfiles) asserting neither utils/version.py nor\nutils/datagen_manifest.py contains a module-level `import benchbox`\nstatement - the actual, checkable claim that was broken before and\nfixed by w1/w2. Added in tests/unit/test_import_isolation.py per the\nprior_art pointer (reused _run_entrypoint, generalized with an optional\nheavy_roots param for the separate NYCTaxi test in w5).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test asserting utils.version does not import the package root", "wid": "w4"}, {"evidence": null, "item_id": "break-root-import-cycle", "needs": ["w2"], "notes": "Removed the eager `from benchbox.nyctaxi import NYCTaxi`, added\n`\"NYCTaxi\": _BenchmarkSpec(\"nyctaxi\", \"NYCTaxi\", ())` to\n_BENCHMARK_REGISTRY, and fixed the pre-existing (oddly already-present)\n`from benchbox.nyctaxi import NYCTaxi as NYCTaxi_ # noqa: F401` in the\nTYPE_CHECKING block to the plain `from benchbox.nyctaxi import\nNYCTaxi` (it was aliased to an unused underscore name, presumably\nleftover from a prior partial attempt at this same fix). Verified via\nsubprocess that `benchbox.nyctaxi`/`benchbox.core.nyctaxi` are absent\nfrom sys.modules after bare `import benchbox` and only load on first\n`benchbox.NYCTaxi` access.\nDeviation: could NOT verify \"numpy absent from sys.modules after bare\nimport benchbox\" as the TODO's w5 notes suggested - confirmed via\n`python -X importtime` that benchbox.tpch_skew's generator module\n(core/tpch_skew/generator.py) already eagerly imports numpy via a\nseparate, pre-existing eager import at benchbox/__init__.py's `from\nbenchbox.tpch_skew import TPCHSkew`, unrelated to NYCTaxi and out of\nthis TODO's scope_limit (only benchbox/__init__.py's NYCTaxi wiring is\nin scope, not core/tpch_skew/). So numpy remains on the `import\nbenchbox` cold-start path today regardless of this fix - the TODO\nbody's claim that NYCTaxi was \"the one heavy library left on the\ncold-start path\" is stale. Flagging as a follow-up candidate for the\nreviewer / a new TODO scoped to tpch_skew.\nREVISED after review: making NYCTaxi lazy exposed an import-order\nhazard in tests/unit/cli/test_benchmark_hooks.py - its per-test\nregistry snapshot/restore fixture wipes registrations performed by\nmodule imports that first happen INSIDE a test body, and sys.modules\ncaching prevents re-registration, so\nTestRealBenchmarkSpecs::test_nyctaxi_taxi_types_parser went red\n(KeyError 'taxi_types') once `import benchbox` stopped guaranteeing\nnyctaxi's registration. Fixed by adding a module-scoped autouse\nfixture in that test file importing all five benchmark modules whose\nspecs the file reads (nyctaxi, tsbs_devops, tpch_skew, vector_search,\njoinorder - the latter two were always lazy and carried the same\nlatent hazard) before the first per-test snapshot is taken. Grepped\ntests/ for other list_option_specs/_option_specs consumers: no other\nfile combines snapshot/restore with in-test registration imports, and\nall pass in isolation with -n 0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the eager NYCTaxi import in benchbox/__init__.py lazy like the other registry benchmarks", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:07Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #141 of databricks-tpc-liquid-clustering-profile-rendering.\nDeferred: Build a comparison-UI cohort warning for cross-mechanism tuned results\nReason deferred: Metadata must land first so the UI warning has a stable contract to consume.", "id": "build-a-comparison-ui-cohort-warning-for-cross-mechanism-tun", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Build a comparison-UI cohort warning for cross-mechanism tuned results", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "make the underlying bundle downloadable from result pages", "dont": "hide the canonical raw bundle behind a UI-only representation", "why": "reproducibility matters"}, {"INSTEAD": "use real result paths or pre-rendered files", "dont": "use hash-only routing for canonical public result URLs", "why": "stable shareable pages matter for a benchmark site"}], "approach": "Build the explorer as a standalone subsite under the BenchBox public domain.\nPrefer static or precomputed routes for canonical result pages, then add\nricher browser-side interactivity on top of the derived read model.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-results-platform-product-and-launch-strategy"], "description": "BenchBox needs a public-facing results explorer that feels like a product, not\na directory listing of JSON files. The explorer should live at\n`benchbox.dev/results/`, integrate with the existing public site, and provide\nstable result pages, browse flows, and compare surfaces over derived\npublic read models.\n\n**Phase assignment:** This is the Phase 1 core deliverable. It does NOT depend\non the hosted governance model or full ingest architecture - Phase 1 uses a\nstatic build pipeline (canonical bundles \u2192 JSON manifests + DuckDB snapshot)\ndeployed to GitHub Pages. The governance and ingest TODOs were removed as\nblockers because Phase 1 is entirely maintainer-curated with no hosted services.\n\nThis TODO is a **tracking umbrella** for the Phase 1 implementation. It does\nnot contain implementation work itself - it mirrors the milestone state of the\nresults-explorer cluster. Mark each work unit done when the corresponding\ncluster item(s) are completed. Mark this item Completed when w3 is done.\n\nNote: resolve-results-explorer-brand-ownership must be resolved before launch\nbut does not block implementation. BenchBox placement is the working default.", "id": "build-results-explorer-subsite-on-benchbox-dev", "preserves": ["Explorer result pages link back to the canonical raw bundle and do not invent a second source of truth", "Landing page, docs, and blog remain deployable under the existing GitHub Pages workflow until the new explorer is proven"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Build a results explorer subsite on benchbox.dev", "verifications": [{"command": "test -d results-explorer/dist/ && test -f results-explorer/dist/index.html && echo PRESENT", "description": "Explorer Vite build output exists with an index", "expected": "PRESENT", "item_id": "build-results-explorer-subsite-on-benchbox-dev", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd landing && npm run build 2>&1 | grep -c '[Ee]rror' || echo 0", "description": "Existing docs and landing pages build cleanly after explorer is integrated", "expected": "0", "item_id": "build-results-explorer-subsite-on-benchbox-dev", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "build-results-explorer-subsite-on-benchbox-dev", "needs": [], "notes": "Tracked by: generate-seed-corpus-for-results-explorer,\nbuild-static-pipeline-bundles-to-read-model\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Milestone: seed corpus generated and build pipeline operational", "wid": "w1"}, {"evidence": null, "item_id": "build-results-explorer-subsite-on-benchbox-dev", "needs": ["w1"], "notes": "Tracked by: scaffold-results-explorer-frontend,\nimplement-explorer-home-and-browse-pages,\nimplement-result-detail-page,\nimplement-results-compare-view\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Milestone: frontend scaffold and all explorer pages implemented", "wid": "w2"}, {"evidence": null, "item_id": "build-results-explorer-subsite-on-benchbox-dev", "needs": ["w2"], "notes": "Tracked by: integrate-explorer-into-github-pages-deployment,\nlaunch-results-explorer-acceptance-and-seed\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Milestone: explorer deployed to GitHub Pages and acceptance-tested", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "gate with per-adapter snapshot diff sign-off", "dont": "claim slice w6 is 'zero behavior change'", "why": "dialect_utils.translate_sql_query applies identify-policy, SQLite fixups, and post-processors that the bare sqlglot.transpile in DialectTranslationMixin does not"}, {"INSTEAD": "derive the phase list from w2 inventory and freeze it in the ADR", "dont": "decree the phase list upfront", "why": "the reviewer proposed phases (dataframe_filter, session_policy as a phase, post_translate) mix phases with actions"}, {"INSTEAD": "onboard them explicitly in their own slice", "dont": "leave vector_search/queries.py in files_affected without an owning work unit", "why": "QUERY_VARIANTS plus platform-version gates are a distinct storage/decision shape"}, {"INSTEAD": "ship behind BENCHBOX_COMPAT_REGISTRY with dual-run telemetry", "dont": "migrate without a reversible feature flag", "why": "four slices touching ~15 sites across the codebase have no safe single-revert path"}, {"INSTEAD": "use structured enforcement + conditions + failure_mode", "dont": "model capabilities as bare booleans (e.g., pk.uniqueness_enforced=True)", "why": "StarRocks silently ignores PKs that violate the first-N-columns rule, which a boolean cannot express"}, {"INSTEAD": "prefer AST; fall back to session policy if sqlglot cannot express the rewrite safely", "dont": "revive the ClickHouse add_subquery_aliases regex rewrite", "why": "it corrupts Q23/Q87 parentheses and is disabled in production for that reason"}, {"INSTEAD": "gate on 'no unregistered-or-unexempted compatibility decision remains' instead", "dont": "ship 'no inline dialect == remains' as the acceptance criterion", "why": "legitimate type-mapping branches in nyctaxi/schema.py:253 and tsbs_devops/schema.py:174 are not compatibility policy"}, {"INSTEAD": "the benchmark_gate phase must back it via registry view", "dont": "treat unsupported_benchmarks as dead metadata", "why": "it is enforced in CLI preflight at cli/commands/run.py:804"}, {"INSTEAD": "find the real site in slice w11 before registering the rule", "dont": "trust the reviewer's tpcdi/benchmark.py:374 citation without verifying", "why": "that line is a get_queries accessor, not a post-translation override"}, {"INSTEAD": "the engine must sit BEHIND the existing managers", "dont": "unify query-storage formats in this workstream", "why": "it conflates decision centralization with data-model churn and the dict-based managers have no blocking semantics today"}], "approach": "Phase-first, not query-first. Rule engine plugs in BEHIND existing\nquery-storage managers (catalog, dict-based, QUERY_VARIANTS) - no\nstorage-format migration. Typed CompatibilityContext + CompatibilityDecision\nwith enumerated support_level and failure_mode. Eight-action vocabulary\ncovering AST, regex-with-issue-link, session policy, variant selection,\nDDL rewrite, benchmark gate, skip, and native passthrough. @compat_local\ndecorator + linter provides attribution so legitimate local rendering\n(type mapping, storage layout) is not swept into the refactor. Migration\nvia feature-flagged dual-run harness with divergence telemetry; flip to\nauthoritative only after divergence reaches zero per slice.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-20T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox's SQL compatibility decisions are currently scattered across five\nstructurally different sites: two bare-sqlglot translation cores with divergent\npolicy (benchbox/utils/dialect_utils.py:108 vs\nbenchbox/platforms/base/dialect_translation.py:33), hardcoded dialect tuples\ninside primitives benchmarks (core/write_primitives/benchmark.py:171 and\ncore/transaction_primitives/benchmark.py:147), a _supports_primary_keys()\nhelper whose body contradicts the inline callers\n(core/write_primitives/schema.py:352), unconditional PRIMARY KEY emission in\nschema helpers (core/transaction_primitives/schema.py:130), regex-based\nadapter transformers disabled in production due to SQL corruption\n(platforms/clickhouse/query_transformer.py:47, routed around with a session\nsetting), benchmark get_queries() overrides patching in class-constant SQL\nvariants (core/h2odb/benchmark.py:106-141), plus scattered \"dialect ==\" checks\nin schema generators (core/tsbs_devops/schema.py, core/nyctaxi/schema.py).\n\nResult: the same capability (e.g., \"does this platform enforce PK\nuniqueness?\") has three answers depending on which site asks, and some of\nthose answers are silent-corruption failure modes - StarRocks, for instance,\naccepts PRIMARY KEY DDL but silently ignores PKs whose columns are not the\nfirst N columns of the table.\n\nThis TODO builds a phase-aware compatibility subsystem (benchbox/sql_compat/)\nthat centralizes decision ownership across every compatibility decision point\nsurfaced by the inventory in w2. The final phase taxonomy is NOT frozen here:\nw1 drafts the ADR with a provisional taxonomy, then w2 derives the final\ntaxonomy from the repo and the ADR is updated before engine work proceeds. Likely\ncandidates include benchmark_gate, query_source, query_compile,\nquery_adapter, schema_emit, ddl_optimize, execution_filter, and\ndataframe_filter. Session policy is intentionally treated as an action, not a\nphase, unless inventory proves a separate phase is required. Decisions are\nmade by a typed rule engine returning typed CompatibilityDecision objects\nwith explicit support_level, failure_mode, rule_id, and action payload.\nExisting query-storage shapes (catalog-backed, dict-based managers,\nQUERY_VARIANTS) remain in place - the engine plugs in behind each manager as\na resolver. Storage-format unification is an explicit non-goal of this\nworkstream and is deferred to a future ADR.\n\nThe migration lands in staged, reversible work units behind a\nBENCHBOX_COMPAT_REGISTRY feature flag with a dual-run harness\n(legacy + registry in parallel, divergences logged), cut over only after\ndivergence reaches zero. A\n@compat_local decorator provides an attribution mechanism so a linter can\ndistinguish legitimate platform-specific rendering (type mapping, storage\nlayout) from unregistered compatibility policy. The linter lands warn-only\nin the early inventory slice and is promoted to error during the\nschema-generation cleanup slice before cutover.\n\nSee the ADR deliverable in w1 for the full design: phase taxonomy derivation\nprotocol, action enum, CompatibilityContext / CompatibilityDecision\ndataclasses, rule_id format, baseline schema v1, dual-run harness spec,\nDataFrame pipeline head design, and the storage-unification non-goal.\n\nDDL FOLLOW-ON: Adapter-owned CREATE TABLE rewrites (ClickHouse, Databricks,\nDatabend, Firebolt, Redshift, Azure Synapse, Snowflake, Presto, Trino, Spark,\nLakesail, Velox) are tracked in `centralize-ddl-translation-phase`, which\nruns in parallel with remaining items here. Those shadow harnesses register\nREWRITE_DDL decisions while keeping the legacy _optimize_table_definition()\npath authoritative until this item's w16 cutover. Fabric Warehouse is\nexcluded - its DDL hook is operational (schema-prefix injection via\nself.schema), not a compat rewrite; see ADR section \"DDL Centralization\nFollow-On\" for the full decision record.", "id": "build-sql-compat-phase-aware-pipeline", "preserves": ["CLI preflight in cli/commands/run.py:804 continues to fail-fast on unsupported (platform, benchmark) pairs", "ClickHouse Q23/Q87 continue to run successfully (do not revive the corrupting regex)", "DataFrame-mode skip decisions remain distinct from SQL-mode skip decisions (two pipeline heads share the registry, not the actions)", "H2O Q9 ClickHouse and StarRocks variants continue to produce identical results to the current class-constant SQL", "SQL/DataFrame engines that do not go through sqlglot (Polars, Pandas) retain their existing code paths", "StarRocks first-N-columns PK rule is expressed structurally and prevents silent data divergence", "Vector Search QUERY_VARIANTS and version-gated behavior continue to work without forcing sqlglot into the path"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/compat/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/coffeeshop/"}, {"kind": "only_modify", "path_glob": "benchbox/core/h2odb/"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_catalog_base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tsbs_devops/"}, {"kind": "only_modify", "path_glob": "benchbox/core/vector_search/"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/dialect_translation.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion_query_transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/starrocks/"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "benchbox/utils/dialect_utils.py"}, {"kind": "only_modify", "path_glob": "docs/compat/"}, {"kind": "only_modify", "path_glob": "docs/development/adr/adr-sql-compat-phase-aware-pipeline.md"}, {"kind": "only_modify", "path_glob": "scripts/compat_lint.py"}, {"kind": "only_modify", "path_glob": "scripts/generate_compat_docs.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}, {"kind": "only_modify", "path_glob": "tests/unit/sql_compat/"}], "state": "done", "title": "Build sql_compat phase-aware compatibility pipeline (centralized decision ownership)", "verifications": [{"command": "test -f docs/development/adr/adr-sql-compat-phase-aware-pipeline.md && echo PRESENT", "description": "ADR committed before implementation slices begin", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 1}, {"command": "test -s _project/compat/inventory.jsonl && echo PRESENT", "description": "Compatibility inventory artifact present", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -E 'cli/commands/run.py|core/vector_search/queries.py' _project/compat/inventory.jsonl | wc -l | tr -d ' '", "description": "Inventory captures both benchmark_gate and QUERY_VARIANTS decision sites before the ADR is frozen", "expected": "2", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 3}, {"command": "test -s _project/compat/baseline.v1.jsonl && echo PRESENT", "description": "Baseline snapshot present and schema-versioned", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 4}, {"command": "for m in context.py decision.py actions.py registry.py resolver.py plan.py; do test -f benchbox/sql_compat/$m || exit 1; done && echo PRESENT", "description": "sql_compat package exists with required modules", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 5}, {"command": "BENCHBOX_COMPAT_REGISTRY=on uv run -- python -m pytest -m fast -q", "description": "Full fast test suite passes under registry-authoritative mode", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 6}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_shadow_parity.py -q", "description": "Shadow-mode parity smoke passes once the dual-run harness lands", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 7}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_primary_key_capabilities.py -q", "description": "Primary-key vertical parity tests pass before the PK cutover slice is enabled", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 8}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_benchmark_rule_parity.py -q", "description": "Benchmark-rule parity tests cover H2O, TPC-DI, and Vector Search onboarding", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 9}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_adapter_rule_fixtures.py -q", "description": "Adapter-rule fixtures cover ClickHouse, DataFusion, and StarRocks migrations", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 10}, {"command": "uv run scripts/compat_lint.py --strict && echo PASS", "description": "Compatibility lint passes in error mode after slice w15", "expected": "PASS", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 11}, {"command": "grep -E 'dialect\\s+in\\s+\\(' benchbox/core/write_primitives/benchmark.py benchbox/core/transaction_primitives/benchmark.py | wc -l | tr -d ' '", "description": "No hardcoded dialect tuples remain in primitives benchmarks after slice w9", "expected": "0", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 12}, {"command": "grep -E '_CLICKHOUSE_Q9|_STARROCKS_Q9' benchbox/core/h2odb/benchmark.py | wc -l | tr -d ' '", "description": "H2O Q9 class constants removed after slice w10", "expected": "0", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 13}, {"command": "test -f scripts/generate_compat_docs.py && grep -l generate_compat_docs Makefile .github/workflows/*.yml 2>/dev/null | head -1 | grep -q . && echo PRESENT", "description": "Doc generator script + CI drift check exist before docs are generated (w17)", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 14}, {"command": "test -f docs/compat/capability-matrix.md && test -f docs/compat/skip-reference.md && echo PRESENT", "description": "Generated capability matrix present after slice w18", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 15}], "work": [{"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": [], "notes": "Blocking step. Deliver docs/development/adr/adr-sql-compat-phase-aware-pipeline.md covering:\n(a) phase list derivation protocol (phases are derived from the inventory\nin w2 then frozen in the ADR - not decreed upfront); (b) the eight-action\nvocabulary (block_benchmark, skip_query, select_variant, rewrite_query,\nrewrite_ddl, set_session_policy, post_translate, native) with payloads;\n(c) CompatibilityContext and CompatibilityDecision dataclass shapes with\nenumerated support_level and failure_mode values; (d) rule_id format\n\"{phase}.{platform}.{scope}.{slug}\"; (e) @compat_local decorator\nsemantics and linter attribution; (f) baseline schema v1 with\nversion-bump and refresh protocol; (g) SQL and DataFrame pipeline head\nshapes and which actions apply to each; (h) explicit non-goal:\nquery-storage format unification stays out of scope; (i) specificity\nprecedence for rule resolution.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write ADR: phase taxonomy, action enum, typed contract, exemption policy, baseline schema v1, dual-run harness spec", "wid": "w1"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w9"], "notes": "Register as select_variant rules behind each benchmark's existing\nquery manager (no storage-format migration):\n(a) H2O Q9 _CLICKHOUSE_Q9 / _STARROCKS_Q9 constants at\ncore/h2odb/benchmark.py:106-141 (delete constants, add rules);\n(b) CoffeeShop platform-specific overrides in core/coffeeshop/.\nDual-run divergence must be 0 for these benchmark-local rule_ids before\nthe slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice C1: Register H2O and CoffeeShop benchmark-local overrides behind existing managers", "wid": "w10"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w10"], "notes": "Register NYC Taxi platform overrides and TPC-DI post-translation\noverrides behind the existing managers. VERIFY actual line before\nstarting - reviewer's cited tpcdi/benchmark.py:374 is a get_queries\naccessor, not a post-translation hook; find the real site first.\nDual-run divergence must be 0 for these benchmark-local rule_ids before\nthe slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice C2: Register NYC Taxi and TPC-DI benchmark-local overrides behind existing managers", "wid": "w11"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w11"], "notes": "Explicitly migrate the non-catalog, non-sqlglot QUERY_VARIANTS path in\ncore/vector_search/queries.py into the registry-backed query_source\npipeline while preserving the existing manager shape. This unit owns the\nversion-gated StarRocks rule called out in the query comments, so the\nregistry proves it can represent platform-version-sensitive variants and\nskips instead of only static dialect decisions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice C3: Onboard Vector Search QUERY_VARIANTS and version-gated compatibility rules", "wid": "w12"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w9"], "notes": "(a) ClickHouse Q23/Q87 alias-injection: attempt AST-first rewrite; if\nsqlglot cannot express it safely, register as set_session_policy rule\n(joined_subquery_requires_alias=0) with issue link documenting the AST\ngap. The current regex in clickhouse/query_transformer.py is disabled\nfor corruption - do not revive it.\n(b) DataFusion query-id-dispatched rewrites in\nplatforms/datafusion_query_transformer.py \u2192 rewrite_query rules keyed\non query_id. Dual-run divergence must be 0 for ClickHouse/DataFusion\nrule_ids before the slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice D1: Adapter/runtime rules for ClickHouse session policy and DataFusion query rewrites", "wid": "w13"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w13"], "notes": "(c) StarRocks _inject_missing_subquery_aliases (workload.py:374,\ntoken-aware) \u2192 rewrite_query rule; _optimize_table_definition\n(workload.py:68+) \u2192 rewrite_ddl rule under the ddl_optimize phase.\n(d) SQL-mode skip path at platforms/base/execution.py:1162 and\nDataFrame-mode skip path at platforms/dataframe/benchmark_mixin.py:828\nboth consult the registry through their respective pipeline heads.\n(e) get_platform_skip_queries / get_df_platform_skip_queries become\nthin facades over the registry (kept for backwards compat until slice\nw16). Dual-run divergence must be 0 for StarRocks/runtime rule_ids before\nthe slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice D2: StarRocks query/DDL rules plus SQL/DataFrame pipeline-head integration", "wid": "w14"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w14"], "notes": "Walk core/tsbs_devops/schema.py and core/nyctaxi/schema.py. Classify\neach dialect branch: compatibility policy (\u2192 rewrite_ddl rule) vs.\ntype mapping / storage layout (\u2192 @compat_local decorator). Use the\ninventory.jsonl classification from w2 as input. Promote the lint from\nwarn to error at the end of this slice.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice E: Schema-generation compatibility - absorb DDL-policy branches, mark type-mapping branches @compat_local", "wid": "w15"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w12", "w14", "w15", "w7"], "notes": "Flip BENCHBOX_COMPAT_REGISTRY default from shadow to on. unsupported_\nbenchmarks field on PlatformCapability (platform_registry.py:77)\nbecomes a view computed from registry benchmark_gate rules;\ncli/commands/run.py:804 preflight reads the view.\nget_platform_skip_queries / get_df_platform_skip_queries deleted.\nDual-run harness code removed. Feature-flag plumbing removed.\nAcceptance: zero unregistered-or-unexempted dialect decisions remain\n(linter error-mode, CI green).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cutover: registry becomes authoritative, dual-run harness deleted, feature flag removed, legacy skip APIs become registry views", "wid": "w16"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w16"], "notes": "Land scripts/generate_compat_docs.py and the CI/build hook that runs it\nand fails when the committed docs drift from regenerated output. This is\nintentionally separate from the generated docs commit so the generator\nitself does not hide inside a doc-refresh work unit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Docs plumbing: generator script plus CI drift check land before generated docs", "wid": "w17"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w17"], "notes": "Run the generator, commit docs/compat/capability-matrix.md and\ndocs/compat/skip-reference.md, and update ADR status from Proposed to\nAccepted with a link to the shipped subsystem.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Docs: capability matrix and skip reference generated from registry; ADR marked accepted", "wid": "w18"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w1"], "notes": "Deliver benchbox/sql_compat/inventory.py that walks benchbox/ and\nenumerates every dialect-branching decision point. Output:\n_project/compat/inventory.jsonl (one row per site: file, line, kind\n{skip | rewrite | ddl | type_mapping | session_setting | benchmark_gate},\nplatforms referenced, suggested phase). Use this to FREEZE the phase\ntaxonomy in the ADR (amend ADR if inventory surfaces phases that were\nmissed). Inventory acceptance is not \"tool runs\"; it must classify at\nleast the current benchmark_gate site in cli/commands/run.py and the\nQUERY_VARIANTS / version-gated site in core/vector_search/queries.py so\nthe registry proves it can see coarse benchmark gating and non-sqlglot\nquery-source variants.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build compatibility inventory tool and freeze the discovered decision-surface taxonomy", "wid": "w2"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w2"], "notes": "Deliver benchbox/sql_compat/baseline_tool.py which, for each\n(platform, benchmark, query_id, phase, mode), emits\n{source_sql_hash, decision, rule_id_or_null, final_sql_hash,\nbenchmark_gate_outcome}. The artifact is checked in at\n_project/compat/baseline.v1.jsonl and is treated as the parity baseline\nfor shadow-mode divergence review. This work unit owns the baseline\nschema, refresh protocol, and schema-version bump rules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record baseline snapshot artifact and schema-version it", "wid": "w3"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w2"], "notes": "Deliver benchbox/sql_compat/local_exemptions.py with the @compat_local\ndecorator (kind: Literal[\"type_mapping\",\"storage_layout\",\"rendering\"],\nplatform_specific: bool, reason: str). Decorate the legitimate\ntype-mapping sites surfaced by w2 (e.g., nyctaxi/schema.py:253,\ntsbs_devops/schema.py type mappings). Deliver\nscripts/compat_lint.py invoked from CI. Lint rule: any dialect-branch\ninside benchbox/core/ or benchbox/benchmarks/ that is not registered in\nthe rule registry AND not inside a @compat_local-decorated callable\nfails the lint. Ship as warn-only (exit 0, log warnings) for this\nslice.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Introduce @compat_local decorator and land the inventory linter in warn-only mode", "wid": "w4"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w1", "w2", "w3"], "notes": "Deliver benchbox/sql_compat/{__init__.py, context.py, decision.py,\nactions.py, registry.py, resolver.py, plan.py}. Registry is import-time\npopulated (rule files under benchbox/sql_compat/rules/{phase}/). Resolver\nproduces a CompilationPlan per (platform, benchmark) at run start,\ncached in BenchmarkResults context so warmup/power/throughput phases do\nnot re-resolve. NATIVE/REPLACED/SKIPPED decisions short-circuit sqlglot.\nNo rules registered yet - just the engine.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Scaffold benchbox/sql_compat/ package: context, decision, action enum, registry, resolver, cache", "wid": "w5"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w5"], "notes": "Env var BENCHBOX_COMPAT_REGISTRY={off|shadow|on}. shadow = both paths\nrun, only legacy decision is acted on, registry decision logged.\non = registry is authoritative, legacy disabled. Divergences write to\n_project/compat/divergence.log with {rule_id, platform, benchmark,\nquery_id, phase, legacy_outcome, registry_outcome}. CI runs in shadow\nmode after w6 lands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build dual-run harness: legacy + registry run in parallel, log divergences, gated by BENCHBOX_COMPAT_REGISTRY flag", "wid": "w6"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w6"], "notes": "Make benchbox/platforms/base/dialect_translation.py:33 delegate to\nbenchbox/utils/dialect_utils.py:108. Explicitly acknowledge this as a\nbehavior change (adapters gain identify-policy, SQLite fixups, GROUP\nBY ALL restoration, pre/post processors). One PR per adapter that\ncurrently uses DialectTranslationMixin. Each PR includes a snapshot\ndiff showing before/after translate() output across the adapter's\ncatalog queries. Merge only after snapshot sign-off.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice A: Unify translation entrypoints with per-adapter diff review PRs", "wid": "w7"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w6"], "notes": "Introduce the structured PK capability model in the registry and use it\nto replace the hardcoded dialect tuple at\ncore/write_primitives/benchmark.py:171 \u2192 capability lookup;\nrewrite body of core/write_primitives/schema.py:352\n_supports_primary_keys() on top of the capability model; and land the\nfirst parity tests for PK semantics under shadow mode. This unit owns\npk.ddl_accepted, pk.uniqueness_enforced, pk.conditions, and\npk.failure_mode as typed data, not booleans.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice B1: Primary-key capability model plus write_primitives lock-path migration", "wid": "w8"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w8"], "notes": "Remove the hardcoded dialect tuple at\ncore/transaction_primitives/benchmark.py:147 \u2192 capability lookup;\nunconditional PK emission at core/transaction_primitives/schema.py:130\nbehind the same capability; then wire CLI preflight at\ncli/commands/run.py:804 to consult benchmark_gate-phase rules for\nPK-requiring benchmarks. Dual-run divergence must be 0 for PK-tagged\nrule_ids across a full shadow-mode pass before this slice can be cut on.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice B2: Transaction-primitives PK/schema migration and CLI benchmark-gate integration", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "derive them from bundle metadata deterministically", "dont": "generate result IDs that depend on file paths or insertion order", "why": "IDs must be stable across rebuilds"}, {"INSTEAD": "serve pre-built read models instead", "dont": "query raw bundles from the browser at runtime", "why": "bundle files are verbose and unindexed"}], "approach": "Pure Python script using only stdlib + duckdb for the snapshot. No web frameworks,\nno databases, no network calls. The script is a build tool, not a service.\nUse `benchbox.core.results.schema` for validation if convenient, but the pipeline\nshould work standalone.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-03T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-results-platform-product-and-launch-strategy", "generate-seed-corpus-for-results-explorer"], "description": "The results explorer consumes derived read models, not raw schema-v2 bundles\ndirectly. This TODO builds the static build pipeline - a Python script that\nreads canonical result bundles from the corpus directory and produces:\n\n1. `manifest.json` - global navigation index (all results with summary metadata)\n2. Per-result detail JSON files - full query timings + metadata for result pages\n3. `results.duckdb` - DuckDB database file for browser-side DuckDB-WASM analytics\n4. Copies raw bundles to `bundles/` for download links\n\nThe pipeline runs in CI as part of the site build and outputs to a directory\nthat the Vite explorer app consumes as static data.\n\nDesign principle: the pipeline is a pure function from bundles \u2192 read models.\nIt has no state, no database, and no network dependencies. It can be re-run\nfrom scratch at any time to rebuild the full read model.", "id": "build-static-pipeline-bundles-to-read-model", "preserves": ["Schema-v2 bundle files are read-only inputs - the pipeline must not modify them", "The derived read model is fully rebuildable from bundles - no state accumulation"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "scripts/build_read_model.py"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/"}], "state": "done", "title": "Build static pipeline that transforms result bundles into explorer read models", "verifications": [{"command": "uv run -- python scripts/build_read_model.py --input results-data/bundles/ --output /tmp/test-read-model/ && ls /tmp/test-read-model/", "description": "Pipeline produces valid read models from seed corpus", "expected": "manifest.json, details/, bundles/, results.duckdb", "item_id": "build-static-pipeline-bundles-to-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/scripts/test_build_read_model.py -q", "description": "Pipeline unit tests pass", "expected": "All tests pass", "item_id": "build-static-pipeline-bundles-to-read-model", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": [], "notes": "Finalize the JSON schemas and DuckDB table schemas. Key decisions:\n\nmanifest.json schema:\n- results[]: id, benchmark, platform, scale_factor, timestamp, total_duration_ms,\n query_count, queries_passed, source (maintainer|community), bundle_path\n- benchmarks[]: unique benchmark names\n- platforms[]: unique platform names\n- generated_at: ISO timestamp\n\nPer-result detail JSON schema (one file per result):\n- id, metadata (benchmark, platform, scale_factor, environment summary)\n- queries[]: id, ms, rows, status (from canonical bundle)\n- summary: total_ms, passed, failed, validation_status\n- bundle_download: relative path to raw bundle\n- companion_files: paths to .plans.json, .tuning.json if present\n\nDuckDB tables:\n- results: one row per result run (id, benchmark, platform, scale_factor,\n timestamp, total_duration_ms, query_count, queries_passed, os, arch,\n cpu_count, memory_gb, source)\n- queries: one row per query execution (result_id, query_id, ms, rows,\n status, run_type)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the derived read model schemas for manifest, detail, and DuckDB tables", "wid": "w1"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w1"], "notes": "Write a Python script (e.g., `scripts/build_read_model.py`) that:\n1. Reads all schema-v2 JSON bundles from the corpus directory\n2. Extracts summary fields for the manifest\n3. Generates a stable, human-readable result ID from metadata\n (e.g., `tpch-duckdb-sf1-20260315`)\n4. Writes `manifest.json` to the output directory\n5. Writes per-result detail JSON files to `details/{result_id}.json`\n6. Copies raw bundles + companions to `bundles/`\n\nThe ID generation must be deterministic and collision-resistant. Use\nbenchmark + platform + scale_factor + date, with a short hash suffix if needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the bundle-to-manifest transformation", "wid": "w2"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w1"], "notes": "Extend the build script to create a `results.duckdb` file containing the\n`results` and `queries` tables defined in w1. Use DuckDB Python API to:\n1. Create tables with appropriate types\n2. Insert flattened data from all bundles\n3. Write the `.duckdb` file to the output directory\n\nThe DuckDB file will be loaded by DuckDB-WASM in the browser for client-side\nSQL queries. Keep the file size reasonable - at 30-100 results this should be\nwell under 1MB.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDB snapshot generation", "wid": "w3"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w2", "w3"], "notes": "Add a CLI entry point: `uv run -- python scripts/build_read_model.py --input results-data/bundles/ --output results-explorer/public/data/`\nThe script should:\n- Accept input corpus directory and output directory as arguments\n- Log progress (N bundles processed, N queries indexed)\n- Exit non-zero on validation errors\n- Be idempotent (clean output dir on each run)\nThis entry point will be called by the GitHub Pages workflow before the\nVite build. The Vite output in `results-explorer/dist/` is then copied to\n`site/results/` during site assembly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add pipeline CLI entry point and CI integration", "wid": "w4"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w2", "w3"], "notes": "Write unit tests that:\n- Feed a small set of known schema-v2 bundles through the pipeline\n- Assert manifest.json has correct structure and counts\n- Assert detail JSONs match expected query data\n- Assert DuckDB snapshot has correct row counts and queryable tables\n- Assert ID generation is deterministic\nUse fixture bundles from the test suite or generate minimal test bundles.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for the static build pipeline", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "hide the canonical raw bundle behind a UI-only representation", "instead": "make the underlying bundle downloadable from result pages", "why": "reproducibility matters"}, {"dont": "use hash-only routing for canonical public result URLs", "instead": "use real result paths or pre-rendered files", "why": "stable shareable pages matter for a benchmark site"}], "approach": "Build the explorer as a standalone subsite under the BenchBox public domain.\nPrefer static or precomputed routes for canonical result pages, then add\nricher browser-side interactivity on top of the derived read model.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-results-platform-product-and-launch-strategy"], "description": "BenchBox needs a public-facing results explorer that feels like a product, not\na directory listing of JSON files. The explorer should live at\n`benchbox.dev/results/`, integrate with the existing public site, and provide\nstable result pages, browse flows, and compare surfaces over derived\npublic read models.\n\n**Phase assignment:** This is the Phase 1 core deliverable. It does NOT depend\non the hosted governance model or full ingest architecture - Phase 1 uses a\nstatic build pipeline (canonical bundles \u2192 JSON manifests + DuckDB snapshot)\ndeployed to GitHub Pages. The governance and ingest TODOs were removed as\nblockers because Phase 1 is entirely maintainer-curated with no hosted services.\n\nThis TODO is a **tracking umbrella** for the Phase 1 implementation. It does\nnot contain implementation work itself - it mirrors the milestone state of the\nresults-explorer cluster. Mark each work unit done when the corresponding\ncluster item(s) are completed. Mark this item Completed when w3 is done.\n\nNote: resolve-results-explorer-brand-ownership must be resolved before launch\nbut does not block implementation. BenchBox placement is the working default.", "id": "build-results-explorer-subsite-on-benchbox-dev", "preserves": ["Explorer result pages link back to the canonical raw bundle and do not invent a second source of truth", "Landing page, docs, and blog remain deployable under the existing GitHub Pages workflow until the new explorer is proven"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Build a results explorer subsite on benchbox.dev", "verifications": [{"command": "test -d results-explorer/dist/ && test -f results-explorer/dist/index.html && echo PRESENT", "description": "Explorer Vite build output exists with an index", "expected": "PRESENT", "item_id": "build-results-explorer-subsite-on-benchbox-dev", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd landing && npm run build 2>&1 | grep -c '[Ee]rror' || echo 0", "description": "Existing docs and landing pages build cleanly after explorer is integrated", "expected": "0", "item_id": "build-results-explorer-subsite-on-benchbox-dev", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "build-results-explorer-subsite-on-benchbox-dev", "needs": [], "notes": "Tracked by: generate-seed-corpus-for-results-explorer,\nbuild-static-pipeline-bundles-to-read-model\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Milestone: seed corpus generated and build pipeline operational", "wid": "w1"}, {"evidence": null, "item_id": "build-results-explorer-subsite-on-benchbox-dev", "needs": ["w1"], "notes": "Tracked by: scaffold-results-explorer-frontend,\nimplement-explorer-home-and-browse-pages,\nimplement-result-detail-page,\nimplement-results-compare-view\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Milestone: frontend scaffold and all explorer pages implemented", "wid": "w2"}, {"evidence": null, "item_id": "build-results-explorer-subsite-on-benchbox-dev", "needs": ["w2"], "notes": "Tracked by: integrate-explorer-into-github-pages-deployment,\nlaunch-results-explorer-acceptance-and-seed\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Milestone: explorer deployed to GitHub Pages and acceptance-tested", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "claim slice w6 is 'zero behavior change'", "instead": "gate with per-adapter snapshot diff sign-off", "why": "dialect_utils.translate_sql_query applies identify-policy, SQLite fixups, and post-processors that the bare sqlglot.transpile in DialectTranslationMixin does not"}, {"dont": "decree the phase list upfront", "instead": "derive the phase list from w2 inventory and freeze it in the ADR", "why": "the reviewer proposed phases (dataframe_filter, session_policy as a phase, post_translate) mix phases with actions"}, {"dont": "leave vector_search/queries.py in files_affected without an owning work unit", "instead": "onboard them explicitly in their own slice", "why": "QUERY_VARIANTS plus platform-version gates are a distinct storage/decision shape"}, {"dont": "migrate without a reversible feature flag", "instead": "ship behind BENCHBOX_COMPAT_REGISTRY with dual-run telemetry", "why": "four slices touching ~15 sites across the codebase have no safe single-revert path"}, {"dont": "model capabilities as bare booleans (e.g., pk.uniqueness_enforced=True)", "instead": "use structured enforcement + conditions + failure_mode", "why": "StarRocks silently ignores PKs that violate the first-N-columns rule, which a boolean cannot express"}, {"dont": "revive the ClickHouse add_subquery_aliases regex rewrite", "instead": "prefer AST; fall back to session policy if sqlglot cannot express the rewrite safely", "why": "it corrupts Q23/Q87 parentheses and is disabled in production for that reason"}, {"dont": "ship 'no inline dialect == remains' as the acceptance criterion", "instead": "gate on 'no unregistered-or-unexempted compatibility decision remains' instead", "why": "legitimate type-mapping branches in nyctaxi/schema.py:253 and tsbs_devops/schema.py:174 are not compatibility policy"}, {"dont": "treat unsupported_benchmarks as dead metadata", "instead": "the benchmark_gate phase must back it via registry view", "why": "it is enforced in CLI preflight at cli/commands/run.py:804"}, {"dont": "trust the reviewer's tpcdi/benchmark.py:374 citation without verifying", "instead": "find the real site in slice w11 before registering the rule", "why": "that line is a get_queries accessor, not a post-translation override"}, {"dont": "unify query-storage formats in this workstream", "instead": "the engine must sit BEHIND the existing managers", "why": "it conflates decision centralization with data-model churn and the dict-based managers have no blocking semantics today"}], "approach": "Phase-first, not query-first. Rule engine plugs in BEHIND existing\nquery-storage managers (catalog, dict-based, QUERY_VARIANTS) - no\nstorage-format migration. Typed CompatibilityContext + CompatibilityDecision\nwith enumerated support_level and failure_mode. Eight-action vocabulary\ncovering AST, regex-with-issue-link, session policy, variant selection,\nDDL rewrite, benchmark gate, skip, and native passthrough. @compat_local\ndecorator + linter provides attribution so legitimate local rendering\n(type mapping, storage layout) is not swept into the refactor. Migration\nvia feature-flagged dual-run harness with divergence telemetry; flip to\nauthoritative only after divergence reaches zero per slice.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-20T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox's SQL compatibility decisions are currently scattered across five\nstructurally different sites: two bare-sqlglot translation cores with divergent\npolicy (benchbox/utils/dialect_utils.py:108 vs\nbenchbox/platforms/base/dialect_translation.py:33), hardcoded dialect tuples\ninside primitives benchmarks (core/write_primitives/benchmark.py:171 and\ncore/transaction_primitives/benchmark.py:147), a _supports_primary_keys()\nhelper whose body contradicts the inline callers\n(core/write_primitives/schema.py:352), unconditional PRIMARY KEY emission in\nschema helpers (core/transaction_primitives/schema.py:130), regex-based\nadapter transformers disabled in production due to SQL corruption\n(platforms/clickhouse/query_transformer.py:47, routed around with a session\nsetting), benchmark get_queries() overrides patching in class-constant SQL\nvariants (core/h2odb/benchmark.py:106-141), plus scattered \"dialect ==\" checks\nin schema generators (core/tsbs_devops/schema.py, core/nyctaxi/schema.py).\n\nResult: the same capability (e.g., \"does this platform enforce PK\nuniqueness?\") has three answers depending on which site asks, and some of\nthose answers are silent-corruption failure modes - StarRocks, for instance,\naccepts PRIMARY KEY DDL but silently ignores PKs whose columns are not the\nfirst N columns of the table.\n\nThis TODO builds a phase-aware compatibility subsystem (benchbox/sql_compat/)\nthat centralizes decision ownership across every compatibility decision point\nsurfaced by the inventory in w2. The final phase taxonomy is NOT frozen here:\nw1 drafts the ADR with a provisional taxonomy, then w2 derives the final\ntaxonomy from the repo and the ADR is updated before engine work proceeds. Likely\ncandidates include benchmark_gate, query_source, query_compile,\nquery_adapter, schema_emit, ddl_optimize, execution_filter, and\ndataframe_filter. Session policy is intentionally treated as an action, not a\nphase, unless inventory proves a separate phase is required. Decisions are\nmade by a typed rule engine returning typed CompatibilityDecision objects\nwith explicit support_level, failure_mode, rule_id, and action payload.\nExisting query-storage shapes (catalog-backed, dict-based managers,\nQUERY_VARIANTS) remain in place - the engine plugs in behind each manager as\na resolver. Storage-format unification is an explicit non-goal of this\nworkstream and is deferred to a future ADR.\n\nThe migration lands in staged, reversible work units behind a\nBENCHBOX_COMPAT_REGISTRY feature flag with a dual-run harness\n(legacy + registry in parallel, divergences logged), cut over only after\ndivergence reaches zero. A\n@compat_local decorator provides an attribution mechanism so a linter can\ndistinguish legitimate platform-specific rendering (type mapping, storage\nlayout) from unregistered compatibility policy. The linter lands warn-only\nin the early inventory slice and is promoted to error during the\nschema-generation cleanup slice before cutover.\n\nSee the ADR deliverable in w1 for the full design: phase taxonomy derivation\nprotocol, action enum, CompatibilityContext / CompatibilityDecision\ndataclasses, rule_id format, baseline schema v1, dual-run harness spec,\nDataFrame pipeline head design, and the storage-unification non-goal.\n\nDDL FOLLOW-ON: Adapter-owned CREATE TABLE rewrites (ClickHouse, Databricks,\nDatabend, Firebolt, Redshift, Azure Synapse, Snowflake, Presto, Trino, Spark,\nLakesail, Velox) are tracked in `centralize-ddl-translation-phase`, which\nruns in parallel with remaining items here. Those shadow harnesses register\nREWRITE_DDL decisions while keeping the legacy _optimize_table_definition()\npath authoritative until this item's w16 cutover. Fabric Warehouse is\nexcluded - its DDL hook is operational (schema-prefix injection via\nself.schema), not a compat rewrite; see ADR section \"DDL Centralization\nFollow-On\" for the full decision record.", "id": "build-sql-compat-phase-aware-pipeline", "preserves": ["CLI preflight in cli/commands/run.py:804 continues to fail-fast on unsupported (platform, benchmark) pairs", "ClickHouse Q23/Q87 continue to run successfully (do not revive the corrupting regex)", "DataFrame-mode skip decisions remain distinct from SQL-mode skip decisions (two pipeline heads share the registry, not the actions)", "H2O Q9 ClickHouse and StarRocks variants continue to produce identical results to the current class-constant SQL", "SQL/DataFrame engines that do not go through sqlglot (Polars, Pandas) retain their existing code paths", "StarRocks first-N-columns PK rule is expressed structurally and prevents silent data divergence", "Vector Search QUERY_VARIANTS and version-gated behavior continue to work without forcing sqlglot into the path"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/compat/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/coffeeshop/"}, {"kind": "only_modify", "path_glob": "benchbox/core/h2odb/"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_catalog_base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tsbs_devops/"}, {"kind": "only_modify", "path_glob": "benchbox/core/vector_search/"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/dialect_translation.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion_query_transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/starrocks/"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "benchbox/utils/dialect_utils.py"}, {"kind": "only_modify", "path_glob": "docs/compat/"}, {"kind": "only_modify", "path_glob": "docs/development/adr/adr-sql-compat-phase-aware-pipeline.md"}, {"kind": "only_modify", "path_glob": "scripts/compat_lint.py"}, {"kind": "only_modify", "path_glob": "scripts/generate_compat_docs.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}, {"kind": "only_modify", "path_glob": "tests/unit/sql_compat/"}], "state": "done", "title": "Build sql_compat phase-aware compatibility pipeline (centralized decision ownership)", "verifications": [{"command": "test -f docs/development/adr/adr-sql-compat-phase-aware-pipeline.md && echo PRESENT", "description": "ADR committed before implementation slices begin", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 1}, {"command": "test -s _project/compat/inventory.jsonl && echo PRESENT", "description": "Compatibility inventory artifact present", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -E 'cli/commands/run.py|core/vector_search/queries.py' _project/compat/inventory.jsonl | wc -l | tr -d ' '", "description": "Inventory captures both benchmark_gate and QUERY_VARIANTS decision sites before the ADR is frozen", "expected": "2", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 3}, {"command": "test -s _project/compat/baseline.v1.jsonl && echo PRESENT", "description": "Baseline snapshot present and schema-versioned", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 4}, {"command": "for m in context.py decision.py actions.py registry.py resolver.py plan.py; do test -f benchbox/sql_compat/$m || exit 1; done && echo PRESENT", "description": "sql_compat package exists with required modules", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 5}, {"command": "BENCHBOX_COMPAT_REGISTRY=on uv run -- python -m pytest -m fast -q", "description": "Full fast test suite passes under registry-authoritative mode", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 6}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_shadow_parity.py -q", "description": "Shadow-mode parity smoke passes once the dual-run harness lands", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 7}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_primary_key_capabilities.py -q", "description": "Primary-key vertical parity tests pass before the PK cutover slice is enabled", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 8}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_benchmark_rule_parity.py -q", "description": "Benchmark-rule parity tests cover H2O, TPC-DI, and Vector Search onboarding", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 9}, {"command": "BENCHBOX_COMPAT_REGISTRY=shadow uv run -- python -m pytest tests/unit/sql_compat/test_adapter_rule_fixtures.py -q", "description": "Adapter-rule fixtures cover ClickHouse, DataFusion, and StarRocks migrations", "expected": "passed", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 10}, {"command": "uv run scripts/compat_lint.py --strict && echo PASS", "description": "Compatibility lint passes in error mode after slice w15", "expected": "PASS", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 11}, {"command": "grep -E 'dialect\\s+in\\s+\\(' benchbox/core/write_primitives/benchmark.py benchbox/core/transaction_primitives/benchmark.py | wc -l | tr -d ' '", "description": "No hardcoded dialect tuples remain in primitives benchmarks after slice w9", "expected": "0", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 12}, {"command": "grep -E '_CLICKHOUSE_Q9|_STARROCKS_Q9' benchbox/core/h2odb/benchmark.py | wc -l | tr -d ' '", "description": "H2O Q9 class constants removed after slice w10", "expected": "0", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 13}, {"command": "test -f scripts/generate_compat_docs.py && grep -l generate_compat_docs Makefile .github/workflows/*.yml 2>/dev/null | head -1 | grep -q . && echo PRESENT", "description": "Doc generator script + CI drift check exist before docs are generated (w17)", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 14}, {"command": "test -f docs/compat/capability-matrix.md && test -f docs/compat/skip-reference.md && echo PRESENT", "description": "Generated capability matrix present after slice w18", "expected": "PRESENT", "item_id": "build-sql-compat-phase-aware-pipeline", "last_result": null, "last_run": null, "seq": 15}], "work": [{"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": [], "notes": "Blocking step. Deliver docs/development/adr/adr-sql-compat-phase-aware-pipeline.md covering:\n(a) phase list derivation protocol (phases are derived from the inventory\nin w2 then frozen in the ADR - not decreed upfront); (b) the eight-action\nvocabulary (block_benchmark, skip_query, select_variant, rewrite_query,\nrewrite_ddl, set_session_policy, post_translate, native) with payloads;\n(c) CompatibilityContext and CompatibilityDecision dataclass shapes with\nenumerated support_level and failure_mode values; (d) rule_id format\n\"{phase}.{platform}.{scope}.{slug}\"; (e) @compat_local decorator\nsemantics and linter attribution; (f) baseline schema v1 with\nversion-bump and refresh protocol; (g) SQL and DataFrame pipeline head\nshapes and which actions apply to each; (h) explicit non-goal:\nquery-storage format unification stays out of scope; (i) specificity\nprecedence for rule resolution.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write ADR: phase taxonomy, action enum, typed contract, exemption policy, baseline schema v1, dual-run harness spec", "wid": "w1"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w9"], "notes": "Register as select_variant rules behind each benchmark's existing\nquery manager (no storage-format migration):\n(a) H2O Q9 _CLICKHOUSE_Q9 / _STARROCKS_Q9 constants at\ncore/h2odb/benchmark.py:106-141 (delete constants, add rules);\n(b) CoffeeShop platform-specific overrides in core/coffeeshop/.\nDual-run divergence must be 0 for these benchmark-local rule_ids before\nthe slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice C1: Register H2O and CoffeeShop benchmark-local overrides behind existing managers", "wid": "w10"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w10"], "notes": "Register NYC Taxi platform overrides and TPC-DI post-translation\noverrides behind the existing managers. VERIFY actual line before\nstarting - reviewer's cited tpcdi/benchmark.py:374 is a get_queries\naccessor, not a post-translation hook; find the real site first.\nDual-run divergence must be 0 for these benchmark-local rule_ids before\nthe slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice C2: Register NYC Taxi and TPC-DI benchmark-local overrides behind existing managers", "wid": "w11"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w11"], "notes": "Explicitly migrate the non-catalog, non-sqlglot QUERY_VARIANTS path in\ncore/vector_search/queries.py into the registry-backed query_source\npipeline while preserving the existing manager shape. This unit owns the\nversion-gated StarRocks rule called out in the query comments, so the\nregistry proves it can represent platform-version-sensitive variants and\nskips instead of only static dialect decisions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice C3: Onboard Vector Search QUERY_VARIANTS and version-gated compatibility rules", "wid": "w12"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w9"], "notes": "(a) ClickHouse Q23/Q87 alias-injection: attempt AST-first rewrite; if\nsqlglot cannot express it safely, register as set_session_policy rule\n(joined_subquery_requires_alias=0) with issue link documenting the AST\ngap. The current regex in clickhouse/query_transformer.py is disabled\nfor corruption - do not revive it.\n(b) DataFusion query-id-dispatched rewrites in\nplatforms/datafusion_query_transformer.py \u2192 rewrite_query rules keyed\non query_id. Dual-run divergence must be 0 for ClickHouse/DataFusion\nrule_ids before the slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice D1: Adapter/runtime rules for ClickHouse session policy and DataFusion query rewrites", "wid": "w13"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w13"], "notes": "(c) StarRocks _inject_missing_subquery_aliases (workload.py:374,\ntoken-aware) \u2192 rewrite_query rule; _optimize_table_definition\n(workload.py:68+) \u2192 rewrite_ddl rule under the ddl_optimize phase.\n(d) SQL-mode skip path at platforms/base/execution.py:1162 and\nDataFrame-mode skip path at platforms/dataframe/benchmark_mixin.py:828\nboth consult the registry through their respective pipeline heads.\n(e) get_platform_skip_queries / get_df_platform_skip_queries become\nthin facades over the registry (kept for backwards compat until slice\nw16). Dual-run divergence must be 0 for StarRocks/runtime rule_ids before\nthe slice is considered done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice D2: StarRocks query/DDL rules plus SQL/DataFrame pipeline-head integration", "wid": "w14"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w14"], "notes": "Walk core/tsbs_devops/schema.py and core/nyctaxi/schema.py. Classify\neach dialect branch: compatibility policy (\u2192 rewrite_ddl rule) vs.\ntype mapping / storage layout (\u2192 @compat_local decorator). Use the\ninventory.jsonl classification from w2 as input. Promote the lint from\nwarn to error at the end of this slice.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice E: Schema-generation compatibility - absorb DDL-policy branches, mark type-mapping branches @compat_local", "wid": "w15"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w12", "w14", "w15", "w7"], "notes": "Flip BENCHBOX_COMPAT_REGISTRY default from shadow to on. unsupported_\nbenchmarks field on PlatformCapability (platform_registry.py:77)\nbecomes a view computed from registry benchmark_gate rules;\ncli/commands/run.py:804 preflight reads the view.\nget_platform_skip_queries / get_df_platform_skip_queries deleted.\nDual-run harness code removed. Feature-flag plumbing removed.\nAcceptance: zero unregistered-or-unexempted dialect decisions remain\n(linter error-mode, CI green).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cutover: registry becomes authoritative, dual-run harness deleted, feature flag removed, legacy skip APIs become registry views", "wid": "w16"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w16"], "notes": "Land scripts/generate_compat_docs.py and the CI/build hook that runs it\nand fails when the committed docs drift from regenerated output. This is\nintentionally separate from the generated docs commit so the generator\nitself does not hide inside a doc-refresh work unit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Docs plumbing: generator script plus CI drift check land before generated docs", "wid": "w17"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w17"], "notes": "Run the generator, commit docs/compat/capability-matrix.md and\ndocs/compat/skip-reference.md, and update ADR status from Proposed to\nAccepted with a link to the shipped subsystem.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Docs: capability matrix and skip reference generated from registry; ADR marked accepted", "wid": "w18"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w1"], "notes": "Deliver benchbox/sql_compat/inventory.py that walks benchbox/ and\nenumerates every dialect-branching decision point. Output:\n_project/compat/inventory.jsonl (one row per site: file, line, kind\n{skip | rewrite | ddl | type_mapping | session_setting | benchmark_gate},\nplatforms referenced, suggested phase). Use this to FREEZE the phase\ntaxonomy in the ADR (amend ADR if inventory surfaces phases that were\nmissed). Inventory acceptance is not \"tool runs\"; it must classify at\nleast the current benchmark_gate site in cli/commands/run.py and the\nQUERY_VARIANTS / version-gated site in core/vector_search/queries.py so\nthe registry proves it can see coarse benchmark gating and non-sqlglot\nquery-source variants.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build compatibility inventory tool and freeze the discovered decision-surface taxonomy", "wid": "w2"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w2"], "notes": "Deliver benchbox/sql_compat/baseline_tool.py which, for each\n(platform, benchmark, query_id, phase, mode), emits\n{source_sql_hash, decision, rule_id_or_null, final_sql_hash,\nbenchmark_gate_outcome}. The artifact is checked in at\n_project/compat/baseline.v1.jsonl and is treated as the parity baseline\nfor shadow-mode divergence review. This work unit owns the baseline\nschema, refresh protocol, and schema-version bump rules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record baseline snapshot artifact and schema-version it", "wid": "w3"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w2"], "notes": "Deliver benchbox/sql_compat/local_exemptions.py with the @compat_local\ndecorator (kind: Literal[\"type_mapping\",\"storage_layout\",\"rendering\"],\nplatform_specific: bool, reason: str). Decorate the legitimate\ntype-mapping sites surfaced by w2 (e.g., nyctaxi/schema.py:253,\ntsbs_devops/schema.py type mappings). Deliver\nscripts/compat_lint.py invoked from CI. Lint rule: any dialect-branch\ninside benchbox/core/ or benchbox/benchmarks/ that is not registered in\nthe rule registry AND not inside a @compat_local-decorated callable\nfails the lint. Ship as warn-only (exit 0, log warnings) for this\nslice.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Introduce @compat_local decorator and land the inventory linter in warn-only mode", "wid": "w4"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w1", "w2", "w3"], "notes": "Deliver benchbox/sql_compat/{__init__.py, context.py, decision.py,\nactions.py, registry.py, resolver.py, plan.py}. Registry is import-time\npopulated (rule files under benchbox/sql_compat/rules/{phase}/). Resolver\nproduces a CompilationPlan per (platform, benchmark) at run start,\ncached in BenchmarkResults context so warmup/power/throughput phases do\nnot re-resolve. NATIVE/REPLACED/SKIPPED decisions short-circuit sqlglot.\nNo rules registered yet - just the engine.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Scaffold benchbox/sql_compat/ package: context, decision, action enum, registry, resolver, cache", "wid": "w5"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w5"], "notes": "Env var BENCHBOX_COMPAT_REGISTRY={off|shadow|on}. shadow = both paths\nrun, only legacy decision is acted on, registry decision logged.\non = registry is authoritative, legacy disabled. Divergences write to\n_project/compat/divergence.log with {rule_id, platform, benchmark,\nquery_id, phase, legacy_outcome, registry_outcome}. CI runs in shadow\nmode after w6 lands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build dual-run harness: legacy + registry run in parallel, log divergences, gated by BENCHBOX_COMPAT_REGISTRY flag", "wid": "w6"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w6"], "notes": "Make benchbox/platforms/base/dialect_translation.py:33 delegate to\nbenchbox/utils/dialect_utils.py:108. Explicitly acknowledge this as a\nbehavior change (adapters gain identify-policy, SQLite fixups, GROUP\nBY ALL restoration, pre/post processors). One PR per adapter that\ncurrently uses DialectTranslationMixin. Each PR includes a snapshot\ndiff showing before/after translate() output across the adapter's\ncatalog queries. Merge only after snapshot sign-off.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice A: Unify translation entrypoints with per-adapter diff review PRs", "wid": "w7"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w6"], "notes": "Introduce the structured PK capability model in the registry and use it\nto replace the hardcoded dialect tuple at\ncore/write_primitives/benchmark.py:171 \u2192 capability lookup;\nrewrite body of core/write_primitives/schema.py:352\n_supports_primary_keys() on top of the capability model; and land the\nfirst parity tests for PK semantics under shadow mode. This unit owns\npk.ddl_accepted, pk.uniqueness_enforced, pk.conditions, and\npk.failure_mode as typed data, not booleans.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice B1: Primary-key capability model plus write_primitives lock-path migration", "wid": "w8"}, {"evidence": null, "item_id": "build-sql-compat-phase-aware-pipeline", "needs": ["w8"], "notes": "Remove the hardcoded dialect tuple at\ncore/transaction_primitives/benchmark.py:147 \u2192 capability lookup;\nunconditional PK emission at core/transaction_primitives/schema.py:130\nbehind the same capability; then wire CLI preflight at\ncli/commands/run.py:804 to consult benchmark_gate-phase rules for\nPK-requiring benchmarks. Dual-run divergence must be 0 for PK-tagged\nrule_ids across a full shadow-mode pass before this slice can be cut on.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Slice B2: Transaction-primitives PK/schema migration and CLI benchmark-gate integration", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "generate result IDs that depend on file paths or insertion order", "instead": "derive them from bundle metadata deterministically", "why": "IDs must be stable across rebuilds"}, {"dont": "query raw bundles from the browser at runtime", "instead": "serve pre-built read models instead", "why": "bundle files are verbose and unindexed"}], "approach": "Pure Python script using only stdlib + duckdb for the snapshot. No web frameworks,\nno databases, no network calls. The script is a build tool, not a service.\nUse `benchbox.core.results.schema` for validation if convenient, but the pipeline\nshould work standalone.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-03T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-results-platform-product-and-launch-strategy", "generate-seed-corpus-for-results-explorer"], "description": "The results explorer consumes derived read models, not raw schema-v2 bundles\ndirectly. This TODO builds the static build pipeline - a Python script that\nreads canonical result bundles from the corpus directory and produces:\n\n1. `manifest.json` - global navigation index (all results with summary metadata)\n2. Per-result detail JSON files - full query timings + metadata for result pages\n3. `results.duckdb` - DuckDB database file for browser-side DuckDB-WASM analytics\n4. Copies raw bundles to `bundles/` for download links\n\nThe pipeline runs in CI as part of the site build and outputs to a directory\nthat the Vite explorer app consumes as static data.\n\nDesign principle: the pipeline is a pure function from bundles \u2192 read models.\nIt has no state, no database, and no network dependencies. It can be re-run\nfrom scratch at any time to rebuild the full read model.", "id": "build-static-pipeline-bundles-to-read-model", "preserves": ["Schema-v2 bundle files are read-only inputs - the pipeline must not modify them", "The derived read model is fully rebuildable from bundles - no state accumulation"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "scripts/build_read_model.py"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/"}], "state": "done", "title": "Build static pipeline that transforms result bundles into explorer read models", "verifications": [{"command": "uv run -- python scripts/build_read_model.py --input results-data/bundles/ --output /tmp/test-read-model/ && ls /tmp/test-read-model/", "description": "Pipeline produces valid read models from seed corpus", "expected": "manifest.json, details/, bundles/, results.duckdb", "item_id": "build-static-pipeline-bundles-to-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/scripts/test_build_read_model.py -q", "description": "Pipeline unit tests pass", "expected": "All tests pass", "item_id": "build-static-pipeline-bundles-to-read-model", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": [], "notes": "Finalize the JSON schemas and DuckDB table schemas. Key decisions:\n\nmanifest.json schema:\n- results[]: id, benchmark, platform, scale_factor, timestamp, total_duration_ms,\n query_count, queries_passed, source (maintainer|community), bundle_path\n- benchmarks[]: unique benchmark names\n- platforms[]: unique platform names\n- generated_at: ISO timestamp\n\nPer-result detail JSON schema (one file per result):\n- id, metadata (benchmark, platform, scale_factor, environment summary)\n- queries[]: id, ms, rows, status (from canonical bundle)\n- summary: total_ms, passed, failed, validation_status\n- bundle_download: relative path to raw bundle\n- companion_files: paths to .plans.json, .tuning.json if present\n\nDuckDB tables:\n- results: one row per result run (id, benchmark, platform, scale_factor,\n timestamp, total_duration_ms, query_count, queries_passed, os, arch,\n cpu_count, memory_gb, source)\n- queries: one row per query execution (result_id, query_id, ms, rows,\n status, run_type)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the derived read model schemas for manifest, detail, and DuckDB tables", "wid": "w1"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w1"], "notes": "Write a Python script (e.g., `scripts/build_read_model.py`) that:\n1. Reads all schema-v2 JSON bundles from the corpus directory\n2. Extracts summary fields for the manifest\n3. Generates a stable, human-readable result ID from metadata\n (e.g., `tpch-duckdb-sf1-20260315`)\n4. Writes `manifest.json` to the output directory\n5. Writes per-result detail JSON files to `details/{result_id}.json`\n6. Copies raw bundles + companions to `bundles/`\n\nThe ID generation must be deterministic and collision-resistant. Use\nbenchmark + platform + scale_factor + date, with a short hash suffix if needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the bundle-to-manifest transformation", "wid": "w2"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w1"], "notes": "Extend the build script to create a `results.duckdb` file containing the\n`results` and `queries` tables defined in w1. Use DuckDB Python API to:\n1. Create tables with appropriate types\n2. Insert flattened data from all bundles\n3. Write the `.duckdb` file to the output directory\n\nThe DuckDB file will be loaded by DuckDB-WASM in the browser for client-side\nSQL queries. Keep the file size reasonable - at 30-100 results this should be\nwell under 1MB.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDB snapshot generation", "wid": "w3"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w2", "w3"], "notes": "Add a CLI entry point: `uv run -- python scripts/build_read_model.py --input results-data/bundles/ --output results-explorer/public/data/`\nThe script should:\n- Accept input corpus directory and output directory as arguments\n- Log progress (N bundles processed, N queries indexed)\n- Exit non-zero on validation errors\n- Be idempotent (clean output dir on each run)\nThis entry point will be called by the GitHub Pages workflow before the\nVite build. The Vite output in `results-explorer/dist/` is then copied to\n`site/results/` during site assembly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add pipeline CLI entry point and CI integration", "wid": "w4"}, {"evidence": null, "item_id": "build-static-pipeline-bundles-to-read-model", "needs": ["w2", "w3"], "notes": "Write unit tests that:\n- Feed a small set of known schema-v2 bundles through the pipeline\n- Assert manifest.json has correct structure and counts\n- Assert detail JSONs match expected query data\n- Assert DuckDB snapshot has correct row counts and queryable tables\n- Assert ID generation is deterministic\nUse fixture bundles from the test suite or generate minimal test bundles.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for the static build pipeline", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:33:46Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #109 of bulk-shard-loading-clickhouse.\nDeferred: Bulk loading for PostgreSQL COPY FROM PROGRAM or COPY FROM STDIN multi-file\nReason deferred: PostgreSQL COPY is fundamentally single-file per statement; would require UNION ALL construction or a PL/pgSQL wrapper - different scope", "id": "bulk-loading-for-postgresql-copy-from-program-or-copy-from-s", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Bulk loading for PostgreSQL COPY FROM PROGRAM or COPY FROM STDIN multi-file", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:33:48Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #110 of bulk-shard-loading-clickhouse.\nDeferred: Bulk S3/GCS loading for ClickHouse Cloud adapter using s3() glob patterns\nReason deferred: Cloud adapter (_load_data_via_s3, _load_data_via_gcs) has its own per-file upload+insert loop that bypasses DataLoader and ClickHouseNativeHandler entirely. Bulk loading requires restructuring the upload pipeline (upload all first, then single INSERT with s3() glob) - a larger refactor tracked separately.", "id": "bulk-s3-gcs-loading-for-clickhouse-cloud-adapter-using-s3-gl", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Bulk S3/GCS loading for ClickHouse Cloud adapter using s3() glob patterns", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use the glob pattern in SQL instead", "dont": "concatenate shard CSV files on disk", "why": "that requires temp storage and file I/O"}, {"INSTEAD": "derive the common prefix programmatically from the actual file names", "dont": "hardcode the glob separator as '.'", "why": "shard naming conventions may differ (e.g., _N vs .N)"}, {"INSTEAD": "the fallback ensures correctness in all cases", "dont": "skip the fallback to per-file loop", "why": "shards may legitimately be in different dirs for re-loaded manifests"}], "approach": "1. Read ClickHouseNativeHandler.load_table() to understand the current SQL\n pattern, column schema retrieval, and dry-run handling.\n2. Implement load_table_bulk(): derive glob from the shard list, build a\n single INSERT ... SELECT * FROM file(glob) statement, execute with\n before/after COUNT(*) for the row count delta.\n3. Add guard: if shard paths span multiple directories, fall back to\n super().load_table_bulk() (the default loop).\n4. For ClickHouse Cloud: audit whether the cloud adapter already handles\n shards in bulk; if so, just add tests.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-22T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-clickhouse", "id": 108, "reason": "CSV via file() is sufficient for current benchmarks; Native format has different escaping requirements", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Speculative optimization; CSV loading deemed sufficient with no concrete trigger identified to revisit Native format.", "summary": "Investigate columnar-native bulk loading (ClickHouse Native format instead of CSV)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-clickhouse", "id": 109, "reason": "PostgreSQL COPY is fundamentally single-file per statement; would require UNION ALL construction or a PL/pgSQL wrapper - different scope", "resolution": "promoted", "resolved_item": "bulk-loading-for-postgresql-copy-from-program-or-copy-from-s", "resolved_reason": null, "summary": "Bulk loading for PostgreSQL COPY FROM PROGRAM or COPY FROM STDIN multi-file"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-clickhouse", "id": 110, "reason": "Cloud adapter (_load_data_via_s3, _load_data_via_gcs) has its own per-file upload+insert loop that bypasses DataLoader and ClickHouseNativeHandler entirely. Bulk loading requires restructuring the upload pipeline (upload all first, then single INSERT with s3() glob) - a larger refactor tracked separately.", "resolution": "promoted", "resolved_item": "bulk-s3-gcs-loading-for-clickhouse-cloud-adapter-using-s3-gl", "resolved_reason": null, "summary": "Bulk S3/GCS loading for ClickHouse Cloud adapter using s3() glob patterns"}], "deps": ["bulk-shard-loading-duckdb"], "description": "ClickHouseNativeHandler.load_table() currently processes one shard file per\ncall. For a sharded benchmark table (e.g., lineitem.tbl.1 \u2026 lineitem.tbl.10)\nthe DataLoader loop issues 10 separate INSERT INTO ... SELECT * FROM file(path)\nstatements.\n\nClickHouse's table function file() (and s3() for cloud) support glob patterns:\n file('/data/lineitem*.tbl', 'CSV', 'col1 UInt64, ...')\nA single INSERT using a glob covers all shards in one query-plan pass.\n\nThis TODO implements load_table_bulk() in ClickHouseNativeHandler, leveraging\nthe load_table_bulk() interface introduced in bulk-shard-loading-duckdb.\n\nScope is ClickHouse (self-hosted) and ClickHouse Cloud adapters only.\nIt explicitly excludes platforms backed by DelimitedFileHandler (SQLite,\nPostgreSQL, DataFusion, etc.) which have no SQL multi-file primitive.", "id": "bulk-shard-loading-clickhouse", "preserves": ["All existing tests/unit/platforms/test_clickhouse_adapter.py tests pass without modification", "All fast-suite tests continue to pass: uv run -- python -m pytest -m fast -q must report 0 failures", "ClickHouseNativeHandler.load_table() (single-file path) continues to work unchanged", "Dry-run mode still captures SQL without executing it", "load_table_bulk() falls back to the default per-file loop when shard paths are in different directories"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/clickhouse/workload.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse_cloud.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_clickhouse_adapter.py"}], "state": "done", "title": "Replace per-shard INSERT loops with glob-pattern bulk INSERT for ClickHouseNativeHandler", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py -v", "description": "ClickHouse handler unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "bulk-shard-loading-clickhouse", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py -v -k bulk", "description": "New bulk loading tests pass", "expected": "all bulk tests pass, 0 failures", "item_id": "bulk-shard-loading-clickhouse", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Full fast suite passes (no regression)", "expected": "all tests pass, 0 failures", "item_id": "bulk-shard-loading-clickhouse", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": [], "notes": "File: benchbox/platforms/clickhouse/workload.py (and any subclass in clickhouse_cloud.py).\nConfirm the exact SQL pattern used (file(), s3(), or COPY), how file paths are\nescaped, and whether there is a dry-run mode to preserve.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit ClickHouseNativeHandler: locate load_table() and identify all execute call sites", "wid": "w1"}, {"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": ["w1"], "notes": "Algorithm:\n1. Derive a glob from the shard list. All shards for a table share a common\n stem: if paths are ['lineitem.tbl.1', 'lineitem.tbl.2', ...] in the same\n directory, the glob is 'lineitem.tbl.*'.\n2. Validate: all paths share the same parent dir and the same stem prefix.\n If not (edge case), fall back to the default loop via super().load_table_bulk().\n3. Build SQL:\n INSERT INTO {table}\n SELECT * FROM file('{dir}/{stem}.*', 'CSV', '{col_schema}')\n4. Execute and return COUNT delta (before/after) as for DuckDB.\n\nThe ClickHouse file() function requires an explicit column schema string for\nCSV - derive it from DESCRIBE TABLE {table} (or equivalent).\nDry-run: capture SQL then return len(file_paths)*1000.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ClickHouseNativeHandler.load_table_bulk() using file() glob pattern for self-hosted ClickHouse", "wid": "w2"}, {"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": ["w2"], "notes": "ClickHouse Cloud benchmarks upload files to S3 before loading.\nTIME CAP: max 2 hrs. If the audit shows the cloud path already issues a\nsingle COPY / INSERT for all shards, add a confirming test and close this unit.\nIf the cloud path still issues one INSERT per shard (using s3() or url()),\nimplement load_table_bulk() analogously to w2 using s3() glob patterns.\nDO NOT scope-creep into refactoring the full cloud upload pipeline -\nonly add/override load_table_bulk() in the cloud adapter class.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle ClickHouse Cloud: verify whether cloud path already bulk-loads; add tests or implement if not", "wid": "w3"}, {"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": ["w2"], "notes": "Test cases:\n- 4 shards in same dir: assert glob pattern in SQL, single execute call, correct row count\n- Shards in different directories: assert fallback to per-shard loop (no glob)\n- Single-shard list: assert result equals load_table() result\n- Dry-run: assert no execute on real connection, return == 4000\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for ClickHouseNativeHandler.load_table_bulk() - glob SQL, row count, fallback, dry-run", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "sample only the first path", "dont": "call has_trailing_delimiter on every shard in load_table_bulk()", "why": "all shards of the same TBL table have the same delimiter pattern"}, {"INSTEAD": "call it once for the first path", "dont": "call pragma_table_info once per shard in load_table_bulk()", "why": "the schema is the same for all shards of a table"}, {"INSTEAD": "leave it untouched", "dont": "change _load_single_file()", "why": "it is still used for the single-file (non-list) branch"}, {"INSTEAD": "only add load_table_bulk() as an override", "dont": "change the load_table() single-file method signatures", "why": "existing tests and non-DuckDB platforms depend on them"}, {"INSTEAD": "use the explicit list passed in", "dont": "use a glob string instead of an explicit path array in load_table_bulk()", "why": "shard files may not be in the same directory or may have non-glob-friendly names"}], "approach": "1. In FileFormatHandler (data_loading.py), add:\n def load_table_bulk(self, table_name, file_paths, connection, benchmark, logger) -> int:\n total = 0\n for path in file_paths:\n total += self.load_table(table_name, path, connection, benchmark, logger)\n return total\n\n2. In DataLoader._load_file_based_data() (lines 1773-1782), replace the\n for-loop + _load_single_file() pattern with:\n handler = self.handler_factory(file_path_or_paths[0], self.adapter, self.benchmark)\n total_rows = handler.load_table_bulk(table_name, [Path(p) for p in file_path_or_paths],\n self.connection, self.benchmark, self.adapter.logger)\n This calls the factory once to pick the correct handler, then delegates\n the entire shard list to load_table_bulk().\n\n3. In DuckDBNativeHandler, override load_table_bulk():\n a. Call pragma_table_info once to get col_names.\n b. Detect trailing delimiter on file_paths[0].\n c. Escape each path: [escape_sql_string_literal(str(p)) for p in file_paths].\n d. Build array literal: \"['\" + \"', '\".join(escaped) + \"']\".\n e. Build INSERT SQL using the array literal inside read_csv().\n f. COUNT before, execute INSERT, COUNT after, return delta.\n\n4. In DuckDBParquetHandler, override load_table_bulk() similarly:\n Build INSERT ... SELECT * FROM read_parquet(['p1', 'p2', ...]) and use\n before/after COUNT(*).\n\n5. Add tests before touching production code (TDD: write failing tests for\n load_table_bulk, then implement to make them pass).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-22T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-duckdb", "id": 111, "reason": "Lower volume - Delta tables are typically single files; validate demand before adding complexity", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly gated on validating demand first; no evidence demand for Delta/Vortex/DuckLake bulk loading was assessed.", "summary": "Bulk loading for DuckDB Delta / Vortex / DuckLake handlers"}], "deps": [], "description": "When a benchmark table is generated with parallel>1, its data is split across\nN shard files (e.g., lineitem.tbl.1, lineitem.tbl.2, ..., lineitem.tbl.N).\nThe base DataLoader._load_file_based_data() currently calls handler.load_table()\nonce per shard in a sequential loop, producing N separate INSERT statements.\n\nFor DuckDB this is inefficient: DuckDB's read_csv() and read_parquet() both\naccept an array of file paths or a glob pattern and process all files in a\nsingle optimised scan, avoiding N round-trips and N separate query plans.\n\nThis TODO adds a load_table_bulk() interface to FileFormatHandler, changes the\nDataLoader loop to use it, and implements the DuckDB-native multi-file variant\nfor DuckDBNativeHandler (CSV/TBL) and DuckDBParquetHandler.\n\nExpected improvement (TPC-H SF1, 4-shard lineitem):\n- Before: 4 \u00d7 (pragma + COUNT_before + INSERT + COUNT_after) = 16 execute calls\n- After: 1 \u00d7 (pragma + COUNT_before + bulk INSERT + COUNT_after) = 4 execute calls\n- Data transfer and query planning overhead reduced ~75% for sharded tables\n\nPostgreSQL, SQLite, DataFusion, and other platforms that use DelimitedFileHandler\n/ executemany() are NOT in scope - they have no SQL multi-file primitive.", "id": "bulk-shard-loading-duckdb", "preserves": ["All 12,970 existing fast-suite tests continue to pass", "DataLoader default behaviour (non-DuckDB platforms, single-file tables) is unaffected", "Dry-run mode still captures SQL without executing it", "DuckDBNativeHandler.load_table() (single-file path) continues to work unchanged for non-sharded tables", "DuckDBParquetHandler.load_table() continues to work unchanged", "benchbox run --platform duckdb --benchmark tpch --scale 1 --phases load produces total rows_loaded == 8,661,245 (lineitem 6,001,215 + orders 1,500,000 + ...)"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "tests/unit/platforms/test_duckdb_adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/test_parquet_handlers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/test_trailing_delimiter_loading.py"}], "state": "done", "title": "Replace per-shard INSERT loops with single bulk INSERT for DuckDB handlers", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "All existing fast-suite tests pass after the framework change", "expected": "12970+ passed, 0 failed", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_trailing_delimiter_loading.py -v -k bulk", "description": "New load_table_bulk unit tests pass for DuckDBNativeHandler", "expected": "all bulk tests pass, 0 failures", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_parquet_handlers.py -v -k bulk", "description": "New load_table_bulk unit tests pass for DuckDBParquetHandler", "expected": "all bulk tests pass, 0 failures", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -k 'duckdb and load' -v", "description": "DuckDB TPC-H SF0.01 smoke test completes with correct row counts", "expected": "all tests pass, 0 failures", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": [], "notes": "Signature: load_table_bulk(table_name, file_paths: list[Path], connection, benchmark, logger) -> int\nDefault: iterate file_paths, call self.load_table() for each, sum results.\nThis makes the interface backward-compatible - all existing handlers auto-inherit\ncorrect (if non-bulk) behaviour.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add load_table_bulk() to FileFormatHandler base class with default loop implementation", "wid": "w1"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w1"], "notes": "In the isinstance(file_path_or_paths, list) branch, instead of:\n for shard_path in file_path_or_paths:\n rows += self._load_single_file(table_name, shard_path)\nDo:\n handler = self.handler_factory(file_path_or_paths[0], ...)\n total_rows = handler.load_table_bulk(table_name, [Path(p) for p in file_path_or_paths],\n self.connection, self.benchmark, self.adapter.logger)\nThe handler_factory is called once (on the first shard) to select the right handler type.\nThe shard_count display in the log line must still be len(file_path_or_paths).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DataLoader._load_file_based_data() to call load_table_bulk() for multi-shard tables", "wid": "w2"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w2"], "notes": "Key design points:\n1. Call pragma_table_info once (using connection) to get col_names.\n2. Detect trailing delimiter on the FIRST shard only (has_trailing_delimiter).\n3. Build names/columns based on that detection - applies to all shards.\n4. Build INSERT SQL with array literal:\n INSERT INTO {table}\n SELECT {cols} FROM read_csv(['{esc_path1}', '{esc_path2}', ...], delim='|', ...)\n5. Use before/after COUNT(*) to get inserted row count (same as load_table()).\n6. Dry-run mode: capture_sql() for each file (or combined) then return 1000*N.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDBNativeHandler.load_table_bulk() using read_csv([list_of_paths])", "wid": "w3"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w2"], "notes": "Simpler than CSV - no pragma or trailing-delimiter detection needed.\nINSERT INTO {table} SELECT * FROM read_parquet(['{path1}', '{path2}', ...])\nUse before/after COUNT(*) to get inserted row count.\nDry-run: capture_sql() then return 1000*N.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDBParquetHandler.load_table_bulk() using read_parquet([list_of_paths])", "wid": "w4"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w3"], "notes": "Add to tests/unit/platforms/base/test_trailing_delimiter_loading.py.\nTest cases:\n- 3 shards, trailing delimiter: assert array syntax in SQL, single INSERT call, correct row count\n- 3 shards, no trailing delimiter: assert no _trailing_delimiter_ in SQL\n- dry-run mode: assert capture_sql called, return value == 3000\n- single-shard list delegates correctly (same result as load_table())\nMock call sequence: [pragma, COUNT_before, INSERT_bulk, COUNT_after]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for DuckDBNativeHandler.load_table_bulk() - correct row count, SQL shape, dry-run", "wid": "w5"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w4"], "notes": "Add to tests/unit/platforms/base/test_parquet_handlers.py.\nTest cases:\n- 2 shards: assert read_parquet([...]) array syntax, 3 execute calls, correct row count\n- dry-run: assert no connection.execute called, return == 2000\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for DuckDBParquetHandler.load_table_bulk() - correct row count, SQL shape, dry-run", "wid": "w6"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w5", "w6"], "notes": "At SF0.01 parallel=1, tables are single-file - verifies no regression.\nAdd a unit/integration test that exercises DataLoader with a mock multi-shard\ntable and confirms load_table_bulk() was called (not individual load_table()).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Smoke test: benchbox run --platform duckdb --benchmark tpch --scale 0.01 --phases load,power reports correct row counts", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add provenance to REQUIRED keys or emit an empty provenance:{} block", "why": "that would change canonical bytes for every existing run."}, {"INSTEAD": "(not specified)", "dont": "bump SCHEMA_VERSION for a purely additive optional block unless the schema policy requires it; if it does, update the accepted-prefix set and note it.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "hardcode funding strings in run.py", "why": "use FUNDING_SOURCES from item 1."}, {"INSTEAD": "(not specified)", "dont": "trust provenance.source as the trust label anywhere in this item (see G-source-on-producer).", "why": "(unstated)"}], "approach": "Follow the exact pattern schema.py already uses for optional sections\n('environment', 'tables', 'errors' are only added when non-empty). Gate the\nprovenance block on `if funding or source:`. Normalize funding through the\nitem-1 helper so an unknown CLI value can never reach the bundle (Click Choice\nalready constrains it, but normalize defends the python-API path). Keep\nresult_source purely advisory in the bundle; label resolution is item 4's job.\n", "blocked_reason": null, "category": "Feature", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": ["provenance-vocabulary-and-labels"], "description": "Make funding a declared property of a run, captured where the run happens.\nToday the schema-v2 bundle (benchbox/core/results/schema.py, CANONICAL_KEY_ORDER\n~lines 40-58, payload build ~lines 311-353) has no provenance/funding/source\nfields, and BenchmarkResults (benchbox/core/results/models.py:268-383) carries\nno funding attribute. The run command (benchbox/cli/commands/run.py) has no way\nto declare who paid for the run.\n\nThis item adds an OPTIONAL top-level \"provenance\" block to the bundle\n({ \"funding\": , \"source\": }), threads a\nfunding field through BenchmarkResults, and adds `benchbox run --funding\n` (plus a maintainer-only `--result-source` that only annotates intent;\nthe authoritative vendor label is still applied at merge in item 3). Absent\nprovenance block == today's behavior, so every existing bundle and test stays\nvalid.\n", "id": "bundle-schema-and-run-cli", "preserves": ["Canonical JSON bytes for a run without --funding are identical to pre-change output (no empty provenance block emitted).", "MINIMAL_BUNDLE and all existing schema-v2 fixtures validate unchanged (no provenance block required).", "SchemaV2Validator required-key set is unchanged; provenance is optional."], "prior_art": [{"concept": "benchbox/cli/commands/run.py:2655-2666 --publish-target/--publish-label options \u2014 reference (mirror option style; --funding lives next to these run-scoped options)", "decision": "reuse", "path": "benchbox/cli/commands/run.py:2655-2666"}, {"concept": "benchbox/cli/commands/run.py:2721 --publish-label help string \u2014 extend (already stale: omits unofficial-research, now also vendor-supplied; refresh while adding --funding here). Found by item-1 code review.", "decision": "extend", "path": "benchbox/cli/commands/run.py:2721"}, {"concept": "benchbox/core/results/models.py:268-383 BenchmarkResults \u2014 extend (add funding + optional result_source fields, defaulted so no call site breaks)", "decision": "extend", "path": "benchbox/core/results/models.py:268-383"}, {"concept": "benchbox/core/results/provenance.py (item 1) \u2014 reference (funding enum + normalize; the ONLY source of valid values)", "decision": "reuse", "path": "benchbox/core/results/provenance.py"}, {"concept": "benchbox/core/results/schema.py:40-58 CANONICAL_KEY_ORDER + :311-353 payload build \u2014 extend (add 'provenance' as an optional, canonically-ordered section like 'environment')", "decision": "extend", "path": "benchbox/core/results/schema.py:40-58"}, {"concept": "tests/unit/core/explorer_pipeline/conftest.py:15-84 MINIMAL_BUNDLE \u2014 reference (must still validate with no provenance block)", "decision": "reuse", "path": "tests/unit/core/explorer_pipeline/conftest.py:15-84"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "do_not_modify", "path_glob": "docs/development/browser-duckdb-schema.sql"}, {"kind": "do_not_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "do_not_modify", "path_glob": "scripts/validate_submission.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/result_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}], "state": "active", "title": "Emit a provenance block (funding) in schema-v2 bundles and add run --funding", "verifications": [{"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --phases power && uv run -- python -c \"import json,glob; b=json.load(open(sorted(glob.glob('benchmark_runs/results/*duckdb*.json'))[-1])); assert 'provenance' not in b, b.get('provenance')\"", "description": "Run without --funding produces a bundle with no provenance block", "expected": "no assertion error", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --phases power --funding personal && uv run -- python -c \"import json,glob; b=json.load(open(sorted(glob.glob('benchmark_runs/results/*duckdb*.json'))[-1])); assert b['provenance']['funding']=='personal'\"", "description": "Run WITH --funding personal emits a valid provenance block", "expected": "no assertion error", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/results/ tests/unit/core/explorer_pipeline/test_transformer.py -q", "description": "Existing schema + explorer fixtures still validate", "expected": "all pass", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --help | grep -- --funding", "description": "run --help lists --funding", "expected": "--funding option shown with the enum values", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "bundle-schema-and-run-cli", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add funding: str | None (default None) and result_source: str | None to BenchmarkResults; ensure result_factory/aggregation paths pass them through without requiring them.", "wid": "w1"}, {"evidence": null, "item_id": "bundle-schema-and-run-cli", "needs": ["w1"], "notes": "Normalize funding via provenance.normalize_funding(); add 'provenance' to CANONICAL_KEY_ORDER.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In schema.py, emit an optional 'provenance' section only when funding or source is present; keep the bundle byte-identical when absent.", "wid": "w2"}, {"evidence": null, "item_id": "bundle-schema-and-run-cli", "needs": ["w1", "w2"], "notes": "Default 'unspecified' -> emitted only if explicitly set; plumb into the result.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `benchbox run --funding` (Choice from FUNDING_SOURCES) and a maintainer-only `--result-source` advisory hint; update run --help.", "wid": "w3"}], "worktree": "results-labels-funding"} +{"anti_patterns": [{"dont": "concatenate shard CSV files on disk", "instead": "use the glob pattern in SQL instead", "why": "that requires temp storage and file I/O"}, {"dont": "hardcode the glob separator as '.'", "instead": "derive the common prefix programmatically from the actual file names", "why": "shard naming conventions may differ (e.g., _N vs .N)"}, {"dont": "skip the fallback to per-file loop", "instead": "the fallback ensures correctness in all cases", "why": "shards may legitimately be in different dirs for re-loaded manifests"}], "approach": "1. Read ClickHouseNativeHandler.load_table() to understand the current SQL\n pattern, column schema retrieval, and dry-run handling.\n2. Implement load_table_bulk(): derive glob from the shard list, build a\n single INSERT ... SELECT * FROM file(glob) statement, execute with\n before/after COUNT(*) for the row count delta.\n3. Add guard: if shard paths span multiple directories, fall back to\n super().load_table_bulk() (the default loop).\n4. For ClickHouse Cloud: audit whether the cloud adapter already handles\n shards in bulk; if so, just add tests.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-22T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-clickhouse", "id": 108, "reason": "CSV via file() is sufficient for current benchmarks; Native format has different escaping requirements", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Speculative optimization; CSV loading deemed sufficient with no concrete trigger identified to revisit Native format.", "summary": "Investigate columnar-native bulk loading (ClickHouse Native format instead of CSV)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-clickhouse", "id": 109, "reason": "PostgreSQL COPY is fundamentally single-file per statement; would require UNION ALL construction or a PL/pgSQL wrapper - different scope", "resolution": "promoted", "resolved_item": "bulk-loading-for-postgresql-copy-from-program-or-copy-from-s", "resolved_reason": null, "summary": "Bulk loading for PostgreSQL COPY FROM PROGRAM or COPY FROM STDIN multi-file"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-clickhouse", "id": 110, "reason": "Cloud adapter (_load_data_via_s3, _load_data_via_gcs) has its own per-file upload+insert loop that bypasses DataLoader and ClickHouseNativeHandler entirely. Bulk loading requires restructuring the upload pipeline (upload all first, then single INSERT with s3() glob) - a larger refactor tracked separately.", "resolution": "promoted", "resolved_item": "bulk-s3-gcs-loading-for-clickhouse-cloud-adapter-using-s3-gl", "resolved_reason": null, "summary": "Bulk S3/GCS loading for ClickHouse Cloud adapter using s3() glob patterns"}], "deps": ["bulk-shard-loading-duckdb"], "description": "ClickHouseNativeHandler.load_table() currently processes one shard file per\ncall. For a sharded benchmark table (e.g., lineitem.tbl.1 \u2026 lineitem.tbl.10)\nthe DataLoader loop issues 10 separate INSERT INTO ... SELECT * FROM file(path)\nstatements.\n\nClickHouse's table function file() (and s3() for cloud) support glob patterns:\n file('/data/lineitem*.tbl', 'CSV', 'col1 UInt64, ...')\nA single INSERT using a glob covers all shards in one query-plan pass.\n\nThis TODO implements load_table_bulk() in ClickHouseNativeHandler, leveraging\nthe load_table_bulk() interface introduced in bulk-shard-loading-duckdb.\n\nScope is ClickHouse (self-hosted) and ClickHouse Cloud adapters only.\nIt explicitly excludes platforms backed by DelimitedFileHandler (SQLite,\nPostgreSQL, DataFusion, etc.) which have no SQL multi-file primitive.", "id": "bulk-shard-loading-clickhouse", "preserves": ["All existing tests/unit/platforms/test_clickhouse_adapter.py tests pass without modification", "All fast-suite tests continue to pass: uv run -- python -m pytest -m fast -q must report 0 failures", "ClickHouseNativeHandler.load_table() (single-file path) continues to work unchanged", "Dry-run mode still captures SQL without executing it", "load_table_bulk() falls back to the default per-file loop when shard paths are in different directories"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/clickhouse/workload.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse_cloud.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_clickhouse_adapter.py"}], "state": "done", "title": "Replace per-shard INSERT loops with glob-pattern bulk INSERT for ClickHouseNativeHandler", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py -v", "description": "ClickHouse handler unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "bulk-shard-loading-clickhouse", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py -v -k bulk", "description": "New bulk loading tests pass", "expected": "all bulk tests pass, 0 failures", "item_id": "bulk-shard-loading-clickhouse", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Full fast suite passes (no regression)", "expected": "all tests pass, 0 failures", "item_id": "bulk-shard-loading-clickhouse", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": [], "notes": "File: benchbox/platforms/clickhouse/workload.py (and any subclass in clickhouse_cloud.py).\nConfirm the exact SQL pattern used (file(), s3(), or COPY), how file paths are\nescaped, and whether there is a dry-run mode to preserve.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit ClickHouseNativeHandler: locate load_table() and identify all execute call sites", "wid": "w1"}, {"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": ["w1"], "notes": "Algorithm:\n1. Derive a glob from the shard list. All shards for a table share a common\n stem: if paths are ['lineitem.tbl.1', 'lineitem.tbl.2', ...] in the same\n directory, the glob is 'lineitem.tbl.*'.\n2. Validate: all paths share the same parent dir and the same stem prefix.\n If not (edge case), fall back to the default loop via super().load_table_bulk().\n3. Build SQL:\n INSERT INTO {table}\n SELECT * FROM file('{dir}/{stem}.*', 'CSV', '{col_schema}')\n4. Execute and return COUNT delta (before/after) as for DuckDB.\n\nThe ClickHouse file() function requires an explicit column schema string for\nCSV - derive it from DESCRIBE TABLE {table} (or equivalent).\nDry-run: capture SQL then return len(file_paths)*1000.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ClickHouseNativeHandler.load_table_bulk() using file() glob pattern for self-hosted ClickHouse", "wid": "w2"}, {"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": ["w2"], "notes": "ClickHouse Cloud benchmarks upload files to S3 before loading.\nTIME CAP: max 2 hrs. If the audit shows the cloud path already issues a\nsingle COPY / INSERT for all shards, add a confirming test and close this unit.\nIf the cloud path still issues one INSERT per shard (using s3() or url()),\nimplement load_table_bulk() analogously to w2 using s3() glob patterns.\nDO NOT scope-creep into refactoring the full cloud upload pipeline -\nonly add/override load_table_bulk() in the cloud adapter class.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle ClickHouse Cloud: verify whether cloud path already bulk-loads; add tests or implement if not", "wid": "w3"}, {"evidence": null, "item_id": "bulk-shard-loading-clickhouse", "needs": ["w2"], "notes": "Test cases:\n- 4 shards in same dir: assert glob pattern in SQL, single execute call, correct row count\n- Shards in different directories: assert fallback to per-shard loop (no glob)\n- Single-shard list: assert result equals load_table() result\n- Dry-run: assert no execute on real connection, return == 4000\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for ClickHouseNativeHandler.load_table_bulk() - glob SQL, row count, fallback, dry-run", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "call has_trailing_delimiter on every shard in load_table_bulk()", "instead": "sample only the first path", "why": "all shards of the same TBL table have the same delimiter pattern"}, {"dont": "call pragma_table_info once per shard in load_table_bulk()", "instead": "call it once for the first path", "why": "the schema is the same for all shards of a table"}, {"dont": "change _load_single_file()", "instead": "leave it untouched", "why": "it is still used for the single-file (non-list) branch"}, {"dont": "change the load_table() single-file method signatures", "instead": "only add load_table_bulk() as an override", "why": "existing tests and non-DuckDB platforms depend on them"}, {"dont": "use a glob string instead of an explicit path array in load_table_bulk()", "instead": "use the explicit list passed in", "why": "shard files may not be in the same directory or may have non-glob-friendly names"}], "approach": "1. In FileFormatHandler (data_loading.py), add:\n def load_table_bulk(self, table_name, file_paths, connection, benchmark, logger) -> int:\n total = 0\n for path in file_paths:\n total += self.load_table(table_name, path, connection, benchmark, logger)\n return total\n\n2. In DataLoader._load_file_based_data() (lines 1773-1782), replace the\n for-loop + _load_single_file() pattern with:\n handler = self.handler_factory(file_path_or_paths[0], self.adapter, self.benchmark)\n total_rows = handler.load_table_bulk(table_name, [Path(p) for p in file_path_or_paths],\n self.connection, self.benchmark, self.adapter.logger)\n This calls the factory once to pick the correct handler, then delegates\n the entire shard list to load_table_bulk().\n\n3. In DuckDBNativeHandler, override load_table_bulk():\n a. Call pragma_table_info once to get col_names.\n b. Detect trailing delimiter on file_paths[0].\n c. Escape each path: [escape_sql_string_literal(str(p)) for p in file_paths].\n d. Build array literal: \"['\" + \"', '\".join(escaped) + \"']\".\n e. Build INSERT SQL using the array literal inside read_csv().\n f. COUNT before, execute INSERT, COUNT after, return delta.\n\n4. In DuckDBParquetHandler, override load_table_bulk() similarly:\n Build INSERT ... SELECT * FROM read_parquet(['p1', 'p2', ...]) and use\n before/after COUNT(*).\n\n5. Add tests before touching production code (TDD: write failing tests for\n load_table_bulk, then implement to make them pass).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-22T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "bulk-shard-loading-duckdb", "id": 111, "reason": "Lower volume - Delta tables are typically single files; validate demand before adding complexity", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly gated on validating demand first; no evidence demand for Delta/Vortex/DuckLake bulk loading was assessed.", "summary": "Bulk loading for DuckDB Delta / Vortex / DuckLake handlers"}], "deps": [], "description": "When a benchmark table is generated with parallel>1, its data is split across\nN shard files (e.g., lineitem.tbl.1, lineitem.tbl.2, ..., lineitem.tbl.N).\nThe base DataLoader._load_file_based_data() currently calls handler.load_table()\nonce per shard in a sequential loop, producing N separate INSERT statements.\n\nFor DuckDB this is inefficient: DuckDB's read_csv() and read_parquet() both\naccept an array of file paths or a glob pattern and process all files in a\nsingle optimised scan, avoiding N round-trips and N separate query plans.\n\nThis TODO adds a load_table_bulk() interface to FileFormatHandler, changes the\nDataLoader loop to use it, and implements the DuckDB-native multi-file variant\nfor DuckDBNativeHandler (CSV/TBL) and DuckDBParquetHandler.\n\nExpected improvement (TPC-H SF1, 4-shard lineitem):\n- Before: 4 \u00d7 (pragma + COUNT_before + INSERT + COUNT_after) = 16 execute calls\n- After: 1 \u00d7 (pragma + COUNT_before + bulk INSERT + COUNT_after) = 4 execute calls\n- Data transfer and query planning overhead reduced ~75% for sharded tables\n\nPostgreSQL, SQLite, DataFusion, and other platforms that use DelimitedFileHandler\n/ executemany() are NOT in scope - they have no SQL multi-file primitive.", "id": "bulk-shard-loading-duckdb", "preserves": ["All 12,970 existing fast-suite tests continue to pass", "DataLoader default behaviour (non-DuckDB platforms, single-file tables) is unaffected", "Dry-run mode still captures SQL without executing it", "DuckDBNativeHandler.load_table() (single-file path) continues to work unchanged for non-sharded tables", "DuckDBParquetHandler.load_table() continues to work unchanged", "benchbox run --platform duckdb --benchmark tpch --scale 1 --phases load produces total rows_loaded == 8,661,245 (lineitem 6,001,215 + orders 1,500,000 + ...)"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "tests/unit/platforms/test_duckdb_adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/test_parquet_handlers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/test_trailing_delimiter_loading.py"}], "state": "done", "title": "Replace per-shard INSERT loops with single bulk INSERT for DuckDB handlers", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "All existing fast-suite tests pass after the framework change", "expected": "12970+ passed, 0 failed", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_trailing_delimiter_loading.py -v -k bulk", "description": "New load_table_bulk unit tests pass for DuckDBNativeHandler", "expected": "all bulk tests pass, 0 failures", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_parquet_handlers.py -v -k bulk", "description": "New load_table_bulk unit tests pass for DuckDBParquetHandler", "expected": "all bulk tests pass, 0 failures", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -k 'duckdb and load' -v", "description": "DuckDB TPC-H SF0.01 smoke test completes with correct row counts", "expected": "all tests pass, 0 failures", "item_id": "bulk-shard-loading-duckdb", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": [], "notes": "Signature: load_table_bulk(table_name, file_paths: list[Path], connection, benchmark, logger) -> int\nDefault: iterate file_paths, call self.load_table() for each, sum results.\nThis makes the interface backward-compatible - all existing handlers auto-inherit\ncorrect (if non-bulk) behaviour.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add load_table_bulk() to FileFormatHandler base class with default loop implementation", "wid": "w1"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w1"], "notes": "In the isinstance(file_path_or_paths, list) branch, instead of:\n for shard_path in file_path_or_paths:\n rows += self._load_single_file(table_name, shard_path)\nDo:\n handler = self.handler_factory(file_path_or_paths[0], ...)\n total_rows = handler.load_table_bulk(table_name, [Path(p) for p in file_path_or_paths],\n self.connection, self.benchmark, self.adapter.logger)\nThe handler_factory is called once (on the first shard) to select the right handler type.\nThe shard_count display in the log line must still be len(file_path_or_paths).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DataLoader._load_file_based_data() to call load_table_bulk() for multi-shard tables", "wid": "w2"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w2"], "notes": "Key design points:\n1. Call pragma_table_info once (using connection) to get col_names.\n2. Detect trailing delimiter on the FIRST shard only (has_trailing_delimiter).\n3. Build names/columns based on that detection - applies to all shards.\n4. Build INSERT SQL with array literal:\n INSERT INTO {table}\n SELECT {cols} FROM read_csv(['{esc_path1}', '{esc_path2}', ...], delim='|', ...)\n5. Use before/after COUNT(*) to get inserted row count (same as load_table()).\n6. Dry-run mode: capture_sql() for each file (or combined) then return 1000*N.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDBNativeHandler.load_table_bulk() using read_csv([list_of_paths])", "wid": "w3"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w2"], "notes": "Simpler than CSV - no pragma or trailing-delimiter detection needed.\nINSERT INTO {table} SELECT * FROM read_parquet(['{path1}', '{path2}', ...])\nUse before/after COUNT(*) to get inserted row count.\nDry-run: capture_sql() then return 1000*N.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDBParquetHandler.load_table_bulk() using read_parquet([list_of_paths])", "wid": "w4"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w3"], "notes": "Add to tests/unit/platforms/base/test_trailing_delimiter_loading.py.\nTest cases:\n- 3 shards, trailing delimiter: assert array syntax in SQL, single INSERT call, correct row count\n- 3 shards, no trailing delimiter: assert no _trailing_delimiter_ in SQL\n- dry-run mode: assert capture_sql called, return value == 3000\n- single-shard list delegates correctly (same result as load_table())\nMock call sequence: [pragma, COUNT_before, INSERT_bulk, COUNT_after]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for DuckDBNativeHandler.load_table_bulk() - correct row count, SQL shape, dry-run", "wid": "w5"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w4"], "notes": "Add to tests/unit/platforms/base/test_parquet_handlers.py.\nTest cases:\n- 2 shards: assert read_parquet([...]) array syntax, 3 execute calls, correct row count\n- dry-run: assert no connection.execute called, return == 2000\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for DuckDBParquetHandler.load_table_bulk() - correct row count, SQL shape, dry-run", "wid": "w6"}, {"evidence": null, "item_id": "bulk-shard-loading-duckdb", "needs": ["w5", "w6"], "notes": "At SF0.01 parallel=1, tables are single-file - verifies no regression.\nAdd a unit/integration test that exercises DataLoader with a mock multi-shard\ntable and confirms load_table_bulk() was called (not individual load_table()).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Smoke test: benchbox run --platform duckdb --benchmark tpch --scale 0.01 --phases load,power reports correct row counts", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add provenance to REQUIRED keys or emit an empty provenance:{} block", "instead": "(not specified)", "why": "that would change canonical bytes for every existing run."}, {"dont": "bump SCHEMA_VERSION for a purely additive optional block unless the schema policy requires it; if it does, update the accepted-prefix set and note it.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "hardcode funding strings in run.py", "instead": "(not specified)", "why": "use FUNDING_SOURCES from item 1."}, {"dont": "trust provenance.source as the trust label anywhere in this item (see G-source-on-producer).", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Follow the exact pattern schema.py already uses for optional sections\n('environment', 'tables', 'errors' are only added when non-empty). Gate the\nprovenance block on `if funding or source:`. Normalize funding through the\nitem-1 helper so an unknown CLI value can never reach the bundle (Click Choice\nalready constrains it, but normalize defends the python-API path). Keep\nresult_source purely advisory in the bundle; label resolution is item 4's job.\n", "blocked_reason": null, "category": "Feature", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": ["provenance-vocabulary-and-labels"], "description": "Make funding a declared property of a run, captured where the run happens.\nToday the schema-v2 bundle (benchbox/core/results/schema.py, CANONICAL_KEY_ORDER\n~lines 40-58, payload build ~lines 311-353) has no provenance/funding/source\nfields, and BenchmarkResults (benchbox/core/results/models.py:268-383) carries\nno funding attribute. The run command (benchbox/cli/commands/run.py) has no way\nto declare who paid for the run.\n\nThis item adds an OPTIONAL top-level \"provenance\" block to the bundle\n({ \"funding\": , \"source\": }), threads a\nfunding field through BenchmarkResults, and adds `benchbox run --funding\n` (plus a maintainer-only `--result-source` that only annotates intent;\nthe authoritative vendor label is still applied at merge in item 3). Absent\nprovenance block == today's behavior, so every existing bundle and test stays\nvalid.\n", "id": "bundle-schema-and-run-cli", "preserves": ["Canonical JSON bytes for a run without --funding are identical to pre-change output (no empty provenance block emitted).", "MINIMAL_BUNDLE and all existing schema-v2 fixtures validate unchanged (no provenance block required).", "SchemaV2Validator required-key set is unchanged; provenance is optional."], "prior_art": [{"concept": "benchbox/cli/commands/run.py:2655-2666 --publish-target/--publish-label options \u2014 reference (mirror option style; --funding lives next to these run-scoped options)", "decision": "reuse", "path": "benchbox/cli/commands/run.py:2655-2666"}, {"concept": "benchbox/cli/commands/run.py:2721 --publish-label help string \u2014 extend (already stale: omits unofficial-research, now also vendor-supplied; refresh while adding --funding here). Found by item-1 code review.", "decision": "extend", "path": "benchbox/cli/commands/run.py:2721"}, {"concept": "benchbox/core/results/models.py:268-383 BenchmarkResults \u2014 extend (add funding + optional result_source fields, defaulted so no call site breaks)", "decision": "extend", "path": "benchbox/core/results/models.py:268-383"}, {"concept": "benchbox/core/results/provenance.py (item 1) \u2014 reference (funding enum + normalize; the ONLY source of valid values)", "decision": "reuse", "path": "benchbox/core/results/provenance.py"}, {"concept": "benchbox/core/results/schema.py:40-58 CANONICAL_KEY_ORDER + :311-353 payload build \u2014 extend (add 'provenance' as an optional, canonically-ordered section like 'environment')", "decision": "extend", "path": "benchbox/core/results/schema.py:40-58"}, {"concept": "tests/unit/core/explorer_pipeline/conftest.py:15-84 MINIMAL_BUNDLE \u2014 reference (must still validate with no provenance block)", "decision": "reuse", "path": "tests/unit/core/explorer_pipeline/conftest.py:15-84"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "do_not_modify", "path_glob": "docs/development/browser-duckdb-schema.sql"}, {"kind": "do_not_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "do_not_modify", "path_glob": "scripts/validate_submission.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/result_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}], "state": "active", "title": "Emit a provenance block (funding) in schema-v2 bundles and add run --funding", "verifications": [{"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --phases power && uv run -- python -c \"import json,glob; b=json.load(open(sorted(glob.glob('benchmark_runs/results/*duckdb*.json'))[-1])); assert 'provenance' not in b, b.get('provenance')\"", "description": "Run without --funding produces a bundle with no provenance block", "expected": "no assertion error", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --phases power --funding personal && uv run -- python -c \"import json,glob; b=json.load(open(sorted(glob.glob('benchmark_runs/results/*duckdb*.json'))[-1])); assert b['provenance']['funding']=='personal'\"", "description": "Run WITH --funding personal emits a valid provenance block", "expected": "no assertion error", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/results/ tests/unit/core/explorer_pipeline/test_transformer.py -q", "description": "Existing schema + explorer fixtures still validate", "expected": "all pass", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --help | grep -- --funding", "description": "run --help lists --funding", "expected": "--funding option shown with the enum values", "item_id": "bundle-schema-and-run-cli", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "bundle-schema-and-run-cli", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add funding: str | None (default None) and result_source: str | None to BenchmarkResults; ensure result_factory/aggregation paths pass them through without requiring them.", "wid": "w1"}, {"evidence": null, "item_id": "bundle-schema-and-run-cli", "needs": ["w1"], "notes": "Normalize funding via provenance.normalize_funding(); add 'provenance' to CANONICAL_KEY_ORDER.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In schema.py, emit an optional 'provenance' section only when funding or source is present; keep the bundle byte-identical when absent.", "wid": "w2"}, {"evidence": null, "item_id": "bundle-schema-and-run-cli", "needs": ["w1", "w2"], "notes": "Default 'unspecified' -> emitted only if explicitly set; plumb into the result.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `benchbox run --funding` (Choice from FUNDING_SOURCES) and a maintainer-only `--result-source` advisory hint; update run --help.", "wid": "w3"}], "worktree": "results-labels-funding"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-23T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox v0.1.0 installed from PyPI (wheel) cannot run TPC-H or TPC-DS benchmarks.\nQuery template files (.sql for TPC-H, .tpl for TPC-DS) live in `_sources/` at the\nproject root - outside the `benchbox/` package tree - and are never included in the\nwheel. The pyproject.toml `[tool.setuptools.package-data]` `\"*\"` glob is ineffective\n(patterns are relative to package dirs, not project root). MANIFEST.in correctly\nincludes them for sdist only.\n\nRoot cause: wheels don't use MANIFEST.in; only package-data matters. Since `_sources/`\nis outside the package, setuptools cannot find the files.\n\nSolution: Create shared `templates/` directories inside `benchbox/_binaries/tpc-{h,ds}/`\n(already in the package tree and included in wheels). Update template resolution code\nto look there first, with `_sources/` fallback for development.", "id": "bundle-tpc-query-templates-in-wheel", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "Bundle TPC-H/TPC-DS query templates inside package for wheel distribution", "verifications": [], "work": [], "worktree": "release-readiness"} -{"anti_patterns": [{"INSTEAD": "prefer REPOINT or REMOVE when the target was never meant to exist.", "dont": "create empty stub docs solely to satisfy a link", "why": "a stub is worse than an honest rewrite/removal"}, {"INSTEAD": "drive the broken COUNT to zero by fixing links.", "dont": "delete or blank the baseline file to make the check pass", "why": "that disables the regression gate"}, {"INSTEAD": "(not specified)", "dont": "re-add entries to the baseline to silence the count", "why": "the baseline is debt being burned down, not a suppression list."}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-26T00:00:00Z", "deferrals": [], "deps": [], "description": "PR #683 (`dry-run-followup-broken-getting-started-link`) added\n`scripts/check_doc_relative_links.py` \u2014 a repo-local Markdown\nrelative-link checker wired into the blocking `make docs-validate`\ngate. Running it surfaced **42 pre-existing broken relative links**\nacross ~20 docs files (the originating dry-run knew of only one,\n`getting-started.rst`, already fixed). Those 42 were recorded in\n`scripts/doc_relative_link_baseline.txt` so the gate blocks *new*\nbreakage immediately without forcing a same-PR mass cleanup.\n\nThis TODO is that cleanup: fix or remove every baselined link and\ndrive the baseline to zero data entries, so the gate enforces\n\"no broken repo-local relative links\" repo-wide rather than only\n\"no new ones\".\n\nTarget groups in the baseline as of 2026-05-26 (regenerate to\nconfirm before starting \u2014 counts may shift as docs change):\n - `../benchmarks/tpch.md` x7 -> `../benchmarks/tpc-h.md` (target exists)\n - `../benchmarks/tpcds.md` x4 -> `../benchmarks/tpc-ds.md` (target exists)\n - `../reference/cli-reference.md` x5 + `../../reference/cli-reference.md` x2\n -> no `docs/reference/cli-reference.md`; locate the real CLI reference\n doc and repoint, or rewrite the links\n - `README.md` x3 (from `docs/benchmarks/*`) -> likely `index.md`\n - `../reference/result_schema_v1.md` x2, `../reference/results-schema.md` x1\n -> schema-doc references; find the canonical schema doc\n - singles: `synapse.md`, `pyspark.md`, `glue.md`,\n `platform-development.md`, `python-api/results.md`,\n `getting-started.md` / `examples.md` / `configuration.md`\n (from `docs/reference/api-reference.md`),\n `benchmarks/tpc-ds.md` (wrong relative depth from `guides/tpc/`),\n `../advanced/tuning.md`, `../advanced/cost-optimization.md`,\n `../api/query-plan-models.md`, `../performance/index.md`,\n `../usage/result-analysis.md`, `../getting-started/quick-start.md`,\n `./profiling.md`, `./duplication-inventory.csv`,\n `./unified_frame_any_survey.csv`\n", "id": "burn-down-baselined-broken-doc-links", "preserves": ["Existing relative links in docs/ that currently resolve continue to resolve (no collateral breakage).", "External-link handling is unchanged \u2014 http/https links remain the Sphinx linkcheck's responsibility, not this checker's.", "The checker's baseline + anti-rot mechanism stays intact \u2014 do not weaken `check_doc_relative_links.py` to pass; fix the links."], "prior_art": [{"concept": "docs/benchmarks/tpc-h.md, docs/benchmarks/tpc-ds.md \u2014 existing canonical targets for the tpch/tpcds rename cluster (REUSE; do not create new benchmark pages).", "decision": "reuse", "path": "docs/benchmarks/tpc-h.md"}, {"concept": "scripts/check_doc_relative_links.py \u2014 the checker introduced in PR #683; REUSE its `--update-baseline` to regenerate the baseline after each batch of fixes.", "decision": "reuse", "path": "scripts/check_doc_relative_links.py"}, {"concept": "scripts/doc_relative_link_baseline.txt \u2014 the 42-entry baseline this TODO empties; the input artifact, not new infrastructure.", "decision": "reuse", "path": "scripts/doc_relative_link_baseline.txt"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "scripts/check_doc_relative_links.py"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "scripts/doc_relative_link_baseline.txt"}], "state": "planning", "title": "Burn down the 42 baselined broken docs relative links", "verifications": [{"command": "uv run -- python scripts/check_doc_relative_links.py", "description": "No broken repo-local relative links remain", "expected": "exit 0 with 'broken total : 0'", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 1}, {"command": "test \"$(grep -cvE '^#|^$' scripts/doc_relative_link_baseline.txt)\" = 0", "description": "Baseline has no data entries (only header comments)", "expected": "exit 0 (no data rows remain)", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 2}, {"command": "! rg -n 'benchmarks/tpch\\.md|benchmarks/tpcds\\.md' docs/", "description": "Spot-check: the wrong-name benchmark links are gone (full gate is check 1)", "expected": "no matches", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 3}, {"command": "make docs-validate", "description": "Docs validation gate passes end to end", "expected": "exit 0", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": [], "notes": "Bind the cited evidence before editing. On the implementation branch run\n`uv run -- python scripts/check_doc_relative_links.py --update-baseline`\nand `git diff scripts/doc_relative_link_baseline.txt`. Record in the PR\ndescription: the develop SHA, the regenerated data-entry count, and any\ndrift from the 42 captured 2026-05-26 (the per-target counts in this\nTODO's description are that snapshot). If the count shifted materially,\nre-derive the w2-w4 groupings before proceeding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-validate the baseline count against current develop", "wid": "w0"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w0"], "notes": "For each (source, target) pair in the regenerated baseline assign one\ndisposition:\n - RENAME: target exists under a corrected name/path (fix the link).\n - REPOINT: the intended doc exists elsewhere (rewrite the link).\n - REMOVE: the referenced doc was never written / has been deleted and\n has no successor (delete the link, keep the surrounding prose sane).\n - CREATE: a genuinely-missing doc that *should* exist (rare; prefer\n REPOINT/REMOVE \u2014 do not stub empty files just to satisfy the gate).\nRecord the table in the PR description; it is the audit trail for w2-w4.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Triage all baselined links into a disposition table", "wid": "w1"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w1"], "notes": "The high-confidence cluster (targets already exist):\n - `../benchmarks/tpch.md` -> `../benchmarks/tpc-h.md` (7 refs)\n - `../benchmarks/tpcds.md` -> `../benchmarks/tpc-ds.md` (4 refs)\n - `docs/guides/tpc/*` `benchmarks/tpc-ds.md` -> correct relative depth\n to the real `docs/benchmarks/tpc-ds.md`.\nAfter editing, regenerate the baseline and confirm these entries drop\nout (the checker reports a stale-baseline error until you do).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Apply the unambiguous benchmark-filename renames", "wid": "w2"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w1"], "notes": "`docs/reference/cli-reference.md` does not exist. Find the actual CLI\nreference doc (search `docs/reference/` and the toctree in\n`docs/index.rst`) and repoint all 7 references, or rewrite them if the\nCLI is documented under a different page. Regenerate the baseline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Resolve the cli-reference.md cluster (7 refs)", "wid": "w3"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w1"], "notes": "Work through the remaining entries per their w1 disposition: README ->\nindex, schema-doc references, platform cross-links (synapse/pyspark/\nglue/platform-development), api-reference.md's getting-started/examples/\nconfiguration links, the advanced/performance/usage/api targets, and the\ntwo `.csv` data-file links (confirm whether the CSVs were removed and\nremove or repoint accordingly). Fix local links only; leave external\n(http/https) links to the Sphinx linkcheck. Regenerate the baseline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Resolve the remaining single-target broken links", "wid": "w4"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w2", "w3", "w4"], "notes": "Final `--update-baseline` should leave zero data entries. Run\n`uv run -- python scripts/check_doc_relative_links.py` (reports\n`broken total : 0`, exit 0) and `make docs-validate` (exit 0). The goal\nis an empty data set. Note `--update-baseline` rewrites the file as the\nfixed header comments plus sorted entries \u2014 it does NOT preserve\nper-entry inline comments, so a genuinely-intentional exception must be\nfixed/removed rather than annotated in place (or hand-added below the\nheader and never regenerated over). Standard cycle:\n`make lint format typecheck`, `make pr-preflight`, `make pr-open`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Drive the baseline to zero and confirm the gate is green", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "create empty stub docs solely to satisfy a link", "instead": "prefer REPOINT or REMOVE when the target was never meant to exist.", "why": "a stub is worse than an honest rewrite/removal"}, {"dont": "delete or blank the baseline file to make the check pass", "instead": "drive the broken COUNT to zero by fixing links.", "why": "that disables the regression gate"}, {"dont": "re-add entries to the baseline to silence the count", "instead": "(not specified)", "why": "the baseline is debt being burned down, not a suppression list."}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-26T00:00:00Z", "deferrals": [], "deps": [], "description": "PR #683 (`dry-run-followup-broken-getting-started-link`) added\n`scripts/check_doc_relative_links.py` \u2014 a repo-local Markdown\nrelative-link checker wired into the blocking `make docs-validate`\ngate. Running it surfaced **42 pre-existing broken relative links**\nacross ~20 docs files (the originating dry-run knew of only one,\n`getting-started.rst`, already fixed). Those 42 were recorded in\n`scripts/doc_relative_link_baseline.txt` so the gate blocks *new*\nbreakage immediately without forcing a same-PR mass cleanup.\n\nThis TODO is that cleanup: fix or remove every baselined link and\ndrive the baseline to zero data entries, so the gate enforces\n\"no broken repo-local relative links\" repo-wide rather than only\n\"no new ones\".\n\nTarget groups in the baseline as of 2026-05-26 (regenerate to\nconfirm before starting \u2014 counts may shift as docs change):\n - `../benchmarks/tpch.md` x7 -> `../benchmarks/tpc-h.md` (target exists)\n - `../benchmarks/tpcds.md` x4 -> `../benchmarks/tpc-ds.md` (target exists)\n - `../reference/cli-reference.md` x5 + `../../reference/cli-reference.md` x2\n -> no `docs/reference/cli-reference.md`; locate the real CLI reference\n doc and repoint, or rewrite the links\n - `README.md` x3 (from `docs/benchmarks/*`) -> likely `index.md`\n - `../reference/result_schema_v1.md` x2, `../reference/results-schema.md` x1\n -> schema-doc references; find the canonical schema doc\n - singles: `synapse.md`, `pyspark.md`, `glue.md`,\n `platform-development.md`, `python-api/results.md`,\n `getting-started.md` / `examples.md` / `configuration.md`\n (from `docs/reference/api-reference.md`),\n `benchmarks/tpc-ds.md` (wrong relative depth from `guides/tpc/`),\n `../advanced/tuning.md`, `../advanced/cost-optimization.md`,\n `../api/query-plan-models.md`, `../performance/index.md`,\n `../usage/result-analysis.md`, `../getting-started/quick-start.md`,\n `./profiling.md`, `./duplication-inventory.csv`,\n `./unified_frame_any_survey.csv`\n", "id": "burn-down-baselined-broken-doc-links", "preserves": ["Existing relative links in docs/ that currently resolve continue to resolve (no collateral breakage).", "External-link handling is unchanged \u2014 http/https links remain the Sphinx linkcheck's responsibility, not this checker's.", "The checker's baseline + anti-rot mechanism stays intact \u2014 do not weaken `check_doc_relative_links.py` to pass; fix the links."], "prior_art": [{"concept": "docs/benchmarks/tpc-h.md, docs/benchmarks/tpc-ds.md \u2014 existing canonical targets for the tpch/tpcds rename cluster (REUSE; do not create new benchmark pages).", "decision": "reuse", "path": "docs/benchmarks/tpc-h.md"}, {"concept": "scripts/check_doc_relative_links.py \u2014 the checker introduced in PR #683; REUSE its `--update-baseline` to regenerate the baseline after each batch of fixes.", "decision": "reuse", "path": "scripts/check_doc_relative_links.py"}, {"concept": "scripts/doc_relative_link_baseline.txt \u2014 the 42-entry baseline this TODO empties; the input artifact, not new infrastructure.", "decision": "reuse", "path": "scripts/doc_relative_link_baseline.txt"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "scripts/check_doc_relative_links.py"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "scripts/doc_relative_link_baseline.txt"}], "state": "planning", "title": "Burn down the 42 baselined broken docs relative links", "verifications": [{"command": "uv run -- python scripts/check_doc_relative_links.py", "description": "No broken repo-local relative links remain", "expected": "exit 0 with 'broken total : 0'", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 1}, {"command": "test \"$(grep -cvE '^#|^$' scripts/doc_relative_link_baseline.txt)\" = 0", "description": "Baseline has no data entries (only header comments)", "expected": "exit 0 (no data rows remain)", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 2}, {"command": "! rg -n 'benchmarks/tpch\\.md|benchmarks/tpcds\\.md' docs/", "description": "Spot-check: the wrong-name benchmark links are gone (full gate is check 1)", "expected": "no matches", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 3}, {"command": "make docs-validate", "description": "Docs validation gate passes end to end", "expected": "exit 0", "item_id": "burn-down-baselined-broken-doc-links", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": [], "notes": "Bind the cited evidence before editing. On the implementation branch run\n`uv run -- python scripts/check_doc_relative_links.py --update-baseline`\nand `git diff scripts/doc_relative_link_baseline.txt`. Record in the PR\ndescription: the develop SHA, the regenerated data-entry count, and any\ndrift from the 42 captured 2026-05-26 (the per-target counts in this\nTODO's description are that snapshot). If the count shifted materially,\nre-derive the w2-w4 groupings before proceeding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-validate the baseline count against current develop", "wid": "w0"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w0"], "notes": "For each (source, target) pair in the regenerated baseline assign one\ndisposition:\n - RENAME: target exists under a corrected name/path (fix the link).\n - REPOINT: the intended doc exists elsewhere (rewrite the link).\n - REMOVE: the referenced doc was never written / has been deleted and\n has no successor (delete the link, keep the surrounding prose sane).\n - CREATE: a genuinely-missing doc that *should* exist (rare; prefer\n REPOINT/REMOVE \u2014 do not stub empty files just to satisfy the gate).\nRecord the table in the PR description; it is the audit trail for w2-w4.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Triage all baselined links into a disposition table", "wid": "w1"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w1"], "notes": "The high-confidence cluster (targets already exist):\n - `../benchmarks/tpch.md` -> `../benchmarks/tpc-h.md` (7 refs)\n - `../benchmarks/tpcds.md` -> `../benchmarks/tpc-ds.md` (4 refs)\n - `docs/guides/tpc/*` `benchmarks/tpc-ds.md` -> correct relative depth\n to the real `docs/benchmarks/tpc-ds.md`.\nAfter editing, regenerate the baseline and confirm these entries drop\nout (the checker reports a stale-baseline error until you do).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Apply the unambiguous benchmark-filename renames", "wid": "w2"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w1"], "notes": "`docs/reference/cli-reference.md` does not exist. Find the actual CLI\nreference doc (search `docs/reference/` and the toctree in\n`docs/index.rst`) and repoint all 7 references, or rewrite them if the\nCLI is documented under a different page. Regenerate the baseline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Resolve the cli-reference.md cluster (7 refs)", "wid": "w3"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w1"], "notes": "Work through the remaining entries per their w1 disposition: README ->\nindex, schema-doc references, platform cross-links (synapse/pyspark/\nglue/platform-development), api-reference.md's getting-started/examples/\nconfiguration links, the advanced/performance/usage/api targets, and the\ntwo `.csv` data-file links (confirm whether the CSVs were removed and\nremove or repoint accordingly). Fix local links only; leave external\n(http/https) links to the Sphinx linkcheck. Regenerate the baseline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Resolve the remaining single-target broken links", "wid": "w4"}, {"evidence": null, "item_id": "burn-down-baselined-broken-doc-links", "needs": ["w2", "w3", "w4"], "notes": "Final `--update-baseline` should leave zero data entries. Run\n`uv run -- python scripts/check_doc_relative_links.py` (reports\n`broken total : 0`, exit 0) and `make docs-validate` (exit 0). The goal\nis an empty data set. Note `--update-baseline` rewrites the file as the\nfixed header comments plus sorted entries \u2014 it does NOT preserve\nper-entry inline comments, so a genuinely-intentional exception must be\nfixed/removed rather than annotated in place (or hand-added below the\nheader and never regenerated over). Standard cycle:\n`make lint format typecheck`, `make pr-preflight`, `make pr-open`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Drive the baseline to zero and confirm the gate is green", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:33:56Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #118 of close-pre-beta-release-blockers.\nDeferred: Burn down the full repository-wide `ty` backlog outside release-critical entrypoints\nReason deferred: Useful follow-up work, but larger than a Beta-release blocker once the critical gate is ratcheted.", "id": "burn-down-the-full-repository-wide-ty-backlog-outside-releas", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Burn down the full repository-wide `ty` backlog outside release-critical entrypoints", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "only correctness-gate-digests-regen writes it.", "dont": "hand-edit tpch_value_digests_sf1.json", "why": "the file is generated with provenance from a live gate run"}, {"INSTEAD": "same change, per the Makefile's documented contract.", "dont": "ship the renderer change and the reference regen in separate PRs", "why": "the window between them is a red gate on a correct tree"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-05T00:00:00Z", "completed_pr": null, "created_at": "2026-07-04T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "calculate-checksum-collision-hardening", "id": 308, "reason": "Separately pinned accepted property with its own DuckDB-pinned-oracle rationale; only touch it here if w3 shows the renderers are actually shared.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: In scope only 'if w3 shows renderers are shared' -- unresolved speculative condition; digest rendering already separately pinned.", "summary": "Reworking result_digest.py's int-vs-float/Decimal rendering asymmetry"}], "deps": [], "description": "SOUNDNESS-CRITICAL COMPARATOR SURFACE: benchbox/core/tpchavoc/validation.py\nis CODEOWNERS-covered; this TODO's PR requires code-owner review.\n\nvalue-digest-collision-pinning (DONE, PR #952) took the deliberate\npin-only half of its \"pin (or fix)\" mandate: three verified collisions in\n`calculate_checksum`'s raw str-join digest are now pinned as accepted\nproperties in tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py:144-193:\n 1. cell-separator aliasing: [(\"a|b\",\"c\")] == [(\"a\",\"b|c\")]\n 2. row-separator aliasing: [(\"a\\nb\",)] == [(\"a\",),(\"b\",)]\n 3. dtype aliasing: [(\"1\",)] == [(1,)]\nplus the pre-existing NULL-vs-\"NULL\" pin (same file, :133-141). The pins\nmake the weakness conscious; they do not remove it. The digest backs BOTH\nTPC-Havoc's validate_results_checksum comparator AND the bounded\ncorrectness gate's value-digest oracle (one shared definition, per\nvalidation.py's module docstring), so a colliding regression on either\npath is invisible today. The DONE notes' stated reason for not fixing was\nscope, not design: any rendering change invalidates the committed\nreference benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json,\nand regeneration machinery discipline was the riskier part at the time.\nThat machinery now exists and is contract-tested\n(`make correctness-gate-digests-regen`, Makefile:147-160, idempotent on a\nclean tree), which is what makes the real fix cheap now.\n\nFix shape: render each cell with a type tag and escape the separator\ncharacters (e.g. escape `\\`, `|`, `\\n` in string payloads; prefix a\none-byte type marker for str/int/float/Decimal/date/NULL), keeping\n_row_sort_key's None-safe ordering untouched. Every pinned collision then\nbecomes a distinctness assertion.", "id": "calculate-checksum-collision-hardening", "preserves": ["_row_sort_key's None-safe row ordering semantics are unchanged (ordering is not part of this fix).", "make test-correctness-gate passes on a correct tree after the same-change regen (Makefile:150-160 idempotency contract: regen && git diff --exit-code is clean).", "validate_results_checksum and the correctness gate's value-digest oracle keep sharing exactly one digest definition \u2014 no forked renderer."], "prior_art": [{"concept": "Makefile:correctness-gate-digests-regen (147-160) \u2014 reuse: the exact same-change regen mechanism this fix's reference-invalidation depends on; its idempotency contract is the post-fix verification.", "decision": "reuse", "path": "Makefile:correctness-gate-digests-regen"}, {"concept": "benchbox/core/results/result_digest.py:_normalize_cell \u2014 the gate-side cell renderer with its own documented int-vs-float rendering asymmetry (pinned in tests/unit/test_correctness_gate_value_oracle.py::test_digest_couples_value_with_numeric_type); align the two renderers' type-tagging decisions or document divergence explicitly.", "decision": "reuse", "path": "benchbox/core/results/result_digest.py:_normalize_cell"}, {"concept": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py:133-193 \u2014 supersede: the four collision pins flip from equality-asserts to inequality-asserts with their docstrings rewritten from 'accepted property' to 'fixed, distinctness guaranteed'.", "decision": "supersede", "path": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py:133-193"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/results/result_digest.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/validation.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py"}], "state": "done", "title": "Fix (not just pin) calculate_checksum's separator/dtype collisions with an escaped, type-tagged rendering plus same-change digest regen", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py -q", "description": "Former collisions now distinct", "expected": "all pass with the flipped distinctness assertions", "item_id": "calculate-checksum-collision-hardening", "last_result": null, "last_run": null, "seq": 1}, {"command": "make correctness-gate-digests-regen && git diff --exit-code benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json && make test-correctness-gate", "description": "Gate green with regenerated reference, regen idempotent", "expected": "second regen produces no diff; gate exits 0 with ran=1 skipped=0", "item_id": "calculate-checksum-collision-hardening", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "calculate-checksum-collision-hardening", "needs": [], "notes": "Keep the digest definition in exactly one place. Decide the NULL\nhandling consciously: the NULL-vs-\"NULL\" pin can be fixed by the same\ntype tag (NULL marker vs str tag) \u2014 fixing it is preferred; if kept,\nre-justify in the pin's docstring why it survives a fix that makes\nthe other three distinct.\nDONE 2026-07-05: _render_cell + _escape_cell_text in validation.py.\nTags: z (NULL), s (str), i (int; bool folds in, matching\n_cell_sort_key's numeric bucket), f (float), d (Decimal), t\n(date/time/datetime), o: fallback. Escapes backslash, |,\nnewline in string payloads. NULL-vs-\"NULL\" FIXED by the same tags\n(z: vs s:NULL) - preferred option taken.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement escaped, type-tagged cell rendering in calculate_checksum", "wid": "w1"}, {"evidence": null, "item_id": "calculate-checksum-collision-hardening", "needs": ["w1"], "notes": "Each former collision asserts digests now differ; add cases where the\nescape character itself appears in payloads (backslash-containing\nstrings) so the escaping is exercised, not just the happy path.\nDONE 2026-07-05: all four pins flipped to distinctness asserts with\nrewritten docstrings; added escape round-trip cases (literal\nbackslash-n vs newline, trailing backslash vs escaped separator,\ndoubled backslash, escape-before-separator across shapes) and\nfloat-vs-Decimal / str-vs-date dtype distinctness. 41 tests pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Flip the collision pins to distinctness tests; extend with escape-char round-trip cases", "wid": "w2"}, {"evidence": null, "item_id": "calculate-checksum-collision-hardening", "needs": ["w1"], "notes": "`make correctness-gate-digests-regen` then\n`git diff --exit-code benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json`\nmust be dirty exactly once (the regen) and clean on a second regen.\nThen `make test-correctness-gate` green. Check whether\nresult_digest.py's oracle path shares the changed renderer \u2014 if the\ntwo renderers are independent, say so in the PR body and leave\nresult_digest's asymmetry to its own pin (deferred below).\nDONE 2026-07-05: regen produced exactly one dirty diff (18/18 digest\nvalues changed, provenance stable, DuckDB 1.3.2); second and third\nregens byte-identical (git diff --exit-code clean vs staged copy).\nmake test-correctness-gate green (ran=1 skipped=0 failures=0\nerrors=0). make tpchavoc-equivalence-report green (220/220\nequivalent). Renderer-sharing answer: result_digest.py REUSES\ncalculate_checksum after numeric normalization (composition, not a\nfork), so the regen covers the gate path; result_digest's own\nint-vs-float pre-normalization asymmetry remains its separately\npinned accepted property (deferred, per this TODO's deferred list).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate the committed reference digests in the SAME change and prove gate + idempotency", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "expand the `special_mappings` dict", "why": "that keeps the heuristic alive and recreates registry drift."}, {"INSTEAD": "(not specified)", "dont": "infer canonical IDs by lowercasing class names", "why": "underscores and acronym boundaries are part of the registry contract."}, {"INSTEAD": "(not specified)", "dont": "launch long-running benchmark runs to verify identity", "why": "targeted instantiation, dry-run, and contract tests are sufficient."}, {"INSTEAD": "(not specified)", "dont": "treat source-data output directories as identity", "why": "primitives may reuse TPC-H data while still being separate benchmarks."}], "approach": "Treat runtime benchmark identity as a registry contract, not a string\nformatting convenience. First pin current mismatches with tests, then replace\nthe heuristic and adjust validation to fail early for unknown IDs. Keep\noutput path tests nuanced: canonical identity must match the registry, while\ndata-source output locations may deliberately refer to the source benchmark.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-22T00:00:00Z", "deferrals": [], "deps": [], "description": "Independent verification at checked SHA e60986923 confirmed the critic's\nhighest-risk finding directionally, with one important correction: the public\n`benchbox.base.BaseBenchmark._get_benchmark_name()` heuristic mismatches 8\nregistry IDs, not 9. `tpcds_obt` does not use `_get_benchmark_name()` because\nit still inherits the deprecated core base; it has a related name heuristic\nrisk through `benchbox.core.base_benchmark.BaseBenchmark.name`, but its\nconstructor explicitly writes to `benchmark_runs/datagen/tpcds_obt_sf1`.\n\nConfirmed mismatches for public-base core classes:\n read_primitives -> readprimitives\n write_primitives -> writeprimitives\n metadata_primitives -> metadataprimitives\n ai_primitives -> aiprimitives\n transaction_primitives -> transactionprimitives\n joinorder_synthetic -> joinordersynthetic\n tpch_skew -> tpchskew\n vector_search -> vectorsearch\n\nRuntime leakage was confirmed through logger names for all 8 mismatches, and\nthrough default output paths for direct callers of classes such as\n`TPCHSkewBenchmark` and `VectorSearchBenchmark`. Some data-sharing benchmarks\nintentionally override their output directory to the canonical TPC-H datagen\npath after base initialization, so output-path impact is not uniform. The\nrunner path is likely shielded when it passes explicit `benchmark_name` and\noutput directories, but this still needs end-to-end confirmation before code\nchanges.\n\nIndependent verification also confirmed `validate_scale_factor(\"unknown_bench\",\n999)` returns silently in `benchbox/core/benchmark_registry.py` before the\nloader eventually fails elsewhere. This is actionably related: benchmark ID\nvalidity and canonical identity should fail early and consistently.", "id": "canonical-benchmark-id-resolution", "preserves": ["Data-sharing benchmarks that intentionally reuse TPC-H datagen paths keep that output behavior unless a separate product decision changes it.", "No live/cloud benchmark execution is required for this fix.", "Scale-factor validation for known benchmarks keeps existing error types for unsupported scales.", "The registry remains the single source of truth for benchmark IDs; do not add an untested parallel dict."], "prior_art": [{"concept": "benchbox/core/benchmark_registry.py: BENCHMARK_CLASS_NAMES and CORE_BENCHMARK_CLASS_NAMES - reuse as the class-to-ID source of truth.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "benchbox/utils/path_utils.py: get_benchmark_runs_datagen_path - preserve intentional data-source output directory conventions.", "decision": "reuse", "path": "benchbox/utils/path_utils.py"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: public contract invariants - reuse for unknown-ID and canonical-ID contract coverage.", "decision": "reuse", "path": "tests/unit/core/test_benchmark_api_contract.py"}, {"concept": "tests/unit/core/test_loader_registry_parity.py: registry/loader parity checks - extend to runtime identity parity.", "decision": "extend", "path": "tests/unit/core/test_loader_registry_parity.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/datavault/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "tests/test_base.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_loader_registry_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_scale_factor_validation.py"}], "state": "done", "title": "Resolve benchmark runtime identity from the registry", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_loader_registry_parity.py tests/unit/core/test_benchmark_api_contract.py tests/test_base.py -q", "description": "Runtime identity parity is pinned for all loader benchmarks.", "expected": "All selected parity and base tests pass.", "item_id": "canonical-benchmark-id-resolution", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py -q", "description": "Unknown benchmark scale validation fails early and clearly.", "expected": "Unknown benchmark ID and scale-factor contract tests pass.", "item_id": "canonical-benchmark-id-resolution", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite remains green after registry/base changes.", "expected": "Fast tests pass.", "item_id": "canonical-benchmark-id-resolution", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": [], "notes": "Re-run a read-only harness that imports registry IDs, resolves every\nloader class, instantiates with `get_benchmark_default_scale()`, and\nrecords `(registry_id, class, runtime_id, output_dir, logger_name)` for\nany mismatch. Commit only a compact markdown or TODO note if durable\nevidence is needed; keep raw stdout in `/tmp`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run and preserve a compact identity verification summary on the current branch", "wid": "w0"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w0"], "notes": "Prefer a registry-derived reverse map from\n`CORE_BENCHMARK_CLASS_NAMES`/`BENCHMARK_CLASS_NAMES` before adding a new\nclass attribute. If a class attribute is chosen, add a registry parity\ntest that fails when it diverges. Account for wrappers, core classes,\nsubclasses such as `TPCHSkewBenchmark`, and data-sharing benchmarks whose\noutput directory intentionally points at the source benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the canonical benchmark-id source for runtime instances", "wid": "w1"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w1"], "notes": "Change `benchbox/base.py` so runtime identity, logger names, validation\nfallbacks, and default direct-call output paths use the canonical registry\nID. Do not hard-code another special-case mapping inside the base class.\nPreserve intentional data-source output overrides for primitives that\nreuse TPC-H data.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace `_get_benchmark_name()` heuristic with registry-backed canonical resolution", "wid": "w2"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w2"], "notes": "Extend the existing parity/contract tests so every loader benchmark that\nexposes runtime identity is checked against its registry ID. Include\nexplicit assertions for `tpch_skew`, `vector_search`,\n`joinorder_synthetic`, and the primitives. Separate assertions for\nintentional data-source output directories from assertions for canonical\nbenchmark identity.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add all-benchmark parity tests for runtime identity, logger names, and direct default paths", "wid": "w3"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w1"], "notes": "Update `validate_scale_factor()` or its caller contract so an unknown\nbenchmark ID raises a clear `ValueError`/`KeyError` before class loading.\nAdd tests for unknown ID, invalid scale for a known ID, and valid default\nscale for all registered benchmarks. Preserve existing\n`ScaleFactorNotSupportedError` behavior for unsupported known scales.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make scale validation reject unknown benchmark IDs at the earliest registry boundary", "wid": "w4"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w2", "w3", "w4"], "notes": "Read and, where practical, exercise the lifecycle runner path that passes\nexplicit `benchmark_name` and output directories. Confirm `benchmark_id`\nin validation metadata, manifests, and run artifacts remains canonical.\nIf a lightweight dry-run is sufficient, use it; do not launch live/cloud\nworkloads.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm orchestrated runner paths are not regressed", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "find the real successor page or leave it with a note.", "dont": "blindly follow a redirect that lands on a generic index page", "why": "you would replace a specific (if stale) link with a less-useful one"}, {"INSTEAD": "leave DOIs alone.", "dont": "repoint a DOI to its current dataverse landing page", "why": "the DOI is the durable citation identifier and the redirect is intentional"}, {"INSTEAD": "repoint the source URL.", "dont": "silence redirects with linkcheck_allowed_redirects as the primary fix", "why": "it hides stale links instead of fixing them and readers still hit the redirect"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-08T00:00:00Z", "completed_pr": null, "created_at": "2026-07-08T00:00:00Z", "deferrals": [], "deps": [], "description": "PR #1051 (`fix/docs-linkcheck-resilience`) hardened the Sphinx\n`linkcheck` job against transient timeouts (60s timeout, retries,\nreport-timeouts-as-broken). A clean run of `make docs-linkcheck` on\nthat branch reports **0 broken / 0 timeouts** but **~49 permanent\nredirects** \u2014 external URLs that resolve fine (so CI stays green) but\npoint at stale locations that the upstream site 301/302-redirects to a\ncanonical URL.\n\nRedirects are informational and do NOT fail the build, so this is\npure hygiene: repoint the source links at their canonical targets so\nreaders (and future linkcheck runs) hit the real page directly. It is\nexplicitly NOT about the internal repo-relative links \u2014 those are the\nseparate `check_doc_relative_links.py` baseline burn-down tracked in\n`burn-down-baselined-broken-doc-links.yaml`. This TODO is external\nhttp/https only, driven off the Sphinx linkcheck redirect report.\n\nNOT every redirect should be chased \u2014 some are canonical/stable by\ndesign and repointing them is wrong or churny. Triage first (w1), then\nfix only the clear wins.\n\nRedirect clusters observed 2026-07-08 (regenerate to confirm before\nstarting \u2014 counts/targets drift as upstreams reorganize):\n REPOINT (clear canonical wins):\n - docs.snowflake.com/...html -> same path without `.html` (~6)\n - cloud.google.com/bigquery|storage -> docs.cloud.google.com/... (~10)\n - clickhouse.com/docs/en/... -> clickhouse.com/docs/... (drop /en) (~4)\n - arrow.apache.org/datafusion* -> datafusion.apache.org/* (2)\n - docs.databricks.com/...html -> docs.databricks.com/aws/en/... (~3)\n - docs.microsoft.com/... -> learn.microsoft.com/... (1)\n - docs.influxdata.com/influxdb/latest -> .../v2 (1)\n - docs.timescale.com -> www.tigerdata.com/docs (1)\n - velodb.io -> www.velodb.io ; singlestore.com/cloud -> /product-overview\n - clickhouse.cloud -> console.clickhouse.cloud\n - github.com/mwc360/LakeBench -> github.com/microsoft/LakeBench (1)\n LEAVE ALONE (do not repoint \u2014 record the decision in the PR):\n - doi.org/10.7910/... -> dataverse.harvard.edu (DOI is the stable\n citation identifier; the redirect is the point of a DOI).\n - github.com/.../issues/new -> github.com/login?return_to=... (auth\n gate for anonymous checkers; the source link is correct).\n - *.readthedocs.io/ -> /en/stable/ (version canonicalization; low\n value and prone to re-churn on the next docs release).\n - http://www.cs.umb.edu/... and other `:443`/scheme-normalization\n redirects that only add an explicit port/scheme.\n\nAlternative considered and rejected as the primary fix: silence stable\nredirects via `linkcheck_allowed_redirects` in `docs/conf.py`. That\nhides the signal rather than fixing the stale link, and readers still\nland on a redirect. Prefer fixing the source; reserve config for the\nLEAVE-ALONE set only if a future stricter policy is adopted.", "id": "canonicalize-redirected-external-doc-links", "preserves": ["DOI, auth-gated, readthedocs-version, and scheme/port-only redirects are left as-is \u2014 repointing them is incorrect.", "Internal repo-relative link handling is untouched \u2014 that is the separate check_doc_relative_links.py baseline burn-down.", "Sphinx linkcheck stays green (0 broken, 0 timeouts) after edits."], "prior_art": [{"concept": "burn-down-baselined-broken-doc-links.yaml \u2014 sibling docs-link TODO; REUSE its triage-table-in-PR pattern, but it covers INTERNAL repo-relative links via check_doc_relative_links.py; this TODO is genuinely new for EXTERNAL redirect canonicalization.", "decision": "reuse", "path": "burn-down-baselined-broken-doc-links.yaml"}, {"concept": "docs/conf.py:linkcheck_* (added in PR #1051) \u2014 the linkcheck config that surfaces the redirect report; REUSE `make docs-linkcheck` output as the input, do not modify the config here.", "decision": "reuse", "path": "docs/conf.py:linkcheck_*"}, {"concept": "docs/linkcheck_ignore.txt \u2014 existing external-link escape hatch for flaky/gated hosts; reference for where LEAVE-ALONE hosts would go IF a stricter policy is later adopted (not in scope now).", "decision": "reuse", "path": "docs/linkcheck_ignore.txt"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "docs/conf.py"}, {"kind": "do_not_modify", "path_glob": "docs/linkcheck_ignore.txt"}, {"kind": "do_not_modify", "path_glob": "scripts/doc_relative_link_baseline.txt"}, {"kind": "only_modify", "path_glob": "docs/"}], "state": "done", "title": "Canonicalize the ~49 redirected external doc links surfaced by linkcheck", "verifications": [{"command": "make docs-linkcheck", "description": "REPOINTed links now resolve directly (no redirect); build clean", "expected": "exit 0; 0 broken, 0 timeouts; redirect count reduced to the LEAVE-ALONE residual", "item_id": "canonicalize-redirected-external-doc-links", "last_result": null, "last_run": null, "seq": 1}, {"command": "! rg -n 'docs\\.snowflake\\.com/[^ )]+\\.html' docs/", "description": "Spot-check a representative canonicalization is gone from source", "expected": "no matches (Snowflake .html links repointed)", "item_id": "canonicalize-redirected-external-doc-links", "last_result": null, "last_run": null, "seq": 2}, {"command": "git diff --name-only develop -- scripts/doc_relative_link_baseline.txt", "description": "No repo-relative links were collaterally touched (that is the other TODO)", "expected": "empty (baseline untouched by this work)", "item_id": "canonicalize-redirected-external-doc-links", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "canonicalize-redirected-external-doc-links", "needs": [], "notes": "Bind the cited evidence before editing. On the implementation\nbranch run `make docs-linkcheck`, then extract the redirect report:\n grep -E '\\[redirected' docs/_build/linkcheck/output.txt\n(or parse `output.json` for `\"status\": \"redirected\"`). Record in\nthe PR description: the develop SHA, the total redirect count, and\nany drift from the ~49 captured 2026-07-08 (the clusters in this\nTODO's description are that snapshot). If targets shifted, re-derive\nthe w2 groupings before proceeding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate the redirect list against current develop", "wid": "w0"}, {"evidence": null, "item_id": "canonicalize-redirected-external-doc-links", "needs": ["w0"], "notes": "For each (source_file:line, from_url -> to_url) assign a disposition\nper the description's rules: REPOINT stale-path canonical rewrites;\nLEAVE-ALONE DOIs, auth redirects, readthedocs version canonicalization,\nand pure scheme/port (`:443`) normalizations. Record the table in the\nPR description as the audit trail for w2. When unsure, verify the\ntarget actually serves the intended content (open it) before\nrepointing \u2014 a redirect can land on a generic index (e.g. one AWS\n`whats-new` link collapses to `aws.amazon.com/about-aws/`), in which\ncase find the real successor page or LEAVE-ALONE with a note.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Triage every redirect into REPOINT vs LEAVE-ALONE", "wid": "w1"}, {"evidence": null, "item_id": "canonicalize-redirected-external-doc-links", "needs": ["w1"], "notes": "Edit the source `.md`/`.rst` files to the canonical target URLs from\nthe w1 table. Fix external http/https links only; leave repo-relative\nlinks to the other checker. Re-run `make docs-linkcheck` and confirm\nthe REPOINTed URLs now report `ok` (working) rather than `redirected`,\nthe redirect count drops to the LEAVE-ALONE residual, and broken/\ntimeout stay at 0. Standard close-out: `make lint format typecheck`,\n`code review` (fix findings), `make pr-open`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the REPOINT edits and re-run linkcheck", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "hand-edit tpch_value_digests_sf1.json", "instead": "only correctness-gate-digests-regen writes it.", "why": "the file is generated with provenance from a live gate run"}, {"dont": "ship the renderer change and the reference regen in separate PRs", "instead": "same change, per the Makefile's documented contract.", "why": "the window between them is a red gate on a correct tree"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-05T00:00:00Z", "completed_pr": null, "created_at": "2026-07-04T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "calculate-checksum-collision-hardening", "id": 308, "reason": "Separately pinned accepted property with its own DuckDB-pinned-oracle rationale; only touch it here if w3 shows the renderers are actually shared.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: In scope only 'if w3 shows renderers are shared' -- unresolved speculative condition; digest rendering already separately pinned.", "summary": "Reworking result_digest.py's int-vs-float/Decimal rendering asymmetry"}], "deps": [], "description": "SOUNDNESS-CRITICAL COMPARATOR SURFACE: benchbox/core/tpchavoc/validation.py\nis CODEOWNERS-covered; this TODO's PR requires code-owner review.\n\nvalue-digest-collision-pinning (DONE, PR #952) took the deliberate\npin-only half of its \"pin (or fix)\" mandate: three verified collisions in\n`calculate_checksum`'s raw str-join digest are now pinned as accepted\nproperties in tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py:144-193:\n 1. cell-separator aliasing: [(\"a|b\",\"c\")] == [(\"a\",\"b|c\")]\n 2. row-separator aliasing: [(\"a\\nb\",)] == [(\"a\",),(\"b\",)]\n 3. dtype aliasing: [(\"1\",)] == [(1,)]\nplus the pre-existing NULL-vs-\"NULL\" pin (same file, :133-141). The pins\nmake the weakness conscious; they do not remove it. The digest backs BOTH\nTPC-Havoc's validate_results_checksum comparator AND the bounded\ncorrectness gate's value-digest oracle (one shared definition, per\nvalidation.py's module docstring), so a colliding regression on either\npath is invisible today. The DONE notes' stated reason for not fixing was\nscope, not design: any rendering change invalidates the committed\nreference benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json,\nand regeneration machinery discipline was the riskier part at the time.\nThat machinery now exists and is contract-tested\n(`make correctness-gate-digests-regen`, Makefile:147-160, idempotent on a\nclean tree), which is what makes the real fix cheap now.\n\nFix shape: render each cell with a type tag and escape the separator\ncharacters (e.g. escape `\\`, `|`, `\\n` in string payloads; prefix a\none-byte type marker for str/int/float/Decimal/date/NULL), keeping\n_row_sort_key's None-safe ordering untouched. Every pinned collision then\nbecomes a distinctness assertion.", "id": "calculate-checksum-collision-hardening", "preserves": ["_row_sort_key's None-safe row ordering semantics are unchanged (ordering is not part of this fix).", "make test-correctness-gate passes on a correct tree after the same-change regen (Makefile:150-160 idempotency contract: regen && git diff --exit-code is clean).", "validate_results_checksum and the correctness gate's value-digest oracle keep sharing exactly one digest definition \u2014 no forked renderer."], "prior_art": [{"concept": "Makefile:correctness-gate-digests-regen (147-160) \u2014 reuse: the exact same-change regen mechanism this fix's reference-invalidation depends on; its idempotency contract is the post-fix verification.", "decision": "reuse", "path": "Makefile:correctness-gate-digests-regen"}, {"concept": "benchbox/core/results/result_digest.py:_normalize_cell \u2014 the gate-side cell renderer with its own documented int-vs-float rendering asymmetry (pinned in tests/unit/test_correctness_gate_value_oracle.py::test_digest_couples_value_with_numeric_type); align the two renderers' type-tagging decisions or document divergence explicitly.", "decision": "reuse", "path": "benchbox/core/results/result_digest.py:_normalize_cell"}, {"concept": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py:133-193 \u2014 supersede: the four collision pins flip from equality-asserts to inequality-asserts with their docstrings rewritten from 'accepted property' to 'fixed, distinctness guaranteed'.", "decision": "supersede", "path": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py:133-193"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/results/result_digest.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/validation.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py"}], "state": "done", "title": "Fix (not just pin) calculate_checksum's separator/dtype collisions with an escaped, type-tagged rendering plus same-change digest regen", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py -q", "description": "Former collisions now distinct", "expected": "all pass with the flipped distinctness assertions", "item_id": "calculate-checksum-collision-hardening", "last_result": null, "last_run": null, "seq": 1}, {"command": "make correctness-gate-digests-regen && git diff --exit-code benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json && make test-correctness-gate", "description": "Gate green with regenerated reference, regen idempotent", "expected": "second regen produces no diff; gate exits 0 with ran=1 skipped=0", "item_id": "calculate-checksum-collision-hardening", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "calculate-checksum-collision-hardening", "needs": [], "notes": "Keep the digest definition in exactly one place. Decide the NULL\nhandling consciously: the NULL-vs-\"NULL\" pin can be fixed by the same\ntype tag (NULL marker vs str tag) \u2014 fixing it is preferred; if kept,\nre-justify in the pin's docstring why it survives a fix that makes\nthe other three distinct.\nDONE 2026-07-05: _render_cell + _escape_cell_text in validation.py.\nTags: z (NULL), s (str), i (int; bool folds in, matching\n_cell_sort_key's numeric bucket), f (float), d (Decimal), t\n(date/time/datetime), o: fallback. Escapes backslash, |,\nnewline in string payloads. NULL-vs-\"NULL\" FIXED by the same tags\n(z: vs s:NULL) - preferred option taken.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement escaped, type-tagged cell rendering in calculate_checksum", "wid": "w1"}, {"evidence": null, "item_id": "calculate-checksum-collision-hardening", "needs": ["w1"], "notes": "Each former collision asserts digests now differ; add cases where the\nescape character itself appears in payloads (backslash-containing\nstrings) so the escaping is exercised, not just the happy path.\nDONE 2026-07-05: all four pins flipped to distinctness asserts with\nrewritten docstrings; added escape round-trip cases (literal\nbackslash-n vs newline, trailing backslash vs escaped separator,\ndoubled backslash, escape-before-separator across shapes) and\nfloat-vs-Decimal / str-vs-date dtype distinctness. 41 tests pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Flip the collision pins to distinctness tests; extend with escape-char round-trip cases", "wid": "w2"}, {"evidence": null, "item_id": "calculate-checksum-collision-hardening", "needs": ["w1"], "notes": "`make correctness-gate-digests-regen` then\n`git diff --exit-code benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json`\nmust be dirty exactly once (the regen) and clean on a second regen.\nThen `make test-correctness-gate` green. Check whether\nresult_digest.py's oracle path shares the changed renderer \u2014 if the\ntwo renderers are independent, say so in the PR body and leave\nresult_digest's asymmetry to its own pin (deferred below).\nDONE 2026-07-05: regen produced exactly one dirty diff (18/18 digest\nvalues changed, provenance stable, DuckDB 1.3.2); second and third\nregens byte-identical (git diff --exit-code clean vs staged copy).\nmake test-correctness-gate green (ran=1 skipped=0 failures=0\nerrors=0). make tpchavoc-equivalence-report green (220/220\nequivalent). Renderer-sharing answer: result_digest.py REUSES\ncalculate_checksum after numeric normalization (composition, not a\nfork), so the regen covers the gate path; result_digest's own\nint-vs-float pre-normalization asymmetry remains its separately\npinned accepted property (deferred, per this TODO's deferred list).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate the committed reference digests in the SAME change and prove gate + idempotency", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "expand the `special_mappings` dict", "instead": "(not specified)", "why": "that keeps the heuristic alive and recreates registry drift."}, {"dont": "infer canonical IDs by lowercasing class names", "instead": "(not specified)", "why": "underscores and acronym boundaries are part of the registry contract."}, {"dont": "launch long-running benchmark runs to verify identity", "instead": "(not specified)", "why": "targeted instantiation, dry-run, and contract tests are sufficient."}, {"dont": "treat source-data output directories as identity", "instead": "(not specified)", "why": "primitives may reuse TPC-H data while still being separate benchmarks."}], "approach": "Treat runtime benchmark identity as a registry contract, not a string\nformatting convenience. First pin current mismatches with tests, then replace\nthe heuristic and adjust validation to fail early for unknown IDs. Keep\noutput path tests nuanced: canonical identity must match the registry, while\ndata-source output locations may deliberately refer to the source benchmark.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-22T00:00:00Z", "deferrals": [], "deps": [], "description": "Independent verification at checked SHA e60986923 confirmed the critic's\nhighest-risk finding directionally, with one important correction: the public\n`benchbox.base.BaseBenchmark._get_benchmark_name()` heuristic mismatches 8\nregistry IDs, not 9. `tpcds_obt` does not use `_get_benchmark_name()` because\nit still inherits the deprecated core base; it has a related name heuristic\nrisk through `benchbox.core.base_benchmark.BaseBenchmark.name`, but its\nconstructor explicitly writes to `benchmark_runs/datagen/tpcds_obt_sf1`.\n\nConfirmed mismatches for public-base core classes:\n read_primitives -> readprimitives\n write_primitives -> writeprimitives\n metadata_primitives -> metadataprimitives\n ai_primitives -> aiprimitives\n transaction_primitives -> transactionprimitives\n joinorder_synthetic -> joinordersynthetic\n tpch_skew -> tpchskew\n vector_search -> vectorsearch\n\nRuntime leakage was confirmed through logger names for all 8 mismatches, and\nthrough default output paths for direct callers of classes such as\n`TPCHSkewBenchmark` and `VectorSearchBenchmark`. Some data-sharing benchmarks\nintentionally override their output directory to the canonical TPC-H datagen\npath after base initialization, so output-path impact is not uniform. The\nrunner path is likely shielded when it passes explicit `benchmark_name` and\noutput directories, but this still needs end-to-end confirmation before code\nchanges.\n\nIndependent verification also confirmed `validate_scale_factor(\"unknown_bench\",\n999)` returns silently in `benchbox/core/benchmark_registry.py` before the\nloader eventually fails elsewhere. This is actionably related: benchmark ID\nvalidity and canonical identity should fail early and consistently.", "id": "canonical-benchmark-id-resolution", "preserves": ["Data-sharing benchmarks that intentionally reuse TPC-H datagen paths keep that output behavior unless a separate product decision changes it.", "No live/cloud benchmark execution is required for this fix.", "Scale-factor validation for known benchmarks keeps existing error types for unsupported scales.", "The registry remains the single source of truth for benchmark IDs; do not add an untested parallel dict."], "prior_art": [{"concept": "benchbox/core/benchmark_registry.py: BENCHMARK_CLASS_NAMES and CORE_BENCHMARK_CLASS_NAMES - reuse as the class-to-ID source of truth.", "decision": "reuse", "path": "benchbox/core/benchmark_registry.py"}, {"concept": "benchbox/utils/path_utils.py: get_benchmark_runs_datagen_path - preserve intentional data-source output directory conventions.", "decision": "reuse", "path": "benchbox/utils/path_utils.py"}, {"concept": "tests/unit/core/test_benchmark_api_contract.py: public contract invariants - reuse for unknown-ID and canonical-ID contract coverage.", "decision": "reuse", "path": "tests/unit/core/test_benchmark_api_contract.py"}, {"concept": "tests/unit/core/test_loader_registry_parity.py: registry/loader parity checks - extend to runtime identity parity.", "decision": "extend", "path": "tests/unit/core/test_loader_registry_parity.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/datavault/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "tests/test_base.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_benchmark_api_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_loader_registry_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_scale_factor_validation.py"}], "state": "done", "title": "Resolve benchmark runtime identity from the registry", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_loader_registry_parity.py tests/unit/core/test_benchmark_api_contract.py tests/test_base.py -q", "description": "Runtime identity parity is pinned for all loader benchmarks.", "expected": "All selected parity and base tests pass.", "item_id": "canonical-benchmark-id-resolution", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_benchmark_api_contract.py -q", "description": "Unknown benchmark scale validation fails early and clearly.", "expected": "Unknown benchmark ID and scale-factor contract tests pass.", "item_id": "canonical-benchmark-id-resolution", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite remains green after registry/base changes.", "expected": "Fast tests pass.", "item_id": "canonical-benchmark-id-resolution", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": [], "notes": "Re-run a read-only harness that imports registry IDs, resolves every\nloader class, instantiates with `get_benchmark_default_scale()`, and\nrecords `(registry_id, class, runtime_id, output_dir, logger_name)` for\nany mismatch. Commit only a compact markdown or TODO note if durable\nevidence is needed; keep raw stdout in `/tmp`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run and preserve a compact identity verification summary on the current branch", "wid": "w0"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w0"], "notes": "Prefer a registry-derived reverse map from\n`CORE_BENCHMARK_CLASS_NAMES`/`BENCHMARK_CLASS_NAMES` before adding a new\nclass attribute. If a class attribute is chosen, add a registry parity\ntest that fails when it diverges. Account for wrappers, core classes,\nsubclasses such as `TPCHSkewBenchmark`, and data-sharing benchmarks whose\noutput directory intentionally points at the source benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the canonical benchmark-id source for runtime instances", "wid": "w1"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w1"], "notes": "Change `benchbox/base.py` so runtime identity, logger names, validation\nfallbacks, and default direct-call output paths use the canonical registry\nID. Do not hard-code another special-case mapping inside the base class.\nPreserve intentional data-source output overrides for primitives that\nreuse TPC-H data.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace `_get_benchmark_name()` heuristic with registry-backed canonical resolution", "wid": "w2"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w2"], "notes": "Extend the existing parity/contract tests so every loader benchmark that\nexposes runtime identity is checked against its registry ID. Include\nexplicit assertions for `tpch_skew`, `vector_search`,\n`joinorder_synthetic`, and the primitives. Separate assertions for\nintentional data-source output directories from assertions for canonical\nbenchmark identity.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add all-benchmark parity tests for runtime identity, logger names, and direct default paths", "wid": "w3"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w1"], "notes": "Update `validate_scale_factor()` or its caller contract so an unknown\nbenchmark ID raises a clear `ValueError`/`KeyError` before class loading.\nAdd tests for unknown ID, invalid scale for a known ID, and valid default\nscale for all registered benchmarks. Preserve existing\n`ScaleFactorNotSupportedError` behavior for unsupported known scales.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make scale validation reject unknown benchmark IDs at the earliest registry boundary", "wid": "w4"}, {"evidence": null, "item_id": "canonical-benchmark-id-resolution", "needs": ["w2", "w3", "w4"], "notes": "Read and, where practical, exercise the lifecycle runner path that passes\nexplicit `benchmark_name` and output directories. Confirm `benchmark_id`\nin validation metadata, manifests, and run artifacts remains canonical.\nIf a lightweight dry-run is sufficient, use it; do not launch live/cloud\nworkloads.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm orchestrated runner paths are not regressed", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "blindly follow a redirect that lands on a generic index page", "instead": "find the real successor page or leave it with a note.", "why": "you would replace a specific (if stale) link with a less-useful one"}, {"dont": "repoint a DOI to its current dataverse landing page", "instead": "leave DOIs alone.", "why": "the DOI is the durable citation identifier and the redirect is intentional"}, {"dont": "silence redirects with linkcheck_allowed_redirects as the primary fix", "instead": "repoint the source URL.", "why": "it hides stale links instead of fixing them and readers still hit the redirect"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-08T00:00:00Z", "completed_pr": null, "created_at": "2026-07-08T00:00:00Z", "deferrals": [], "deps": [], "description": "PR #1051 (`fix/docs-linkcheck-resilience`) hardened the Sphinx\n`linkcheck` job against transient timeouts (60s timeout, retries,\nreport-timeouts-as-broken). A clean run of `make docs-linkcheck` on\nthat branch reports **0 broken / 0 timeouts** but **~49 permanent\nredirects** \u2014 external URLs that resolve fine (so CI stays green) but\npoint at stale locations that the upstream site 301/302-redirects to a\ncanonical URL.\n\nRedirects are informational and do NOT fail the build, so this is\npure hygiene: repoint the source links at their canonical targets so\nreaders (and future linkcheck runs) hit the real page directly. It is\nexplicitly NOT about the internal repo-relative links \u2014 those are the\nseparate `check_doc_relative_links.py` baseline burn-down tracked in\n`burn-down-baselined-broken-doc-links.yaml`. This TODO is external\nhttp/https only, driven off the Sphinx linkcheck redirect report.\n\nNOT every redirect should be chased \u2014 some are canonical/stable by\ndesign and repointing them is wrong or churny. Triage first (w1), then\nfix only the clear wins.\n\nRedirect clusters observed 2026-07-08 (regenerate to confirm before\nstarting \u2014 counts/targets drift as upstreams reorganize):\n REPOINT (clear canonical wins):\n - docs.snowflake.com/...html -> same path without `.html` (~6)\n - cloud.google.com/bigquery|storage -> docs.cloud.google.com/... (~10)\n - clickhouse.com/docs/en/... -> clickhouse.com/docs/... (drop /en) (~4)\n - arrow.apache.org/datafusion* -> datafusion.apache.org/* (2)\n - docs.databricks.com/...html -> docs.databricks.com/aws/en/... (~3)\n - docs.microsoft.com/... -> learn.microsoft.com/... (1)\n - docs.influxdata.com/influxdb/latest -> .../v2 (1)\n - docs.timescale.com -> www.tigerdata.com/docs (1)\n - velodb.io -> www.velodb.io ; singlestore.com/cloud -> /product-overview\n - clickhouse.cloud -> console.clickhouse.cloud\n - github.com/mwc360/LakeBench -> github.com/microsoft/LakeBench (1)\n LEAVE ALONE (do not repoint \u2014 record the decision in the PR):\n - doi.org/10.7910/... -> dataverse.harvard.edu (DOI is the stable\n citation identifier; the redirect is the point of a DOI).\n - github.com/.../issues/new -> github.com/login?return_to=... (auth\n gate for anonymous checkers; the source link is correct).\n - *.readthedocs.io/ -> /en/stable/ (version canonicalization; low\n value and prone to re-churn on the next docs release).\n - http://www.cs.umb.edu/... and other `:443`/scheme-normalization\n redirects that only add an explicit port/scheme.\n\nAlternative considered and rejected as the primary fix: silence stable\nredirects via `linkcheck_allowed_redirects` in `docs/conf.py`. That\nhides the signal rather than fixing the stale link, and readers still\nland on a redirect. Prefer fixing the source; reserve config for the\nLEAVE-ALONE set only if a future stricter policy is adopted.", "id": "canonicalize-redirected-external-doc-links", "preserves": ["DOI, auth-gated, readthedocs-version, and scheme/port-only redirects are left as-is \u2014 repointing them is incorrect.", "Internal repo-relative link handling is untouched \u2014 that is the separate check_doc_relative_links.py baseline burn-down.", "Sphinx linkcheck stays green (0 broken, 0 timeouts) after edits."], "prior_art": [{"concept": "burn-down-baselined-broken-doc-links.yaml \u2014 sibling docs-link TODO; REUSE its triage-table-in-PR pattern, but it covers INTERNAL repo-relative links via check_doc_relative_links.py; this TODO is genuinely new for EXTERNAL redirect canonicalization.", "decision": "reuse", "path": "burn-down-baselined-broken-doc-links.yaml"}, {"concept": "docs/conf.py:linkcheck_* (added in PR #1051) \u2014 the linkcheck config that surfaces the redirect report; REUSE `make docs-linkcheck` output as the input, do not modify the config here.", "decision": "reuse", "path": "docs/conf.py:linkcheck_*"}, {"concept": "docs/linkcheck_ignore.txt \u2014 existing external-link escape hatch for flaky/gated hosts; reference for where LEAVE-ALONE hosts would go IF a stricter policy is later adopted (not in scope now).", "decision": "reuse", "path": "docs/linkcheck_ignore.txt"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "docs/conf.py"}, {"kind": "do_not_modify", "path_glob": "docs/linkcheck_ignore.txt"}, {"kind": "do_not_modify", "path_glob": "scripts/doc_relative_link_baseline.txt"}, {"kind": "only_modify", "path_glob": "docs/"}], "state": "done", "title": "Canonicalize the ~49 redirected external doc links surfaced by linkcheck", "verifications": [{"command": "make docs-linkcheck", "description": "REPOINTed links now resolve directly (no redirect); build clean", "expected": "exit 0; 0 broken, 0 timeouts; redirect count reduced to the LEAVE-ALONE residual", "item_id": "canonicalize-redirected-external-doc-links", "last_result": null, "last_run": null, "seq": 1}, {"command": "! rg -n 'docs\\.snowflake\\.com/[^ )]+\\.html' docs/", "description": "Spot-check a representative canonicalization is gone from source", "expected": "no matches (Snowflake .html links repointed)", "item_id": "canonicalize-redirected-external-doc-links", "last_result": null, "last_run": null, "seq": 2}, {"command": "git diff --name-only develop -- scripts/doc_relative_link_baseline.txt", "description": "No repo-relative links were collaterally touched (that is the other TODO)", "expected": "empty (baseline untouched by this work)", "item_id": "canonicalize-redirected-external-doc-links", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "canonicalize-redirected-external-doc-links", "needs": [], "notes": "Bind the cited evidence before editing. On the implementation\nbranch run `make docs-linkcheck`, then extract the redirect report:\n grep -E '\\[redirected' docs/_build/linkcheck/output.txt\n(or parse `output.json` for `\"status\": \"redirected\"`). Record in\nthe PR description: the develop SHA, the total redirect count, and\nany drift from the ~49 captured 2026-07-08 (the clusters in this\nTODO's description are that snapshot). If targets shifted, re-derive\nthe w2 groupings before proceeding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate the redirect list against current develop", "wid": "w0"}, {"evidence": null, "item_id": "canonicalize-redirected-external-doc-links", "needs": ["w0"], "notes": "For each (source_file:line, from_url -> to_url) assign a disposition\nper the description's rules: REPOINT stale-path canonical rewrites;\nLEAVE-ALONE DOIs, auth redirects, readthedocs version canonicalization,\nand pure scheme/port (`:443`) normalizations. Record the table in the\nPR description as the audit trail for w2. When unsure, verify the\ntarget actually serves the intended content (open it) before\nrepointing \u2014 a redirect can land on a generic index (e.g. one AWS\n`whats-new` link collapses to `aws.amazon.com/about-aws/`), in which\ncase find the real successor page or LEAVE-ALONE with a note.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Triage every redirect into REPOINT vs LEAVE-ALONE", "wid": "w1"}, {"evidence": null, "item_id": "canonicalize-redirected-external-doc-links", "needs": ["w1"], "notes": "Edit the source `.md`/`.rst` files to the canonical target URLs from\nthe w1 table. Fix external http/https links only; leave repo-relative\nlinks to the other checker. Re-run `make docs-linkcheck` and confirm\nthe REPOINTed URLs now report `ok` (working) rather than `redirected`,\nthe redirect count drops to the LEAVE-ALONE residual, and broken/\ntimeout stay at 0. Standard close-out: `make lint format typecheck`,\n`code review` (fix findings), `make pr-open`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the REPOINT edits and re-run linkcheck", "wid": "w2"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (Should-Fix Before Release)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Align all exporters, docs, and tests on a single benchmark result JSON schema version 1.0 and remove legacy version branching.", "id": "canonicalize-result-schema", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Canonicalize Result Schema", "verifications": [], "work": [], "worktree": "release-readiness"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (Should-Fix Before Release)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Unified CLI and core exporters on schema v1.0, added canonical validation helpers, and documented the contract for downstream consumers.", "id": "canonicalize-result-schema-2", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Canonicalize Result Schema", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "make the smoke run in CI on every PR", "why": "it pulls fixtures and runs persist+merge cycles. It is an on-demand sweep tool, run quarterly or when bounds are touched."}, {"INSTEAD": "(not specified)", "dont": "replace the catalog `verified` comments with structured YAML annotations", "why": "the comments are reviewer-friendly prose; the smoke + test are the enforcement layer."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-06T00:00:00Z", "completed_pr": null, "created_at": "2026-05-05T00:00:00Z", "deferrals": [], "deps": ["extend-five-axis-review-rubric"], "description": "Blind-spot `2026-05-05-003357` documented that\n`benchbox/core/write_primitives/catalog/operations.yaml` carries hand-\nstamped `verified at SF=` comments tied to point-in-time\nobservations (e.g. 60003 / 4314 / 317 bytes for the ClickHouse smoke;\nsimilar numbers for DuckDB). Three durability problems:\n\n1. No regression alarm if observations drift after a tool upgrade.\n2. Doc-vs-yaml-bound skew is unguarded\n (`docs/benchmarks/write-primitives-sketch-functions.md` quotes\n \"~4.3KB on ClickHouse\" while operations.yaml carries `[1500, 8000]`\n bounds -- they can drift independently with no signal).\n3. Verification methodology is not codified -- the ad-hoc SQL script\n that captured the observations lived only in the session that wrote\n them.\n\nThis TODO ships a checked-in companion smoke + a doc-yaml consistency\ntest, and sweeps existing comments for staleness.", "id": "catalog-empirical-claim-durability", "preserves": ["Doc prose around storage-size claims continues to read naturally; the consistency test parses the existing prose pattern, not a machine-readable annotation.", "Existing `expected_value_min/max` bounds in `operations.yaml` are not loosened to make a stale comment pass -- if the smoke shows drift, the bound moves WITH a new dated comment line, not silently.", "Existing chDB or other in-flight sketch work is not blocked by this TODO -- the consistency test fails only on declared (yaml-bounded + doc-claimed) ops."], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".claude/skills/code/references/five-axis-review.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/catalog/operations.yaml"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/write-primitives-sketch-functions.md"}, {"kind": "only_modify", "path_glob": "scripts/sketch_storage_smoke.sh"}, {"kind": "only_modify", "path_glob": "tests/unit/core/write_primitives/test_doc_yaml_consistency.py"}], "state": "done", "title": "Add storage-bytes smoke and doc-yaml consistency check for catalog empirical claims", "verifications": [{"command": "test -f scripts/sketch_storage_smoke.sh", "description": "Smoke script exists", "expected": "exit 0", "item_id": "catalog-empirical-claim-durability", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/write_primitives/test_doc_yaml_consistency.py -v", "description": "Doc-yaml consistency test exists and passes", "expected": "exit 0", "item_id": "catalog-empirical-claim-durability", "last_result": null, "last_run": null, "seq": 2}, {"command": "ls _project/handoffs/catalog-verified-comment-sweep-*.md 2>/dev/null", "description": "Sweep handoff document exists", "expected": ">= 1 file (after w1 lands)", "item_id": "catalog-empirical-claim-durability", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": [], "notes": "Grep `benchbox/core/write_primitives/catalog/operations.yaml` for\n`verified` and produce a table: comment | tool | tool version at\nwrite time | current tool version | bound | observed bytes today\n(if measurable). Any row where current observation is outside the\nbound or where the tool version has moved is a sweep finding --\nrecord the table as a checked-in handoff at\n`_project/handoffs/catalog-verified-comment-sweep-2026-05-05.md`\n(or current date) so a reviewer can diff the next sweep against\nit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sweep existing `verified ` comments for date-staleness or version drift", "wid": "w1"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w1"], "notes": "Run the persist+merge cycles against the bundled SF=0.01 fixtures\nfor each tool (DuckDB, ClickHouse-local, optionally chDB) and\nprint observed bytes per op in a stable format (TSV: tool /\nop_id / observed_bytes). Stash it under\n`scripts/sketch_storage_smoke.sh` so the next person who refreshes\nthe numbers has a single command.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write `scripts/sketch_storage_smoke.sh` (or `make` target) that prints observed bytes", "wid": "w2"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w1"], "notes": "Fast-marked test in `tests/unit/core/write_primitives/\ntest_doc_yaml_consistency.py` that parses\n`docs/benchmarks/write-primitives-sketch-functions.md` for \"~Xkb\non \" claims and asserts each falls inside the matching\n`operations.yaml` `expected_value_min/max` bound. If the doc claim\nis missing a tool the catalog covers, fail with a clear \"claim\nmissing\" message. If the bound is missing, fail with \"doc claim\nhas no yaml bound.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add doc-yaml consistency test", "wid": "w3"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w1", "w2"], "notes": "Use the smoke output from w2 to refresh the comments. If a bound\nlegitimately moved (tool upgrade, schema change), document the\nbound update with a date + observed-bytes line under the existing\n`verified` comment so the next sweep can diff against it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-stamp any out-of-date catalog comments uncovered by w1", "wid": "w4"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w3"], "notes": "Extend `.claude/skills/code/references/five-axis-review.md` with a\nsub-axis under Correctness: \"For changes that update empirically-\nobserved numbers in catalog comments or docs, require a checked-in\nsmoke (or make target) that re-produces the observation, and a\nconsistency test that fails when doc claims drift outside catalog\nbounds.\" This sub-axis is explicitly delegated to this TODO by\n`extend-five-axis-review-rubric` (see its `deferred:` block) so\nthere is no double-edit risk. The `deps.needs` link below ensures\nthe parent TODO's six branches land first.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'durability of empirical claims' sub-axis to the review framework", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "subclass PostgreSQLAdapter (from benchbox.platforms.postgresql import PostgreSQLAdapter) as done in timescaledb.py, pg_mooncake.py, pg_duckdb.py", "dont": "duplicate PostgreSQL adapter logic", "why": "CedarDB uses PG wire protocol"}, {"INSTEAD": "make port configurable with a CedarDB-appropriate default", "dont": "hardcode PostgreSQL default port 5432", "why": "CedarDB may use a different port"}, {"INSTEAD": "create a new cedardb.py file", "dont": "modify existing adapter files", "why": "CedarDB is additive"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-07T00:00:00Z", "completed_pr": null, "created_at": "2026-01-26T00:00:00Z", "deferrals": [], "deps": [], "description": "Add support for CedarDB (formerly Umbra), a high-performance RDBMS that handles both\nOLAP and OLTP workloads with PostgreSQL wire protocol compatibility.\n\nCedarDB is notable for:\n- Blazingly fast analytical and transactional query execution\n- Full PostgreSQL SQL dialect and wire protocol compatibility\n- Strong ACID guarantees with MVCC\n- Works with existing PostgreSQL tools (psql, DBeaver, DataGrip)\n- Python connectivity via psycopg2/psycopg3 (PostgreSQL drivers)\n\nReference: https://cedardb.com/docs/", "id": "cedardb-adapter", "preserves": ["All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "PlatformRegistry.get_adapter_class() resolves all existing platforms - CedarDB registration in __init__.py must not break the eager-load block pattern (see timescaledb/pg_duckdb/pg_mooncake sections in __init__.py)", "PostgreSQLAdapter.__init__(), connect(), execute_query(), _load_data() behavior unchanged - CedarDB overrides only, no parent edits"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/timescaledb.py"}, {"kind": "only_modify", "path_glob": "_project/specs/platforms/cedardb.md"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/cedardb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/cedardb.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_cedardb.py"}], "state": "done", "title": "Add CedarDB Platform Adapter", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.cedardb import CedarDBAdapter; print('import OK')\"", "description": "New adapter imports correctly", "expected": "import OK", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_cedardb.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.platform_registry import PlatformRegistry; assert 'cedardb' in PlatformRegistry.get_available_platforms(), 'not registered'; print('registered OK')\"", "description": "CedarDB registered in PlatformRegistry", "expected": "registered OK", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "cedardb-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document CedarDB-specific SQL extensions and limitations", "wid": "w1"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Identify bulk loading capabilities and COPY support", "wid": "w2"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research tuning parameters and configuration options", "wid": "w3"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create CedarDBAdapter subclass with connection defaults and register in __init__.py eager-load block (follow timescaledb.py)", "wid": "w4"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w2", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add bulk loading support (COPY or custom) based on w2 research findings", "wid": "w5"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement platform-specific tuning configuration based on w3 research findings", "wid": "w6"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests: connection config construction, bulk load path, tuning config, dialect properties, get_platform_info() (follow test_pg_mooncake_adapter.py pattern with monkeypatched psycopg2)", "wid": "w7"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-H/TPC-DS validation at SF 0.01 and SF 1", "wid": "w8"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create platform documentation and spec page (_project/specs/platforms/cedardb.md)", "wid": "w9"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "route logical-to-dialect generation through `schema_emit` and post-generation surgery through `ddl_optimize`", "dont": "add a `ddl_compile` phase", "why": "`schema_emit` and `ddl_optimize` already cover DDL ownership and are frozen in the ADR"}, {"INSTEAD": "add rule modules under the existing phase directories", "dont": "create a parallel `benchbox/sql_compat/transformers/` tree", "why": "the shipped convention is `benchbox/sql_compat/rules/{phase}/`"}, {"INSTEAD": "if statement metadata is required, amend the existing DDL payload contract first", "dont": "extend `CompatibilityContext` with object_name, statement_kind, source_dialect, or target_dialect in this item", "why": "resolver specificity and plan caching are query-centric today"}, {"INSTEAD": "delete them or reduce them to documented `@compat_local` rendering only", "dont": "leave benchmark-local DDL compatibility branches in place once equivalent rules exist", "why": "shadowed inline logic defeats centralized ownership"}, {"INSTEAD": "centralize only pure DDL statement rewrites", "dont": "move adapter operational setup into the registry", "why": "catalog selection, search_path/session state, permissions, and pre-drop behavior are runtime concerns"}, {"INSTEAD": "capture any proven IR gap as a separate follow-on TODO", "dont": "promise a repo-wide schema-IR migration in this item", "why": "the current schema primitives cannot yet express all benchmark semantics cleanly"}], "approach": "Work inside the existing sql_compat architecture, not around it.\n\n1. Amend the active ADR and TODO notes first so the contract is explicit:\n DDL centralization is achieved with `schema_emit` plus `ddl_optimize`, not\n a new phase or a new resolver model.\n2. Use benchbox/sql_compat/rules/ddl_optimize/starrocks_ddl_rewrites.py as\n the implementation template for every remaining adapter-owned CREATE TABLE\n rewrite. Each adapter migration should produce a rule module plus adapter\n consumer cleanup in the same slice.\n3. Keep runtime-only behavior in adapters. Centralize only pure SQL rewrite\n logic; do not move catalog creation, connection/session setup, permissions,\n or other operational flow into the registry.\n4. After adapter parity is achieved, remove the base\n `platform_transform_fn` hook and then clean up residual benchmark-level\n policy branches. Do not begin with a speculative schema-IR migration:\n BaseSchemaTable is not rich enough today to justify that refactor here.\n5. Finish by tightening the inventory/lint/docs so new benchmark-level DDL\n monkey-patching becomes detectable and reviewable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-22T00:00:00Z", "completed_pr": null, "created_at": "2026-04-22T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "centralize-ddl-translation-phase", "id": 112, "reason": "BaseSchemaTable does not currently encode vector, partition, or time-series semantics well enough to justify a repo-wide migration in this item", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on a w16 audit proving current models can't remove local renderers; no evidence that condition was met.", "summary": "Expand schema primitives / IR only if the w16 audit proves current benchmark models cannot remove remaining local renderers"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "centralize-ddl-translation-phase", "id": 113, "reason": "Keep this item focused on SQL DDL ownership first", "resolution": "promoted", "resolved_item": "unify-dataframe-mode-ddl-generation-behind-the-same-registry", "resolved_reason": null, "summary": "Unify DataFrame-mode DDL generation behind the same registry-owned path"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "centralize-ddl-translation-phase", "id": 114, "reason": "First move ownership centrally; de-duplication is a separate cleanup pass", "resolution": "promoted", "resolved_item": "factor-shared-helper-libraries-across-presto-trino-spark-fam", "resolved_reason": null, "summary": "Factor shared helper libraries across Presto/Trino/Spark-family DDL optimizers after parity is established"}], "deps": ["build-sql-compat-phase-aware-pipeline"], "description": "Follow-on / amendment plan for `build-sql-compat-phase-aware-pipeline`. The ADR,\nphase enum, registry, resolver, `schema_emit` rules, and the first\n`ddl_optimize` rule already exist. The remaining DDL gap is narrower and more\nconcrete than the earlier draft assumed: many adapters still own late\n`CREATE TABLE` rewrites (`_optimize_table_definition()` or Databricks\n`_convert_to_delta_table()`), and base schema execution still exposes the\n`platform_transform_fn` callback.\n\nThis item does NOT introduce a new `ddl_compile` phase, a new\n`CompatibilityContext` shape, or a parallel `benchbox/sql_compat/transformers/`\ntree. The frozen phase model is sufficient:\n- `schema_emit` owns logical-to-dialect DDL policy\n- `ddl_optimize` owns post-generation statement rewrites\n- adapter `create_schema()` overrides keep only operational setup\n\nGoal: finish central DDL ownership inside the shipped sql_compat architecture,\nretire late adapter monkey-patching, and remove residual benchmark-level DDL\ncompatibility branches so benchmarks no longer need platform-specific\ncompatibility policy at schema generation time. If\n`build-sql-compat-phase-aware-pipeline` is still active when this work starts,\nfold overlapping slices into that TODO rather than landing a competing design.", "id": "centralize-ddl-translation-phase", "preserves": ["Adapter create_schema() methods keep operational responsibilities such as catalog/schema setup, search_path/session state, permissions, and pre-drop sequencing", "Base data-loading code keeps tuning lookup, dry-run support, logging, and table-level error reporting", "CompatibilityResolver.build_plan() and registry specificity remain query-centric in this item; no `CompatibilityContext` DDL-field expansion", "Existing schema_emit rules for nyctaxi and tsbs_devops remain the source of truth for benchmark-level DDL compatibility policy", "The Phase enum in benchbox/sql_compat/context.py remains the ADR-frozen set; do not add `ddl_compile`", "The shipped StarRocks ddl_optimize rule remains authoritative and unchanged except for any shared helper extraction that preserves behavior"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/schema_primitives.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/resolver.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/active/build-sql-compat-phase-aware-pipeline.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tsbs_devops/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/vector_search/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/azure_synapse.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/workload.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/fabric_warehouse.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/redshift.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/snowflake.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/trino.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/velox.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/inventory.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/ddl_optimize/"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/schema_emit/"}, {"kind": "only_modify", "path_glob": "docs/development/adr/adr-sql-compat-phase-aware-pipeline.md"}, {"kind": "only_modify", "path_glob": "scripts/compat_lint.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/sql_compat/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Finish DDL centralization in existing sql_compat phases (retire adapter DDL hooks)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/sql_compat/test_harness.py tests/unit/core/sql_compat/test_schema_emit_rules.py tests/unit/core/sql_compat/test_starrocks_pipeline_rules.py -q", "description": "sql_compat rule and harness tests still pass after DDL-optimize expansion", "expected": "all tests pass, 0 failures", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py tests/unit/platforms/test_databricks_adapter.py tests/unit/platforms/test_databend_adapter.py tests/unit/platforms/test_firebolt_adapter.py tests/unit/platforms/test_redshift_adapter.py tests/unit/platforms/test_azure_synapse_adapter.py tests/unit/platforms/test_snowflake_adapter.py tests/unit/platforms/test_presto_adapter.py tests/unit/platforms/test_trino_adapter.py tests/unit/platforms/test_spark_adapter.py tests/unit/platforms/test_lakesail_adapter.py tests/unit/platforms/test_velox_adapter.py tests/unit/platforms/test_fabric_warehouse_coverage.py -q", "description": "Platform adapter tests covering migrated CREATE TABLE rewrites still pass", "expected": "all tests pass, 0 failures", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_data_loading_operations.py tests/unit/platforms/test_platform_adapter_init_and_loading.py -q", "description": "Base schema execution remains correct after hook removal", "expected": "all tests pass, 0 failures", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --platform duckdb --benchmark nyctaxi --scale 0.01 --phases load --dry-run ./benchmark_runs/_tmp_nyctaxi_ddl --non-interactive && uv run -- benchbox run --platform duckdb --benchmark tsbs_devops --scale 0.01 --phases load --dry-run ./benchmark_runs/_tmp_tsbs_ddl --non-interactive", "description": "Benchmarks with prior inline DDL policy branches still dry-run successfully", "expected": "both dry-runs complete with valid CREATE TABLE output and no schema-generation exceptions", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python scripts/compat_lint.py --strict", "description": "Compatibility lint enforces zero unregistered benchmark-level DDL policy branches", "expected": "0 unregistered dialect branch(es) found in the scanned roots", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": [], "notes": "Update docs/development/adr/adr-sql-compat-phase-aware-pipeline.md and, if\nneeded, the remaining notes in _project/TODO/main/active/build-sql-compat-phase-aware-pipeline.yaml\nto record the real scope:\n- no new `ddl_compile` phase\n- no `CompatibilityContext` extension in this item\n- no parallel `sql_compat/transformers/` tree\n- remaining adapter-owned DDL hooks are the work to finish\n\nEnumerate the exact migration inventory here: ClickHouse, Databricks,\nDatabend, Firebolt, Redshift, Azure Synapse, Snowflake, Presto, Trino,\nSpark, Lakesail, Velox, and Fabric Warehouse. If a DDL rewrite needs\nstatement metadata beyond raw SQL (for example Fabric Warehouse's tuning-\naware path), amend the existing `RewriteDDLPayload` / pipeline plumbing in\nthe ADR first rather than overloading `CompatibilityContext`.\n\nREQUIRED before closing w1: explicitly resolve and document the Fabric\nWarehouse question from open_questions - \"is raw SQL sufficient or does\nRewriteDDLPayload need a richer execution contract?\" Record the decision\nand its rationale in the ADR amendment. This decision gates w14, which\nin turn gates the entire w15-w18 tail; leaving it open past w1 will stall\nthe work.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Amend the existing ADR and active sql_compat TODO with the correct DDL-centralization target", "wid": "w1"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Mirror the Presto migration for benchbox/platforms/trino.py, but keep the\nTrino-specific create_schema() flow and connector/runtime setup local.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Trino CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w10"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Centralize Spark's DDL rewrite logic and preserve adapter-local runtime\nsession/configuration behavior. Verify `USING`-clause handling continues to\nproduce exactly one storage clause after migration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Spark CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w11"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Lakesail DDL rewrite centrally and update\nbenchbox/platforms/lakesail.py to consume it. Preserve runtime-only cloud\nsetup and connection behavior in the adapter.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Lakesail CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w12"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Velox DDL rewrite centrally and update\nbenchbox/platforms/velox.py to consume it. Keep non-DDL execution setup\nlocal to the adapter.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Velox CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w13"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Fabric Warehouse currently has a tuning-aware `_optimize_table_definition`\nsignature. Migrate only after w1 resolves whether raw SQL is sufficient or\nwhether `RewriteDDLPayload` needs a richer execution contract. Do not solve\nthis by adding DDL fields to `CompatibilityContext`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Fabric Warehouse CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w14"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w10", "w11", "w12", "w13", "w14", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "Revisit benchbox/core/nyctaxi/schema.py and benchbox/core/tsbs_devops/schema.py.\nW15 of the active sql_compat item already registered central `schema_emit`\nrules for the compatibility-policy branches. This cleanup unit removes any\nnow-shadowed inline policy logic, leaving only true `@compat_local`\nrendering/type-mapping behavior. If a branch cannot be removed, document\nexactly why in the ADR amendment and inventory output.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Delete now-redundant inline DDL policy branches in nyctaxi and tsbs_devops", "wid": "w15"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w15"], "notes": "Review benchbox/core/vector_search/schema.py,\nbenchbox/core/write_primitives/schema.py, and\nbenchbox/core/read_primitives/schema.py for benchmark-level DDL logic that\nstill encodes platform compatibility policy rather than pure rendering.\nMove compatibility-policy branches into existing `schema_emit` rules.\nDo NOT force a repo-wide schema-IR migration here; if true zero-local-\nrendering requires richer schema primitives than the current codebase can\nrepresent, capture that as a separate follow-on item with concrete\nevidence instead of reopening the architecture in this TODO.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit vector_search and primitive schema helpers for residual benchmark-level DDL compatibility policy", "wid": "w16"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w10", "w11", "w12", "w13", "w14", "w15", "w16", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "Delete the `platform_transform_fn` parameter and all call sites in\nbenchbox/platforms/base/data_loading.py once every remaining CREATE TABLE\nrewrite flows through resolver-backed `ddl_optimize` rules. Preserve\nlogging, tuning lookup, statement execution, and error-reporting behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove platform_transform_fn from base schema execution after all callers migrate", "wid": "w17"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w17"], "notes": "Update benchbox/sql_compat/inventory.py and scripts/compat_lint.py so the\ntooling can catch residual benchmark-level DDL policy sites beyond simple\n`if dialect ...` branches where feasible, including the map-based cases\naudited in w16. Amend the existing ADR to document the final contract and\nexplicit disposition of any remaining `@compat_local` rendering-only sites.\nDo not add a second ADR for DDL compilation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Tighten lint, inventory, tests, and ADR wording to lock in centralized DDL ownership", "wid": "w18"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Add a ClickHouse rule module under benchbox/sql_compat/rules/ddl_optimize/\nfollowing the shipped StarRocks precedent, then update\nbenchbox/platforms/clickhouse/workload.py so registry-backed DDL rewrite\nis authoritative for CREATE TABLE statements. Keep ClickHouse-specific\nruntime setup in the adapter; centralize only the SQL rewrite logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move ClickHouse CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w2"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Databricks is a separate shape from the `_optimize_table_definition()`\nadapters. Register `_convert_to_delta_table()` as a DDL rewrite rule and\nreplace the `platform_transform_fn` usage in\nbenchbox/platforms/databricks/adapter.py with the registry-owned path.\nKeep catalog/schema creation, permissions, and other runtime setup local.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Databricks Delta-table CREATE TABLE rewrite under Phase.DDL_OPTIMIZE", "wid": "w3"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Databend DDL rewrite in benchbox/sql_compat/rules/ddl_optimize/\nand make benchbox/platforms/databend/adapter.py consume the central rule.\nDelete adapter-local rewrite logic once parity tests pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Databend CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w4"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Firebolt DDL rewrite in benchbox/sql_compat/rules/ddl_optimize/\nand make benchbox/platforms/firebolt.py consume the central rule. Preserve\nnon-DDL operational behavior in create_schema().\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Firebolt CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w5"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Centralize the Redshift DDL rewrite while leaving adapter runtime concerns\nsuch as search_path/session setup and pre-drop behavior local to\nbenchbox/platforms/redshift.py. Do not mix operational setup into the rule.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Redshift CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w6"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Azure Synapse table-definition rewrite centrally and update\nbenchbox/platforms/azure_synapse.py to consume it. External object setup\nand connection/session behavior remain adapter-local.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Azure Synapse CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w7"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Centralize Snowflake's CREATE TABLE rewrite and preserve the adapter's\nruntime-only responsibilities. Verify clustering/tuning-related SQL\nremains correct after moving the rewrite authority into the rule registry.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Snowflake CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w8"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Presto DDL rewrite in the existing rules layout and make\nbenchbox/platforms/presto.py consume it. Keep connector/catalog setup\noutside the rule path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Presto CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "make the smoke run in CI on every PR", "instead": "(not specified)", "why": "it pulls fixtures and runs persist+merge cycles. It is an on-demand sweep tool, run quarterly or when bounds are touched."}, {"dont": "replace the catalog `verified` comments with structured YAML annotations", "instead": "(not specified)", "why": "the comments are reviewer-friendly prose; the smoke + test are the enforcement layer."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-06T00:00:00Z", "completed_pr": null, "created_at": "2026-05-05T00:00:00Z", "deferrals": [], "deps": ["extend-five-axis-review-rubric"], "description": "Blind-spot `2026-05-05-003357` documented that\n`benchbox/core/write_primitives/catalog/operations.yaml` carries hand-\nstamped `verified at SF=` comments tied to point-in-time\nobservations (e.g. 60003 / 4314 / 317 bytes for the ClickHouse smoke;\nsimilar numbers for DuckDB). Three durability problems:\n\n1. No regression alarm if observations drift after a tool upgrade.\n2. Doc-vs-yaml-bound skew is unguarded\n (`docs/benchmarks/write-primitives-sketch-functions.md` quotes\n \"~4.3KB on ClickHouse\" while operations.yaml carries `[1500, 8000]`\n bounds -- they can drift independently with no signal).\n3. Verification methodology is not codified -- the ad-hoc SQL script\n that captured the observations lived only in the session that wrote\n them.\n\nThis TODO ships a checked-in companion smoke + a doc-yaml consistency\ntest, and sweeps existing comments for staleness.", "id": "catalog-empirical-claim-durability", "preserves": ["Doc prose around storage-size claims continues to read naturally; the consistency test parses the existing prose pattern, not a machine-readable annotation.", "Existing `expected_value_min/max` bounds in `operations.yaml` are not loosened to make a stale comment pass -- if the smoke shows drift, the bound moves WITH a new dated comment line, not silently.", "Existing chDB or other in-flight sketch work is not blocked by this TODO -- the consistency test fails only on declared (yaml-bounded + doc-claimed) ops."], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".claude/skills/code/references/five-axis-review.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/catalog/operations.yaml"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/write-primitives-sketch-functions.md"}, {"kind": "only_modify", "path_glob": "scripts/sketch_storage_smoke.sh"}, {"kind": "only_modify", "path_glob": "tests/unit/core/write_primitives/test_doc_yaml_consistency.py"}], "state": "done", "title": "Add storage-bytes smoke and doc-yaml consistency check for catalog empirical claims", "verifications": [{"command": "test -f scripts/sketch_storage_smoke.sh", "description": "Smoke script exists", "expected": "exit 0", "item_id": "catalog-empirical-claim-durability", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/write_primitives/test_doc_yaml_consistency.py -v", "description": "Doc-yaml consistency test exists and passes", "expected": "exit 0", "item_id": "catalog-empirical-claim-durability", "last_result": null, "last_run": null, "seq": 2}, {"command": "ls _project/handoffs/catalog-verified-comment-sweep-*.md 2>/dev/null", "description": "Sweep handoff document exists", "expected": ">= 1 file (after w1 lands)", "item_id": "catalog-empirical-claim-durability", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": [], "notes": "Grep `benchbox/core/write_primitives/catalog/operations.yaml` for\n`verified` and produce a table: comment | tool | tool version at\nwrite time | current tool version | bound | observed bytes today\n(if measurable). Any row where current observation is outside the\nbound or where the tool version has moved is a sweep finding --\nrecord the table as a checked-in handoff at\n`_project/handoffs/catalog-verified-comment-sweep-2026-05-05.md`\n(or current date) so a reviewer can diff the next sweep against\nit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sweep existing `verified ` comments for date-staleness or version drift", "wid": "w1"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w1"], "notes": "Run the persist+merge cycles against the bundled SF=0.01 fixtures\nfor each tool (DuckDB, ClickHouse-local, optionally chDB) and\nprint observed bytes per op in a stable format (TSV: tool /\nop_id / observed_bytes). Stash it under\n`scripts/sketch_storage_smoke.sh` so the next person who refreshes\nthe numbers has a single command.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write `scripts/sketch_storage_smoke.sh` (or `make` target) that prints observed bytes", "wid": "w2"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w1"], "notes": "Fast-marked test in `tests/unit/core/write_primitives/\ntest_doc_yaml_consistency.py` that parses\n`docs/benchmarks/write-primitives-sketch-functions.md` for \"~Xkb\non \" claims and asserts each falls inside the matching\n`operations.yaml` `expected_value_min/max` bound. If the doc claim\nis missing a tool the catalog covers, fail with a clear \"claim\nmissing\" message. If the bound is missing, fail with \"doc claim\nhas no yaml bound.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add doc-yaml consistency test", "wid": "w3"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w1", "w2"], "notes": "Use the smoke output from w2 to refresh the comments. If a bound\nlegitimately moved (tool upgrade, schema change), document the\nbound update with a date + observed-bytes line under the existing\n`verified` comment so the next sweep can diff against it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-stamp any out-of-date catalog comments uncovered by w1", "wid": "w4"}, {"evidence": null, "item_id": "catalog-empirical-claim-durability", "needs": ["w3"], "notes": "Extend `.claude/skills/code/references/five-axis-review.md` with a\nsub-axis under Correctness: \"For changes that update empirically-\nobserved numbers in catalog comments or docs, require a checked-in\nsmoke (or make target) that re-produces the observation, and a\nconsistency test that fails when doc claims drift outside catalog\nbounds.\" This sub-axis is explicitly delegated to this TODO by\n`extend-five-axis-review-rubric` (see its `deferred:` block) so\nthere is no double-edit risk. The `deps.needs` link below ensures\nthe parent TODO's six branches land first.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'durability of empirical claims' sub-axis to the review framework", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "duplicate PostgreSQL adapter logic", "instead": "subclass PostgreSQLAdapter (from benchbox.platforms.postgresql import PostgreSQLAdapter) as done in timescaledb.py, pg_mooncake.py, pg_duckdb.py", "why": "CedarDB uses PG wire protocol"}, {"dont": "hardcode PostgreSQL default port 5432", "instead": "make port configurable with a CedarDB-appropriate default", "why": "CedarDB may use a different port"}, {"dont": "modify existing adapter files", "instead": "create a new cedardb.py file", "why": "CedarDB is additive"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-07T00:00:00Z", "completed_pr": null, "created_at": "2026-01-26T00:00:00Z", "deferrals": [], "deps": [], "description": "Add support for CedarDB (formerly Umbra), a high-performance RDBMS that handles both\nOLAP and OLTP workloads with PostgreSQL wire protocol compatibility.\n\nCedarDB is notable for:\n- Blazingly fast analytical and transactional query execution\n- Full PostgreSQL SQL dialect and wire protocol compatibility\n- Strong ACID guarantees with MVCC\n- Works with existing PostgreSQL tools (psql, DBeaver, DataGrip)\n- Python connectivity via psycopg2/psycopg3 (PostgreSQL drivers)\n\nReference: https://cedardb.com/docs/", "id": "cedardb-adapter", "preserves": ["All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "PlatformRegistry.get_adapter_class() resolves all existing platforms - CedarDB registration in __init__.py must not break the eager-load block pattern (see timescaledb/pg_duckdb/pg_mooncake sections in __init__.py)", "PostgreSQLAdapter.__init__(), connect(), execute_query(), _load_data() behavior unchanged - CedarDB overrides only, no parent edits"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/timescaledb.py"}, {"kind": "only_modify", "path_glob": "_project/specs/platforms/cedardb.md"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/cedardb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/cedardb.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_cedardb.py"}], "state": "done", "title": "Add CedarDB Platform Adapter", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.cedardb import CedarDBAdapter; print('import OK')\"", "description": "New adapter imports correctly", "expected": "import OK", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_cedardb.py -v --tb=short", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.platform_registry import PlatformRegistry; assert 'cedardb' in PlatformRegistry.get_available_platforms(), 'not registered'; print('registered OK')\"", "description": "CedarDB registered in PlatformRegistry", "expected": "registered OK", "item_id": "cedardb-adapter", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "cedardb-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document CedarDB-specific SQL extensions and limitations", "wid": "w1"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Identify bulk loading capabilities and COPY support", "wid": "w2"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research tuning parameters and configuration options", "wid": "w3"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create CedarDBAdapter subclass with connection defaults and register in __init__.py eager-load block (follow timescaledb.py)", "wid": "w4"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w2", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add bulk loading support (COPY or custom) based on w2 research findings", "wid": "w5"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement platform-specific tuning configuration based on w3 research findings", "wid": "w6"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests: connection config construction, bulk load path, tuning config, dialect properties, get_platform_info() (follow test_pg_mooncake_adapter.py pattern with monkeypatched psycopg2)", "wid": "w7"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-H/TPC-DS validation at SF 0.01 and SF 1", "wid": "w8"}, {"evidence": null, "item_id": "cedardb-adapter", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create platform documentation and spec page (_project/specs/platforms/cedardb.md)", "wid": "w9"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "add a `ddl_compile` phase", "instead": "route logical-to-dialect generation through `schema_emit` and post-generation surgery through `ddl_optimize`", "why": "`schema_emit` and `ddl_optimize` already cover DDL ownership and are frozen in the ADR"}, {"dont": "create a parallel `benchbox/sql_compat/transformers/` tree", "instead": "add rule modules under the existing phase directories", "why": "the shipped convention is `benchbox/sql_compat/rules/{phase}/`"}, {"dont": "extend `CompatibilityContext` with object_name, statement_kind, source_dialect, or target_dialect in this item", "instead": "if statement metadata is required, amend the existing DDL payload contract first", "why": "resolver specificity and plan caching are query-centric today"}, {"dont": "leave benchmark-local DDL compatibility branches in place once equivalent rules exist", "instead": "delete them or reduce them to documented `@compat_local` rendering only", "why": "shadowed inline logic defeats centralized ownership"}, {"dont": "move adapter operational setup into the registry", "instead": "centralize only pure DDL statement rewrites", "why": "catalog selection, search_path/session state, permissions, and pre-drop behavior are runtime concerns"}, {"dont": "promise a repo-wide schema-IR migration in this item", "instead": "capture any proven IR gap as a separate follow-on TODO", "why": "the current schema primitives cannot yet express all benchmark semantics cleanly"}], "approach": "Work inside the existing sql_compat architecture, not around it.\n\n1. Amend the active ADR and TODO notes first so the contract is explicit:\n DDL centralization is achieved with `schema_emit` plus `ddl_optimize`, not\n a new phase or a new resolver model.\n2. Use benchbox/sql_compat/rules/ddl_optimize/starrocks_ddl_rewrites.py as\n the implementation template for every remaining adapter-owned CREATE TABLE\n rewrite. Each adapter migration should produce a rule module plus adapter\n consumer cleanup in the same slice.\n3. Keep runtime-only behavior in adapters. Centralize only pure SQL rewrite\n logic; do not move catalog creation, connection/session setup, permissions,\n or other operational flow into the registry.\n4. After adapter parity is achieved, remove the base\n `platform_transform_fn` hook and then clean up residual benchmark-level\n policy branches. Do not begin with a speculative schema-IR migration:\n BaseSchemaTable is not rich enough today to justify that refactor here.\n5. Finish by tightening the inventory/lint/docs so new benchmark-level DDL\n monkey-patching becomes detectable and reviewable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-22T00:00:00Z", "completed_pr": null, "created_at": "2026-04-22T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "centralize-ddl-translation-phase", "id": 112, "reason": "BaseSchemaTable does not currently encode vector, partition, or time-series semantics well enough to justify a repo-wide migration in this item", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on a w16 audit proving current models can't remove local renderers; no evidence that condition was met.", "summary": "Expand schema primitives / IR only if the w16 audit proves current benchmark models cannot remove remaining local renderers"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "centralize-ddl-translation-phase", "id": 113, "reason": "Keep this item focused on SQL DDL ownership first", "resolution": "promoted", "resolved_item": "unify-dataframe-mode-ddl-generation-behind-the-same-registry", "resolved_reason": null, "summary": "Unify DataFrame-mode DDL generation behind the same registry-owned path"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "centralize-ddl-translation-phase", "id": 114, "reason": "First move ownership centrally; de-duplication is a separate cleanup pass", "resolution": "promoted", "resolved_item": "factor-shared-helper-libraries-across-presto-trino-spark-fam", "resolved_reason": null, "summary": "Factor shared helper libraries across Presto/Trino/Spark-family DDL optimizers after parity is established"}], "deps": ["build-sql-compat-phase-aware-pipeline"], "description": "Follow-on / amendment plan for `build-sql-compat-phase-aware-pipeline`. The ADR,\nphase enum, registry, resolver, `schema_emit` rules, and the first\n`ddl_optimize` rule already exist. The remaining DDL gap is narrower and more\nconcrete than the earlier draft assumed: many adapters still own late\n`CREATE TABLE` rewrites (`_optimize_table_definition()` or Databricks\n`_convert_to_delta_table()`), and base schema execution still exposes the\n`platform_transform_fn` callback.\n\nThis item does NOT introduce a new `ddl_compile` phase, a new\n`CompatibilityContext` shape, or a parallel `benchbox/sql_compat/transformers/`\ntree. The frozen phase model is sufficient:\n- `schema_emit` owns logical-to-dialect DDL policy\n- `ddl_optimize` owns post-generation statement rewrites\n- adapter `create_schema()` overrides keep only operational setup\n\nGoal: finish central DDL ownership inside the shipped sql_compat architecture,\nretire late adapter monkey-patching, and remove residual benchmark-level DDL\ncompatibility branches so benchmarks no longer need platform-specific\ncompatibility policy at schema generation time. If\n`build-sql-compat-phase-aware-pipeline` is still active when this work starts,\nfold overlapping slices into that TODO rather than landing a competing design.", "id": "centralize-ddl-translation-phase", "preserves": ["Adapter create_schema() methods keep operational responsibilities such as catalog/schema setup, search_path/session state, permissions, and pre-drop sequencing", "Base data-loading code keeps tuning lookup, dry-run support, logging, and table-level error reporting", "CompatibilityResolver.build_plan() and registry specificity remain query-centric in this item; no `CompatibilityContext` DDL-field expansion", "Existing schema_emit rules for nyctaxi and tsbs_devops remain the source of truth for benchmark-level DDL compatibility policy", "The Phase enum in benchbox/sql_compat/context.py remains the ADR-frozen set; do not add `ddl_compile`", "The shipped StarRocks ddl_optimize rule remains authoritative and unchanged except for any shared helper extraction that preserves behavior"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/schema_primitives.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/resolver.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/active/build-sql-compat-phase-aware-pipeline.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tsbs_devops/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/vector_search/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/azure_synapse.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/workload.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/fabric_warehouse.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/redshift.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/snowflake.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/trino.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/velox.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/inventory.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/ddl_optimize/"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/schema_emit/"}, {"kind": "only_modify", "path_glob": "docs/development/adr/adr-sql-compat-phase-aware-pipeline.md"}, {"kind": "only_modify", "path_glob": "scripts/compat_lint.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/sql_compat/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Finish DDL centralization in existing sql_compat phases (retire adapter DDL hooks)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/sql_compat/test_harness.py tests/unit/core/sql_compat/test_schema_emit_rules.py tests/unit/core/sql_compat/test_starrocks_pipeline_rules.py -q", "description": "sql_compat rule and harness tests still pass after DDL-optimize expansion", "expected": "all tests pass, 0 failures", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py tests/unit/platforms/test_databricks_adapter.py tests/unit/platforms/test_databend_adapter.py tests/unit/platforms/test_firebolt_adapter.py tests/unit/platforms/test_redshift_adapter.py tests/unit/platforms/test_azure_synapse_adapter.py tests/unit/platforms/test_snowflake_adapter.py tests/unit/platforms/test_presto_adapter.py tests/unit/platforms/test_trino_adapter.py tests/unit/platforms/test_spark_adapter.py tests/unit/platforms/test_lakesail_adapter.py tests/unit/platforms/test_velox_adapter.py tests/unit/platforms/test_fabric_warehouse_coverage.py -q", "description": "Platform adapter tests covering migrated CREATE TABLE rewrites still pass", "expected": "all tests pass, 0 failures", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_data_loading_operations.py tests/unit/platforms/test_platform_adapter_init_and_loading.py -q", "description": "Base schema execution remains correct after hook removal", "expected": "all tests pass, 0 failures", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --platform duckdb --benchmark nyctaxi --scale 0.01 --phases load --dry-run ./benchmark_runs/_tmp_nyctaxi_ddl --non-interactive && uv run -- benchbox run --platform duckdb --benchmark tsbs_devops --scale 0.01 --phases load --dry-run ./benchmark_runs/_tmp_tsbs_ddl --non-interactive", "description": "Benchmarks with prior inline DDL policy branches still dry-run successfully", "expected": "both dry-runs complete with valid CREATE TABLE output and no schema-generation exceptions", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python scripts/compat_lint.py --strict", "description": "Compatibility lint enforces zero unregistered benchmark-level DDL policy branches", "expected": "0 unregistered dialect branch(es) found in the scanned roots", "item_id": "centralize-ddl-translation-phase", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": [], "notes": "Update docs/development/adr/adr-sql-compat-phase-aware-pipeline.md and, if\nneeded, the remaining notes in _project/TODO/main/active/build-sql-compat-phase-aware-pipeline.yaml\nto record the real scope:\n- no new `ddl_compile` phase\n- no `CompatibilityContext` extension in this item\n- no parallel `sql_compat/transformers/` tree\n- remaining adapter-owned DDL hooks are the work to finish\n\nEnumerate the exact migration inventory here: ClickHouse, Databricks,\nDatabend, Firebolt, Redshift, Azure Synapse, Snowflake, Presto, Trino,\nSpark, Lakesail, Velox, and Fabric Warehouse. If a DDL rewrite needs\nstatement metadata beyond raw SQL (for example Fabric Warehouse's tuning-\naware path), amend the existing `RewriteDDLPayload` / pipeline plumbing in\nthe ADR first rather than overloading `CompatibilityContext`.\n\nREQUIRED before closing w1: explicitly resolve and document the Fabric\nWarehouse question from open_questions - \"is raw SQL sufficient or does\nRewriteDDLPayload need a richer execution contract?\" Record the decision\nand its rationale in the ADR amendment. This decision gates w14, which\nin turn gates the entire w15-w18 tail; leaving it open past w1 will stall\nthe work.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Amend the existing ADR and active sql_compat TODO with the correct DDL-centralization target", "wid": "w1"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Mirror the Presto migration for benchbox/platforms/trino.py, but keep the\nTrino-specific create_schema() flow and connector/runtime setup local.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Trino CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w10"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Centralize Spark's DDL rewrite logic and preserve adapter-local runtime\nsession/configuration behavior. Verify `USING`-clause handling continues to\nproduce exactly one storage clause after migration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Spark CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w11"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Lakesail DDL rewrite centrally and update\nbenchbox/platforms/lakesail.py to consume it. Preserve runtime-only cloud\nsetup and connection behavior in the adapter.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Lakesail CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w12"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Velox DDL rewrite centrally and update\nbenchbox/platforms/velox.py to consume it. Keep non-DDL execution setup\nlocal to the adapter.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Velox CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w13"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Fabric Warehouse currently has a tuning-aware `_optimize_table_definition`\nsignature. Migrate only after w1 resolves whether raw SQL is sufficient or\nwhether `RewriteDDLPayload` needs a richer execution contract. Do not solve\nthis by adding DDL fields to `CompatibilityContext`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Fabric Warehouse CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w14"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w10", "w11", "w12", "w13", "w14", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "Revisit benchbox/core/nyctaxi/schema.py and benchbox/core/tsbs_devops/schema.py.\nW15 of the active sql_compat item already registered central `schema_emit`\nrules for the compatibility-policy branches. This cleanup unit removes any\nnow-shadowed inline policy logic, leaving only true `@compat_local`\nrendering/type-mapping behavior. If a branch cannot be removed, document\nexactly why in the ADR amendment and inventory output.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Delete now-redundant inline DDL policy branches in nyctaxi and tsbs_devops", "wid": "w15"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w15"], "notes": "Review benchbox/core/vector_search/schema.py,\nbenchbox/core/write_primitives/schema.py, and\nbenchbox/core/read_primitives/schema.py for benchmark-level DDL logic that\nstill encodes platform compatibility policy rather than pure rendering.\nMove compatibility-policy branches into existing `schema_emit` rules.\nDo NOT force a repo-wide schema-IR migration here; if true zero-local-\nrendering requires richer schema primitives than the current codebase can\nrepresent, capture that as a separate follow-on item with concrete\nevidence instead of reopening the architecture in this TODO.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit vector_search and primitive schema helpers for residual benchmark-level DDL compatibility policy", "wid": "w16"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w10", "w11", "w12", "w13", "w14", "w15", "w16", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "Delete the `platform_transform_fn` parameter and all call sites in\nbenchbox/platforms/base/data_loading.py once every remaining CREATE TABLE\nrewrite flows through resolver-backed `ddl_optimize` rules. Preserve\nlogging, tuning lookup, statement execution, and error-reporting behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove platform_transform_fn from base schema execution after all callers migrate", "wid": "w17"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w17"], "notes": "Update benchbox/sql_compat/inventory.py and scripts/compat_lint.py so the\ntooling can catch residual benchmark-level DDL policy sites beyond simple\n`if dialect ...` branches where feasible, including the map-based cases\naudited in w16. Amend the existing ADR to document the final contract and\nexplicit disposition of any remaining `@compat_local` rendering-only sites.\nDo not add a second ADR for DDL compilation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Tighten lint, inventory, tests, and ADR wording to lock in centralized DDL ownership", "wid": "w18"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Add a ClickHouse rule module under benchbox/sql_compat/rules/ddl_optimize/\nfollowing the shipped StarRocks precedent, then update\nbenchbox/platforms/clickhouse/workload.py so registry-backed DDL rewrite\nis authoritative for CREATE TABLE statements. Keep ClickHouse-specific\nruntime setup in the adapter; centralize only the SQL rewrite logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move ClickHouse CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w2"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Databricks is a separate shape from the `_optimize_table_definition()`\nadapters. Register `_convert_to_delta_table()` as a DDL rewrite rule and\nreplace the `platform_transform_fn` usage in\nbenchbox/platforms/databricks/adapter.py with the registry-owned path.\nKeep catalog/schema creation, permissions, and other runtime setup local.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Databricks Delta-table CREATE TABLE rewrite under Phase.DDL_OPTIMIZE", "wid": "w3"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Databend DDL rewrite in benchbox/sql_compat/rules/ddl_optimize/\nand make benchbox/platforms/databend/adapter.py consume the central rule.\nDelete adapter-local rewrite logic once parity tests pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Databend CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w4"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Firebolt DDL rewrite in benchbox/sql_compat/rules/ddl_optimize/\nand make benchbox/platforms/firebolt.py consume the central rule. Preserve\nnon-DDL operational behavior in create_schema().\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Firebolt CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w5"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Centralize the Redshift DDL rewrite while leaving adapter runtime concerns\nsuch as search_path/session setup and pre-drop behavior local to\nbenchbox/platforms/redshift.py. Do not mix operational setup into the rule.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Redshift CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w6"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Azure Synapse table-definition rewrite centrally and update\nbenchbox/platforms/azure_synapse.py to consume it. External object setup\nand connection/session behavior remain adapter-local.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Azure Synapse CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w7"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Centralize Snowflake's CREATE TABLE rewrite and preserve the adapter's\nruntime-only responsibilities. Verify clustering/tuning-related SQL\nremains correct after moving the rewrite authority into the rule registry.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Snowflake CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w8"}, {"evidence": null, "item_id": "centralize-ddl-translation-phase", "needs": ["w1"], "notes": "Register the Presto DDL rewrite in the existing rules layout and make\nbenchbox/platforms/presto.py consume it. Keep connector/catalog setup\noutside the rule path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move Presto CREATE TABLE rewriting under Phase.DDL_OPTIMIZE rule ownership", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-25T00:00:00Z", "completed_pr": null, "created_at": "2026-01-25T00:00:00Z", "deferrals": [], "deps": [], "description": "Extract file format detection and compression extension handling to a centralized utility.\n\nCurrently, compression extension sets and format detection logic are duplicated across 5+ files\nwith inconsistent coverage (some include .snappy, others don't; some handle .lz4, others don't).\n\nThis violates DRY and creates maintenance burden - adding a new compression format requires\nupdating multiple scattered locations.", "id": "centralize-file-format-detection", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Centralize File Format and Compression Extension Detection", "verifications": [], "work": [], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "use benchbox.cli.shared.console or injected quiet-aware console", "dont": "keep command-local Console() singletons", "why": "they bypass shared quiet policy"}, {"INSTEAD": "route runtime output through centralized helpers/logging", "dont": "rely only on call-site silence_output wrappers", "why": "transitive core calls can still leak output in other entrypoints"}, {"INSTEAD": "use centralized printing/logging helpers", "dont": "use bare print() in runtime execution paths", "why": "it bypasses quiet controls and MCP stdout constraints"}], "approach": "Extend benchbox.utils.printing as the canonical output API and forbid new direct Console()/print() usage\nin runtime modules. Replace command-local Console() instances with benchbox.cli.shared.console.\nFor core execution paths that can be called by CLI and MCP, route output through verbosity-aware logger\nor centralized helpers; use silence_output only as a compatibility boundary where upstream third-party\nwrites cannot be controlled directly. Add targeted tests for quiet behavior in CLI and MCP tool execution.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Audit findings show multiple user-visible output paths bypassing the centralized output mechanism:\n- direct Console() instances in CLI commands\n- raw print() in core execution/data-generation flows\n- mixed output channels in interactive shell paths\n- transitive leak risk for MCP tools that invoke core code paths\n\nThis item consolidates all user-facing output behind a central output module and enforces quiet-mode\nguarantees across CLI and MCP execution paths.", "id": "centralize-output-control-and-eliminate-print-leaks", "preserves": ["Existing non-quiet CLI UX (tables/panels/help text) remains functionally unchanged", "MCP server stdout remains reserved for JSON-RPC messages only", "benchbox run --quiet produces no user-visible stdout/stderr output while still returning correct exit codes"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/DONE/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/compare_plans.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/plan_history.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/report.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/results.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/shell.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/show_plan.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/shared.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/config_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/display.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/filesystem.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/streaming.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/server.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/printing.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_quiet_mode.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_benchmark_stdout_suppression.py"}], "state": "done", "title": "Centralize user output control and eliminate print leaks across CLI/core/MCP paths", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/test_quiet_mode.py -q", "description": "Quiet-mode unit tests pass for CLI output suppression", "expected": "all tests pass", "item_id": "centralize-output-control-and-eliminate-print-leaks", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/mcp -q", "description": "MCP tools do not emit non-JSON output to stdout during benchmark execution paths", "expected": "all tests pass with no stdout protocol violations", "item_id": "centralize-output-control-and-eliminate-print-leaks", "last_result": null, "last_run": null, "seq": 2}, {"command": "rg -n '^\\s*print\\(' benchbox/cli/commands/run.py benchbox/cli/commands/results.py benchbox/cli/commands/report.py benchbox/cli/commands/show_plan.py benchbox/cli/commands/compare_plans.py benchbox/cli/commands/plan_history.py benchbox/cli/commands/shell.py benchbox/core/runner/runner.py benchbox/core/tpcds/generator/manager.py benchbox/core/tpcds/generator/streaming.py benchbox/core/tpcds/generator/filesystem.py benchbox/core/tpcds/generator/runner.py benchbox/core/tpcds/benchmark/runner.py benchbox/core/results/display.py benchbox/core/config_utils.py benchbox/mcp/tools/benchmark.py", "description": "No direct runtime print leaks remain in CLI/core runtime modules", "expected": "no matches", "item_id": "centralize-output-control-and-eliminate-print-leaks", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create canonical output policy and API in a central module", "wid": "w1"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full verification suite and close remaining output-control gaps", "wid": "w10"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate CLI commands (results/report/show-plan/compare-plans/plan-history) from Console() to shared quiet-aware console", "wid": "w2"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Normalize shell command output by removing raw print() paths and using centralized console helpers", "wid": "w3"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace raw print() in core runner/config/results display modules with centralized output/logging", "wid": "w4"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace raw print() in TPC-DS generator and benchmark runtime modules with centralized output/logging", "wid": "w5"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wrap MCP benchmark execution/generation entrypoints with stdout/stderr suppression boundaries", "wid": "w6"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add static output-leak enforcement checks for runtime modules", "wid": "w7"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand CLI quiet-mode tests to cover consolidated output channels", "wid": "w8"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add MCP regression tests asserting no non-JSON stdout leakage in benchmark tool paths", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "keep command-local Console() singletons", "instead": "use benchbox.cli.shared.console or injected quiet-aware console", "why": "they bypass shared quiet policy"}, {"dont": "rely only on call-site silence_output wrappers", "instead": "route runtime output through centralized helpers/logging", "why": "transitive core calls can still leak output in other entrypoints"}, {"dont": "use bare print() in runtime execution paths", "instead": "use centralized printing/logging helpers", "why": "it bypasses quiet controls and MCP stdout constraints"}], "approach": "Extend benchbox.utils.printing as the canonical output API and forbid new direct Console()/print() usage\nin runtime modules. Replace command-local Console() instances with benchbox.cli.shared.console.\nFor core execution paths that can be called by CLI and MCP, route output through verbosity-aware logger\nor centralized helpers; use silence_output only as a compatibility boundary where upstream third-party\nwrites cannot be controlled directly. Add targeted tests for quiet behavior in CLI and MCP tool execution.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Audit findings show multiple user-visible output paths bypassing the centralized output mechanism:\n- direct Console() instances in CLI commands\n- raw print() in core execution/data-generation flows\n- mixed output channels in interactive shell paths\n- transitive leak risk for MCP tools that invoke core code paths\n\nThis item consolidates all user-facing output behind a central output module and enforces quiet-mode\nguarantees across CLI and MCP execution paths.", "id": "centralize-output-control-and-eliminate-print-leaks", "preserves": ["Existing non-quiet CLI UX (tables/panels/help text) remains functionally unchanged", "MCP server stdout remains reserved for JSON-RPC messages only", "benchbox run --quiet produces no user-visible stdout/stderr output while still returning correct exit codes"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/DONE/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/compare_plans.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/plan_history.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/report.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/results.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/shell.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/show_plan.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/shared.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/config_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/display.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/filesystem.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/streaming.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/server.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/printing.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_quiet_mode.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_mcp_benchmark_stdout_suppression.py"}], "state": "done", "title": "Centralize user output control and eliminate print leaks across CLI/core/MCP paths", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/test_quiet_mode.py -q", "description": "Quiet-mode unit tests pass for CLI output suppression", "expected": "all tests pass", "item_id": "centralize-output-control-and-eliminate-print-leaks", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/mcp -q", "description": "MCP tools do not emit non-JSON output to stdout during benchmark execution paths", "expected": "all tests pass with no stdout protocol violations", "item_id": "centralize-output-control-and-eliminate-print-leaks", "last_result": null, "last_run": null, "seq": 2}, {"command": "rg -n '^\\s*print\\(' benchbox/cli/commands/run.py benchbox/cli/commands/results.py benchbox/cli/commands/report.py benchbox/cli/commands/show_plan.py benchbox/cli/commands/compare_plans.py benchbox/cli/commands/plan_history.py benchbox/cli/commands/shell.py benchbox/core/runner/runner.py benchbox/core/tpcds/generator/manager.py benchbox/core/tpcds/generator/streaming.py benchbox/core/tpcds/generator/filesystem.py benchbox/core/tpcds/generator/runner.py benchbox/core/tpcds/benchmark/runner.py benchbox/core/results/display.py benchbox/core/config_utils.py benchbox/mcp/tools/benchmark.py", "description": "No direct runtime print leaks remain in CLI/core runtime modules", "expected": "no matches", "item_id": "centralize-output-control-and-eliminate-print-leaks", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create canonical output policy and API in a central module", "wid": "w1"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full verification suite and close remaining output-control gaps", "wid": "w10"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate CLI commands (results/report/show-plan/compare-plans/plan-history) from Console() to shared quiet-aware console", "wid": "w2"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Normalize shell command output by removing raw print() paths and using centralized console helpers", "wid": "w3"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace raw print() in core runner/config/results display modules with centralized output/logging", "wid": "w4"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace raw print() in TPC-DS generator and benchmark runtime modules with centralized output/logging", "wid": "w5"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w1", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wrap MCP benchmark execution/generation entrypoints with stdout/stderr suppression boundaries", "wid": "w6"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add static output-leak enforcement checks for runtime modules", "wid": "w7"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand CLI quiet-mode tests to cover consolidated output channels", "wid": "w8"}, {"evidence": null, "item_id": "centralize-output-control-and-eliminate-print-leaks", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add MCP regression tests asserting no non-JSON stdout leakage in benchmark tool paths", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "chromium-blocking-suite-not-in-required-checks", "id": 12, "reason": "Requires a repo admin (gh api PUT on the ruleset); an agent PR cannot change branch protection. Only needed if w1 chooses option (b) over the aggregate-job approach.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Sub-step of the parent item's own still-open scope; already tracked there, only needed if that item's w1 picks option (b). (tracked by chromium-blocking-suite-not-in-required-checks)", "summary": "If wiring via the ruleset (option b): apply the develop ruleset required_status_checks change to include the Chromium gate."}], "deps": [], "description": "Surfaced during the funding-chip work: PR #1091's \"Chromium (full suite,\nblocking)\" job concluded `failure` at 12:24, and the PR squash-auto-merged at\n12:29 anyway, landing a red browser suite on develop. The job name advertises\n\"blocking\" but it is not wired into the required merge gate.\n\nEvidence (read-only, captured 2026-07-10):\n - The develop ruleset `develop-squash-only` (id 15611785) has exactly ONE\n required status check: `ci-required-result`\n (`gh api repos/joeharris76/BenchBox/rulesets/15611785`).\n - `ci-required-result` (.github/workflows/pr.yml:949) aggregates only:\n ci-paths, content-guard, code-lint, code-test, correctness-gate,\n plan-capture-gate, medium-test, explorer-tokens, audit-sha, package-smoke,\n dependency-audit, parity-check.\n - The browser jobs \u2014 \"Chromium (full suite, blocking)\", WebKit @smoke,\n Firefox @smoke \u2014 live in a SEPARATE workflow\n (.github/workflows/results-explorer-browser.yml) and are NOT in that\n aggregate's `needs:` list. So `ci-required-result` can be green while\n Chromium is red, and auto-merge proceeds.\n\nScope note: WebKit and Firefox @smoke are INTENTIONALLY advisory today\n(`continue-on-error: true`), tracked separately by\ngraduate-browser-smoke-to-blocking-gates. This item is specifically about the\nChromium FULL suite, which is named \"blocking\" and is meant to gate merges but\ndoes not. Shared root cause with auto-merge-review-gate-admin-enforcement (the\ndevelop ruleset is under-configured), but a distinct remediation: that item\ntargets require_code_owner_review for soundness paths and deliberately\nexcludes status-check wiring; this item targets the required status-check set.\n\nCaveat to design around: the browser workflow is path-gated, so it does not\nrun on every PR. Whatever wiring is chosen must make a code-only PR that does\nnot touch explorer paths still able to merge (the aggregate must treat a\ncorrectly-skipped Chromium job as pass, not pending/fail) \u2014 mirror the\nexisting `ci-required-result` skip-handling for path-gated jobs rather than\nhard-requiring the raw job in the ruleset.\n", "id": "chromium-blocking-suite-not-in-required-checks", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "planning", "title": "Wire the Chromium full-suite e2e into the required merge gate \u2014 its job says 'blocking' but nothing enforces it", "verifications": [], "work": [{"evidence": null, "item_id": "chromium-blocking-suite-not-in-required-checks", "needs": [], "notes": "Re-read the develop ruleset required_status_checks and ci-required-result's\nneeds list. Decide between: (a) add the Chromium full-suite job to\nci-required-result's needs + aggregation with correct skip-as-pass\nhandling (keeps one roll-up check in the ruleset \u2014 preferred, matches the\nexisting pattern); or (b) add the Chromium context directly to the ruleset\nrequired_status_checks (admin action, and must handle path-skip). Record\nthe choice and why in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Confirm the gap live and decide the wiring point (aggregate job vs ruleset required-checks)", "wid": "w1"}, {"evidence": null, "item_id": "chromium-blocking-suite-not-in-required-checks", "needs": ["w1"], "notes": "If (a): extend .github/workflows/pr.yml ci-required-result to consume the\nChromium full-suite result across workflows (workflow_run or a reusable\njob), treating 'skipped' (path-gated off) as pass and 'failure' as fail.\nIf (b): the ruleset PUT is a deferred admin action (see deferred).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Implement the chosen wiring so a red Chromium full suite blocks merge", "wid": "w2"}, {"evidence": null, "item_id": "chromium-blocking-suite-not-in-required-checks", "needs": ["w2"], "notes": "Add a drift check (model on the existing pr.yml gh-api ruleset/aggregate\ndrift steps) that fails if the Chromium full suite ever falls out of the\nrequired set. Verify with a throwaway PR that forces a Chromium failure and\nconfirm it cannot merge, and that a non-explorer code-only PR still can.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Regression/drift guard + verify end-to-end", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:51Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #223 of platform-live-integration-tests.\nDeferred: CI pipeline for Docker-based integration tests\nReason deferred: Requires GitHub Actions with Docker compose support (services or docker compose in workflow). Design after local Docker tests are proven stable. Could use scheduled runs (nightly) rather than per-PR.", "id": "ci-pipeline-for-docker-based-integration-tests", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "CI pipeline for Docker-based integration tests", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed - All circular import issues resolved and validation added\"\n- \"**Description**: Several files contained circular imports within the benchbox package\"\n- \"**Completed Tasks**:\"\n- \"**Files Affected**:\"\n- \"**Final Implementation**:\"\n- \"**Impact**: \u2705 **COMPLETED** - Eliminated potential import issues, improved code maintainability, and added automated validation to prevent future circular import problems\"", "id": "circular-import-issues-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Circular Import Issues \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (COMPLETED)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Successfully implemented comprehensive alpha software warnings and disclaimers to set appropriate user expectations for the experimental nature of BenchBox, transforming marketing-heavy tone into realistic communication.", "id": "clarify-alpha-status-and-set-user-expectations", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Clarify Alpha Status and Set User Expectations", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation and Developer Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "\"Added the `cloudstorage` extra, centralized dependency metadata, refreshed adapter errors, and documented an installation matrix so optional platform installs are predictable.\"", "id": "clarify-optional-dependency-installation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Clarify Optional Dependency Installation", "verifications": [], "work": [], "worktree": "docs-experience"} -{"anti_patterns": [{"INSTEAD": "file a separate TODO if copy buttons are wanted later", "dont": "touch `docs/conf.py` extensions list", "why": "the goal is removal, not feature work"}], "approach": "Two-line removal in pyproject.toml. Verify by building the docs.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "clean-up-unused-sphinx-dev-deps", "id": 115, "reason": "Out of scope for this cleanup. If wanted, file a separate TODO for the docs feature add.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: sphinx-copybutton wiring explicitly framed as optional/out-of-scope ('if wanted, file separately'); no signal it's wanted.", "summary": "Wire up sphinx-copybutton if copy buttons are desired"}], "deps": [], "description": "Two Sphinx-related deps are declared but inert:\n\n- `sphinx-rtd-theme>=3.1.0` (`pyproject.toml` line 487) - `docs/conf.py`\n sets `html_theme = \"furo\"`. The rtd theme is never loaded. Not\n referenced anywhere in repo source.\n- `sphinx-copybutton>=0.5.2` (`pyproject.toml` line 491) - `docs/conf.py`\n extensions list does **not** include `\"sphinx_copybutton\"`. The\n package is installed but Sphinx will not activate it.\n\nTwo valid resolutions for each:\n A. Drop the declaration (removes installed-but-unused weight).\n B. Wire it up in `docs/conf.py` if the feature is actually wanted.\n\nRecommendation: drop both. If a future contributor wants copy buttons\non code blocks, they can re-add the dep + extensions entry as a single\nreviewable change.\n\nSource for this finding: `docs/development/dependency-inventory.md` (F5\nand F6), produced by `audit-imported-dependencies-for-elimination`.", "id": "clean-up-unused-sphinx-dev-deps", "preserves": ["Docs build (`make docs` or equivalent) succeeds after the change", "Existing `extensions` list in `docs/conf.py` is unchanged (no addition or removal)", "`docs/conf.py` continues to use `html_theme = 'furo'`"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Clean up unused Sphinx dev deps (sphinx-rtd-theme, sphinx-copybutton)", "verifications": [{"command": "uv run -- python -c \"import pathlib; t = pathlib.Path('pyproject.toml').read_text(); assert 'sphinx-rtd-theme' not in t, 'sphinx-rtd-theme still present'; assert 'sphinx-copybutton' not in t, 'sphinx-copybutton still present'; print('OK: both removed')\"", "description": "Both deps absent from pyproject", "expected": "OK: both removed", "item_id": "clean-up-unused-sphinx-dev-deps", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"import importlib; m = importlib.import_module('sphinx'); print('sphinx importable:', m.__version__)\"", "description": "Docs build still succeeds (replace with project docs target)", "expected": "sphinx importable: ", "item_id": "clean-up-unused-sphinx-dev-deps", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm sphinx-rtd-theme is not referenced in docs/, examples/, or any conf.py: `grep -rn 'sphinx_rtd_theme\\|rtd_theme' docs examples`.", "wid": "w1"}, {"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm sphinx-copybutton is not in `docs/conf.py:extensions`: read the file and verify.", "wid": "w2"}, {"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove both lines from `[dependency-groups] dev` in pyproject.toml. Refresh uv.lock with `uv lock`.", "wid": "w3"}, {"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build docs locally to confirm no regression: `make docs` (or the project's docs build target).", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "always use git mv even when moving to _archive/", "dont": "archive or move PROJECT_TODO.yaml and PROJECT_DONE.yaml without git mv", "why": "git rm + git add would lose file history"}, {"INSTEAD": "(not specified)", "dont": "attempt to condense DATAFRAME_SUPPORT_MASTER.md into a short docs page without the research gate confirming it is still actively consulted", "why": "if it's unused, plain archiving is the right call"}, {"INSTEAD": "the research gate (w1) must answer this first", "dont": "bulk-move _project/specs/ without reading them first", "why": "specs may be internal-only design notes that would be inappropriate to publish in docs/"}, {"INSTEAD": "moving them would break every command in CLAUDE.md and AGENTS.md", "dont": "move _project/scripts/ or _project/config/", "why": "they form an intentionally isolated Python environment with `uv run --project _project/scripts` as the canonical invocation"}, {"INSTEAD": "stale links in historical docs are acceptable", "dont": "update reference links inside _project/handoffs/ files", "why": "those are historical records and patching them wastes time with no benefit"}, {"INSTEAD": "add each path explicitly", "dont": "use `git add -A` or `git add .`", "why": "this repo has strict per-file staging policy and generated files could accidentally be staged"}], "approach": "Work in parallel where possible: w2 (archive clutter) and w3 (move ADRs) and w4 (move analysis docs)\nhave no mutual dependencies and can be done in any order or simultaneously in one commit.\nw1 (research gate) must complete before w5. w6 (stale reference scan) must run after all moves.\n\nRule of thumb for the research gate: if a file's last git commit is >90 days old AND no open\nTODO item explicitly depends on it, treat it as archivable. If a file is referenced by grep hits\noutside _project/, it is misplaced and should move to docs/.\n\nFor reference updates: update live docs (docs/, results-data/, AGENTS.md) but do NOT update\n_project/handoffs/ - those are historical records where broken internal links are acceptable.\n\nStage files explicitly by path (never `git add -A`). Use `git mv` not `cp` + `rm` for all\nfile relocations to preserve history. Bundle w2 + w3 + w4 into a single commit if they have\nno reference-update entanglements; otherwise separate for clarity.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cleanup-project-scratch-directory", "id": 116, "reason": "A grep-based CI step could catch future cases where docs are created inside\n_project/ and linked from outside. Deferred because it requires writing a\ncheck_project_references.py script and wiring it into the Makefile - small\nenough to add as a follow-up TODO once this cleanup is done.\n", "resolution": "promoted", "resolved_item": "automate-stale-reference-detection-via-ci-lint", "resolved_reason": null, "summary": "Automate stale-reference detection via CI lint"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cleanup-project-scratch-directory", "id": 117, "reason": "The 67 KB master doc is dense. Breaking it into per-topic pages in docs/\nwould improve discoverability, but requires careful editing to avoid losing\ncontext. Deferred until the research gate confirms the doc is still actively\nconsulted (if it's not, archiving is sufficient).\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Same DONE item's research gate found the doc not actively consulted and archived it instead; compress/split condition not met, file no longer present.", "summary": "Compress or split DATAFRAME_SUPPORT_MASTER.md into topic-scoped reference docs"}], "deps": [], "description": "The `_project/` directory was designed as a scratch/transient working area.\nA 2026-04-15 audit revealed it now contains a mix of intentional permanent\nassets and misplaced canonical ones:\n\nCONFIRMED CLUTTER (unambiguous - archive or delete):\n - 3 superseded monolithic TODO YAMLs totalling ~1.1 MB\n (PROJECT_TODO.yaml, PROJECT_DONE.yaml, PROJECT_TODO.yaml.before-fix)\n - 10 MB coverage.json (generated artifact, should not be tracked in git)\n - 2 timing report text files (generated output, reproducible)\n - 1 POC script (dataframe_poc.py, superseded by production code)\n - 1 one-off analysis artifact (reclassification.yaml)\n\nCONFIRMED MISPLACED (should live in docs/):\n - _project/adr/ - 2 ADRs belong in docs/development/adr/\n - _project/analysis/hosted-results-contract.md - live contract referenced by\n results-data/README.md and docs/operations/results-phase-2-runbook.md\n - _project/analysis/threat-model.md - canonical security doc\n - _project/analysis/benchbox-results-platform-strategy.md - active strategy doc\n\nOPEN QUESTIONS requiring a research gate (see w1):\n - _project/specs/ (30+ files, 1.7 MB) - authoritative or internal-only?\n - _project/DESIGN-integrated-task-management.md - stays near the TODO system\n it describes, or moves to docs/development/?\n - _project/PLATFORM_INTEGRATION_ROADMAP.md, RELEASE_PRIORITIZATION_v1.md,\n PRE_RELEASE_CHECKLIST.md - active agent reference artifacts or publishable docs?\n - _project/DATAFRAME_SUPPORT_MASTER.md + format-converter/query-plan docs -\n live architecture references or completed-milestone archives?\n - _project/handoffs/ - active context for in-flight work or historical records?\n\nINTENTIONALLY STAYS in _project/ (do not touch):\n - _project/scripts/ - isolated pyproject.toml prevents dependency leakage into\n the main wheel; invocation `uv run --project _project/scripts` is canonical\n - _project/config/ - config for the scripts above; co-located by design\n - _project/TODO_SCHEMA.yaml, TODO_ENTRY_TEMPLATE.yaml - used by validate_todo.py\n - _project/TODO/, _project/DONE/ - the active project management system\n - _project/scratch/, _project/notes/, _project/_archive/, _project/_trash/", "id": "cleanup-project-scratch-directory", "preserves": ["All git mv operations must use `git mv`, not shell cp+rm, to preserve file history", "_project/TODO/ and _project/DONE/ directory structure - the active project management system must remain intact", "_project/TODO_SCHEMA.yaml and _project/TODO_ENTRY_TEMPLATE.yaml paths - referenced in AGENTS.md and by validate_todo.py", "_project/config/timing_wall_clock_allowlist.json path - referenced verbatim in AGENTS.md line 64 and by timing_policy_check.py", "_project/scripts/ invocation path - `uv run --project _project/scripts` must continue to work; do not move any file under _project/scripts/"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/DONE/"}, {"kind": "do_not_modify", "path_glob": "_project/TODO/"}, {"kind": "do_not_modify", "path_glob": "_project/TODO_ENTRY_TEMPLATE.yaml"}, {"kind": "do_not_modify", "path_glob": "_project/TODO_SCHEMA.yaml"}, {"kind": "do_not_modify", "path_glob": "_project/config/"}, {"kind": "do_not_modify", "path_glob": "_project/handoffs/"}, {"kind": "do_not_modify", "path_glob": "_project/notes/"}, {"kind": "do_not_modify", "path_glob": "_project/scratch/"}, {"kind": "do_not_modify", "path_glob": "_project/scripts/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": ".gitignore"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_blog/building-benchbox/outlines/06-extracting-textcharts.md"}, {"kind": "only_modify", "path_glob": "_project/DATAFRAME_CONSOLIDATION_SUMMARY.md"}, {"kind": "only_modify", "path_glob": "_project/DATAFRAME_SUPPORT_MASTER.md"}, {"kind": "only_modify", "path_glob": "_project/DESIGN-integrated-task-management.md"}, {"kind": "only_modify", "path_glob": "_project/FORMAT_CONVERTER_COMPLETION_SUMMARY.md"}, {"kind": "only_modify", "path_glob": "_project/FORMAT_CONVERTER_IMPLEMENTATION_STATUS.md"}, {"kind": "only_modify", "path_glob": "_project/PLATFORM_INTEGRATION_ROADMAP.md"}, {"kind": "only_modify", "path_glob": "_project/PRE_RELEASE_CHECKLIST.md"}, {"kind": "only_modify", "path_glob": "_project/PROJECT_DONE.yaml"}, {"kind": "only_modify", "path_glob": "_project/PROJECT_TODO.yaml"}, {"kind": "only_modify", "path_glob": "_project/PROJECT_TODO.yaml.before-fix"}, {"kind": "only_modify", "path_glob": "_project/RELEASE_PRIORITIZATION_v1.md"}, {"kind": "only_modify", "path_glob": "_project/_archive/"}, {"kind": "only_modify", "path_glob": "_project/adr/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "_project/coverage.json"}, {"kind": "only_modify", "path_glob": "_project/dataframe_poc.py"}, {"kind": "only_modify", "path_glob": "_project/format-converter-adversarial-review-and-fixes.md"}, {"kind": "only_modify", "path_glob": "_project/handoffs/"}, {"kind": "only_modify", "path_glob": "_project/query_plan_capture_overview.md"}, {"kind": "only_modify", "path_glob": "_project/query_plan_workstream_summary.md"}, {"kind": "only_modify", "path_glob": "_project/reclassification.yaml"}, {"kind": "only_modify", "path_glob": "_project/specs/"}, {"kind": "only_modify", "path_glob": "_project/timing_report.txt"}, {"kind": "only_modify", "path_glob": "_project/timing_report_all.txt"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/"}, {"kind": "only_modify", "path_glob": "docs/development/adr/"}, {"kind": "only_modify", "path_glob": "docs/development/benchbox-results-platform-strategy.md"}, {"kind": "only_modify", "path_glob": "docs/operations/results-phase-2-runbook.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "results-data/README.md"}], "state": "done", "title": "Relocate canonical assets out of _project/ scratch area to proper homes", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py stats", "description": "TODO system still works after all moves", "expected": "Item counts match pre-cleanup totals; no errors about missing schema or config files", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -r '_project/adr' . --include='*.md' --include='*.rst' --include='*.yaml'", "description": "No stale references to moved ADR paths", "expected": "Zero matches", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -r '_project/analysis/hosted-results-contract\\|_project/analysis/threat-model\\|_project/analysis/benchbox-results-platform-strategy' . --include='*.md'", "description": "No stale references to moved analysis docs", "expected": "Zero matches", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 3}, {"command": "ls _project/PROJECT_TODO.yaml _project/PROJECT_DONE.yaml 2>&1", "description": "Archived monolithic YAMLs are gone from _project/ root", "expected": "ls: cannot access (files no longer at root)", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 4}, {"command": "ls _project/coverage.json 2>&1", "description": "coverage.json absent from _project/ root", "expected": "ls: cannot access", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests still pass", "expected": "All tests pass, no failures", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph validation passes", "expected": "No cycles or dangling dependency edges", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "VERDICT Q1 - _project/specs/ \u2192 OPTION C: ARCHIVE. docs/benchmarks/ and\ndocs/platforms/ are comprehensive (25+ and 50+ files respectively) and cover\nall the same benchmarks and platforms. specs/ has zero external references\noutside _project/. Decision: git mv _project/specs/ _project/_archive/specs/\n\nVERDICT Q2 - DESIGN-integrated-task-management.md \u2192 OPTION B: MOVE.\nZero external references found (grep confirmed no hits outside the TODO file).\nStandalone architecture doc describing task system design. Move to\ndocs/development/task-management-design.md.\n\nVERDICT Q3 - Roadmap/quality docs \u2192 OPTION C: ARCHIVE. PLATFORM_INTEGRATION_ROADMAP.md\nlast commit Nov 2024. PRE_RELEASE_CHECKLIST.md and RELEASE_PRIORITIZATION_v1.md only\nreferenced in completed DONE items (PROJECT_DONE.yaml, _project/DONE/). All describe\ncompleted pre-release planning. Archive to _project/_archive/.\n\nVERDICT Q4 - DATAFRAME_SUPPORT_MASTER.md + related \u2192 OPTION A: ARCHIVE. No open\nTODO depends on these files. Only self-referential _project/ links. No active\nimplementation work references them. Archive to _project/_archive/.\n\nVERDICT Q5 - _project/handoffs/ \u2192 OPTION B: PARTIAL ARCHIVE. Most recent file\n(results-publishing-handoff-2026-04-04.md) was updated 2026-04-12 describing active\nPhase 2 results-publishing work (PR #53). Keep it active. Archive the other 3 older\nhandoffs to _project/_archive/handoffs/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research gate: read and classify the 5 ambiguous asset groups", "wid": "w1"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "No research gate required - these are unambiguously clutter.\n\nARCHIVE with git mv (preserve history):\n git mv _project/PROJECT_TODO.yaml _project/_archive/PROJECT_TODO.yaml\n git mv _project/PROJECT_DONE.yaml _project/_archive/PROJECT_DONE.yaml\n git mv _project/PROJECT_TODO.yaml.before-fix _project/_archive/PROJECT_TODO.yaml.before-fix\n git mv _project/timing_report.txt _project/_archive/timing_report.txt\n git mv _project/timing_report_all.txt _project/_archive/timing_report_all.txt\n git mv _project/dataframe_poc.py _project/_archive/dataframe_poc.py\n git mv _project/reclassification.yaml _project/_archive/reclassification.yaml\n\nDELETE (generated, reproducible, not worth archiving):\n git rm _project/coverage.json\n\nGITIGNORE: check if _project/coverage.json is in .gitignore.\n grep \"coverage.json\" .gitignore\n If absent, add the line: _project/coverage.json\n\nPOST-MOVE VERIFICATION:\n uv run --project _project/scripts -- python _project/scripts/todo_cli.py stats\n Expected: same item counts as before - confirms no TODO path breakage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Archive superseded monolithic YAMLs and delete generated artifacts", "wid": "w2"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "Source files (confirmed no production code dependency on these paths):\n _project/adr/0001-unified-platform-registration.md\n _project/adr/adr-textcharts-extraction.md\n\nSteps:\n mkdir -p docs/development/adr/\n git mv _project/adr/0001-unified-platform-registration.md docs/development/adr/0001-unified-platform-registration.md\n git mv _project/adr/adr-textcharts-extraction.md docs/development/adr/adr-textcharts-extraction.md\n\nUPDATE REFERENCES (exactly one external reference confirmed):\n File: _blog/building-benchbox/outlines/06-extracting-textcharts.md\n Search: grep -n \"_project/adr\" _blog/building-benchbox/outlines/06-extracting-textcharts.md\n Update the path from `_project/adr/adr-textcharts-extraction.md`\n to `docs/development/adr/adr-textcharts-extraction.md`.\n\nAlso scan for any other references:\n grep -r \"_project/adr\" . --include=\"*.md\" --include=\"*.rst\" --include=\"*.yaml\"\n Expected after fix: zero matches.\n\nConsider adding docs/development/adr/README.md (one line: \"Architecture Decision Records\")\nto make the directory self-documenting if it doesn't already have one.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move ADRs from _project/adr/ to docs/development/adr/", "wid": "w3"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "These 3 files are actively linked from outside _project/ and belong in docs/.\nConfirmed external references exist to all three.\n\nMOVE A - hosted-results-contract.md:\n git mv _project/analysis/hosted-results-contract.md docs/reference/hosted-results-contract.md\n Update references:\n grep -rn \"hosted-results-contract\" . --include=\"*.md\" --include=\"*.rst\"\n Files confirmed to reference it:\n - results-data/README.md\n - docs/operations/results-phase-2-runbook.md\n - _project/analysis/benchbox-results-platform-strategy.md (update in-place after move B)\n - _project/handoffs/ (do NOT update - historical records, broken links acceptable)\n\nMOVE B - threat-model.md:\n git mv _project/analysis/threat-model.md docs/reference/threat-model.md\n Update references:\n grep -rn \"threat-model\" . --include=\"*.md\" --include=\"*.rst\"\n Update any hits outside _project/handoffs/\n\nMOVE C - benchbox-results-platform-strategy.md:\n git mv _project/analysis/benchbox-results-platform-strategy.md docs/development/benchbox-results-platform-strategy.md\n Update references:\n grep -rn \"benchbox-results-platform-strategy\" . --include=\"*.md\"\n Update any hits outside _project/handoffs/\n\nFINAL VERIFICATION:\n grep -r \"_project/analysis/hosted-results-contract\\|_project/analysis/threat-model\\|_project/analysis/benchbox-results-platform-strategy\" \\\n . --include=\"*.md\" --include=\"*.rst\"\n Expected: zero matches (handoff internal links excluded from this check).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move 3 canonical analysis docs to proper docs/ locations", "wid": "w4"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": ["w1"], "notes": "Apply one of the classified actions per research gate outcome. The exact moves\ndepend entirely on w1 findings - the options are specified here:\n\nFOR _project/specs/ - pick ONE:\n Option A (authoritative content not in docs/):\n For each unique spec file: git mv _project/specs/{name}.md docs/benchmarks/{name}.md\n OR git mv _project/specs/platforms/{name}.md docs/platforms/{name}.md\n Archive anything already covered: git mv _project/specs/{dup}.md _project/_archive/specs/{dup}.md\n Option B (internal design notes only):\n Write _project/specs/README.md: \"Internal design notes - not published. See docs/ for authoritative content.\"\n No moves needed.\n Option C (superseded by public docs):\n git mv _project/specs/ _project/_archive/specs/\n\nFOR _project/DESIGN-integrated-task-management.md - pick ONE:\n Option A (agent navigation reference \u2192 stay in _project/): no action.\n Option B (publishable architecture doc \u2192 move):\n git mv _project/DESIGN-integrated-task-management.md docs/development/task-management-design.md\n grep -rn \"DESIGN-integrated-task-management\" . --include=\"*.md\" --include=\"*.py\" --include=\"*.yaml\"\n Update any hits.\n\nFOR roadmap/quality docs (PLATFORM_INTEGRATION_ROADMAP.md, RELEASE_PRIORITIZATION_v1.md, PRE_RELEASE_CHECKLIST.md) - pick ONE:\n Option A (agent-facing action lists \u2192 stay in _project/): no action.\n Option B (publishable \u2192 merge/move to docs/development/):\n git mv each file to appropriate docs/ path.\n Option C (completed milestones \u2192 archive):\n git mv each file to _project/_archive/\n\nFOR _project/DATAFRAME_SUPPORT_MASTER.md + FORMAT_CONVERTER_*.md + query_plan_*.md - pick ONE per file:\n Option A (not consulted in >90 days, no open TODO dependency): git mv to _project/_archive/\n Option B (active reference \u2192 condense): Do NOT bulk-move the 67 KB master doc.\n Create a condensed docs/development/dataframe-support-architecture.md (key decisions only)\n and archive the master doc. This is non-trivial - only do this if clearly warranted.\n\nFOR _project/handoffs/ - pick ONE:\n Option A (all historical): git mv _project/handoffs/ _project/_archive/handoffs/\n Option B (most recent is active): git mv older handoffs individually to _project/_archive/handoffs/\n Leave the most recent 1-2 in _project/handoffs/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Execute research-gate decisions for specs, design docs, roadmaps, and handoffs", "wid": "w5"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": ["w3", "w4", "w5"], "notes": "Scan for any paths moved in w3, w4, w5 that still have stale references:\n\n grep -r \"_project/adr\" . --include=\"*.md\" --include=\"*.rst\" --include=\"*.py\" --include=\"*.yaml\"\n grep -r \"_project/analysis/hosted-results-contract\\|_project/analysis/threat-model\\|_project/analysis/benchbox-results-platform-strategy\" \\\n . --include=\"*.md\" --include=\"*.rst\"\n\nIf w5 moved specs/ or design docs, also run:\n grep -r \"_project/specs\\|_project/DESIGN-integrated-task-management\\|_project/PLATFORM_INTEGRATION_ROADMAP\\|_project/RELEASE_PRIORITIZATION\\|_project/PRE_RELEASE_CHECKLIST\" \\\n . --include=\"*.md\" --include=\"*.rst\" --include=\"*.yaml\"\n\nFix every match that falls outside _project/handoffs/ (historical docs - broken links there are acceptable).\nIf AGENTS.md references any moved file, update it. The timing_wall_clock_allowlist.json reference\nin AGENTS.md line 64 MUST NOT be changed (that file is not moving).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update AGENTS.md and any remaining stale cross-references", "wid": "w6"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": "Run full validation suite:\n uv run --project _project/scripts -- python _project/scripts/validate_todo.py --all\n uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph\n uv run --project _project/scripts -- python _project/scripts/generate_indexes.py\n\nRun fast tests to confirm nothing is broken:\n uv run -- python -m pytest -m fast -q\n\nCommit with explicit staging - DO NOT use git add -A:\n git add \n git commit -m \"chore(cleanup): relocate canonical assets out of _project/ scratch area\"\n\nCommit message should list the major moves: ADRs, analysis docs, archived YAMLs,\nand note which research-gate decisions were made.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Validate TODO system integrity and commit all changes", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "touch `docs/conf.py` extensions list", "instead": "file a separate TODO if copy buttons are wanted later", "why": "the goal is removal, not feature work"}], "approach": "Two-line removal in pyproject.toml. Verify by building the docs.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "clean-up-unused-sphinx-dev-deps", "id": 115, "reason": "Out of scope for this cleanup. If wanted, file a separate TODO for the docs feature add.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: sphinx-copybutton wiring explicitly framed as optional/out-of-scope ('if wanted, file separately'); no signal it's wanted.", "summary": "Wire up sphinx-copybutton if copy buttons are desired"}], "deps": [], "description": "Two Sphinx-related deps are declared but inert:\n\n- `sphinx-rtd-theme>=3.1.0` (`pyproject.toml` line 487) - `docs/conf.py`\n sets `html_theme = \"furo\"`. The rtd theme is never loaded. Not\n referenced anywhere in repo source.\n- `sphinx-copybutton>=0.5.2` (`pyproject.toml` line 491) - `docs/conf.py`\n extensions list does **not** include `\"sphinx_copybutton\"`. The\n package is installed but Sphinx will not activate it.\n\nTwo valid resolutions for each:\n A. Drop the declaration (removes installed-but-unused weight).\n B. Wire it up in `docs/conf.py` if the feature is actually wanted.\n\nRecommendation: drop both. If a future contributor wants copy buttons\non code blocks, they can re-add the dep + extensions entry as a single\nreviewable change.\n\nSource for this finding: `docs/development/dependency-inventory.md` (F5\nand F6), produced by `audit-imported-dependencies-for-elimination`.", "id": "clean-up-unused-sphinx-dev-deps", "preserves": ["Docs build (`make docs` or equivalent) succeeds after the change", "Existing `extensions` list in `docs/conf.py` is unchanged (no addition or removal)", "`docs/conf.py` continues to use `html_theme = 'furo'`"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Clean up unused Sphinx dev deps (sphinx-rtd-theme, sphinx-copybutton)", "verifications": [{"command": "uv run -- python -c \"import pathlib; t = pathlib.Path('pyproject.toml').read_text(); assert 'sphinx-rtd-theme' not in t, 'sphinx-rtd-theme still present'; assert 'sphinx-copybutton' not in t, 'sphinx-copybutton still present'; print('OK: both removed')\"", "description": "Both deps absent from pyproject", "expected": "OK: both removed", "item_id": "clean-up-unused-sphinx-dev-deps", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"import importlib; m = importlib.import_module('sphinx'); print('sphinx importable:', m.__version__)\"", "description": "Docs build still succeeds (replace with project docs target)", "expected": "sphinx importable: ", "item_id": "clean-up-unused-sphinx-dev-deps", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm sphinx-rtd-theme is not referenced in docs/, examples/, or any conf.py: `grep -rn 'sphinx_rtd_theme\\|rtd_theme' docs examples`.", "wid": "w1"}, {"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm sphinx-copybutton is not in `docs/conf.py:extensions`: read the file and verify.", "wid": "w2"}, {"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove both lines from `[dependency-groups] dev` in pyproject.toml. Refresh uv.lock with `uv lock`.", "wid": "w3"}, {"evidence": null, "item_id": "clean-up-unused-sphinx-dev-deps", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build docs locally to confirm no regression: `make docs` (or the project's docs build target).", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "archive or move PROJECT_TODO.yaml and PROJECT_DONE.yaml without git mv", "instead": "always use git mv even when moving to _archive/", "why": "git rm + git add would lose file history"}, {"dont": "attempt to condense DATAFRAME_SUPPORT_MASTER.md into a short docs page without the research gate confirming it is still actively consulted", "instead": "(not specified)", "why": "if it's unused, plain archiving is the right call"}, {"dont": "bulk-move _project/specs/ without reading them first", "instead": "the research gate (w1) must answer this first", "why": "specs may be internal-only design notes that would be inappropriate to publish in docs/"}, {"dont": "move _project/scripts/ or _project/config/", "instead": "moving them would break every command in CLAUDE.md and AGENTS.md", "why": "they form an intentionally isolated Python environment with `uv run --project _project/scripts` as the canonical invocation"}, {"dont": "update reference links inside _project/handoffs/ files", "instead": "stale links in historical docs are acceptable", "why": "those are historical records and patching them wastes time with no benefit"}, {"dont": "use `git add -A` or `git add .`", "instead": "add each path explicitly", "why": "this repo has strict per-file staging policy and generated files could accidentally be staged"}], "approach": "Work in parallel where possible: w2 (archive clutter) and w3 (move ADRs) and w4 (move analysis docs)\nhave no mutual dependencies and can be done in any order or simultaneously in one commit.\nw1 (research gate) must complete before w5. w6 (stale reference scan) must run after all moves.\n\nRule of thumb for the research gate: if a file's last git commit is >90 days old AND no open\nTODO item explicitly depends on it, treat it as archivable. If a file is referenced by grep hits\noutside _project/, it is misplaced and should move to docs/.\n\nFor reference updates: update live docs (docs/, results-data/, AGENTS.md) but do NOT update\n_project/handoffs/ - those are historical records where broken internal links are acceptable.\n\nStage files explicitly by path (never `git add -A`). Use `git mv` not `cp` + `rm` for all\nfile relocations to preserve history. Bundle w2 + w3 + w4 into a single commit if they have\nno reference-update entanglements; otherwise separate for clarity.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cleanup-project-scratch-directory", "id": 116, "reason": "A grep-based CI step could catch future cases where docs are created inside\n_project/ and linked from outside. Deferred because it requires writing a\ncheck_project_references.py script and wiring it into the Makefile - small\nenough to add as a follow-up TODO once this cleanup is done.\n", "resolution": "promoted", "resolved_item": "automate-stale-reference-detection-via-ci-lint", "resolved_reason": null, "summary": "Automate stale-reference detection via CI lint"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cleanup-project-scratch-directory", "id": 117, "reason": "The 67 KB master doc is dense. Breaking it into per-topic pages in docs/\nwould improve discoverability, but requires careful editing to avoid losing\ncontext. Deferred until the research gate confirms the doc is still actively\nconsulted (if it's not, archiving is sufficient).\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Same DONE item's research gate found the doc not actively consulted and archived it instead; compress/split condition not met, file no longer present.", "summary": "Compress or split DATAFRAME_SUPPORT_MASTER.md into topic-scoped reference docs"}], "deps": [], "description": "The `_project/` directory was designed as a scratch/transient working area.\nA 2026-04-15 audit revealed it now contains a mix of intentional permanent\nassets and misplaced canonical ones:\n\nCONFIRMED CLUTTER (unambiguous - archive or delete):\n - 3 superseded monolithic TODO YAMLs totalling ~1.1 MB\n (PROJECT_TODO.yaml, PROJECT_DONE.yaml, PROJECT_TODO.yaml.before-fix)\n - 10 MB coverage.json (generated artifact, should not be tracked in git)\n - 2 timing report text files (generated output, reproducible)\n - 1 POC script (dataframe_poc.py, superseded by production code)\n - 1 one-off analysis artifact (reclassification.yaml)\n\nCONFIRMED MISPLACED (should live in docs/):\n - _project/adr/ - 2 ADRs belong in docs/development/adr/\n - _project/analysis/hosted-results-contract.md - live contract referenced by\n results-data/README.md and docs/operations/results-phase-2-runbook.md\n - _project/analysis/threat-model.md - canonical security doc\n - _project/analysis/benchbox-results-platform-strategy.md - active strategy doc\n\nOPEN QUESTIONS requiring a research gate (see w1):\n - _project/specs/ (30+ files, 1.7 MB) - authoritative or internal-only?\n - _project/DESIGN-integrated-task-management.md - stays near the TODO system\n it describes, or moves to docs/development/?\n - _project/PLATFORM_INTEGRATION_ROADMAP.md, RELEASE_PRIORITIZATION_v1.md,\n PRE_RELEASE_CHECKLIST.md - active agent reference artifacts or publishable docs?\n - _project/DATAFRAME_SUPPORT_MASTER.md + format-converter/query-plan docs -\n live architecture references or completed-milestone archives?\n - _project/handoffs/ - active context for in-flight work or historical records?\n\nINTENTIONALLY STAYS in _project/ (do not touch):\n - _project/scripts/ - isolated pyproject.toml prevents dependency leakage into\n the main wheel; invocation `uv run --project _project/scripts` is canonical\n - _project/config/ - config for the scripts above; co-located by design\n - _project/TODO_SCHEMA.yaml, TODO_ENTRY_TEMPLATE.yaml - used by validate_todo.py\n - _project/TODO/, _project/DONE/ - the active project management system\n - _project/scratch/, _project/notes/, _project/_archive/, _project/_trash/", "id": "cleanup-project-scratch-directory", "preserves": ["All git mv operations must use `git mv`, not shell cp+rm, to preserve file history", "_project/TODO/ and _project/DONE/ directory structure - the active project management system must remain intact", "_project/TODO_SCHEMA.yaml and _project/TODO_ENTRY_TEMPLATE.yaml paths - referenced in AGENTS.md and by validate_todo.py", "_project/config/timing_wall_clock_allowlist.json path - referenced verbatim in AGENTS.md line 64 and by timing_policy_check.py", "_project/scripts/ invocation path - `uv run --project _project/scripts` must continue to work; do not move any file under _project/scripts/"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/DONE/"}, {"kind": "do_not_modify", "path_glob": "_project/TODO/"}, {"kind": "do_not_modify", "path_glob": "_project/TODO_ENTRY_TEMPLATE.yaml"}, {"kind": "do_not_modify", "path_glob": "_project/TODO_SCHEMA.yaml"}, {"kind": "do_not_modify", "path_glob": "_project/config/"}, {"kind": "do_not_modify", "path_glob": "_project/handoffs/"}, {"kind": "do_not_modify", "path_glob": "_project/notes/"}, {"kind": "do_not_modify", "path_glob": "_project/scratch/"}, {"kind": "do_not_modify", "path_glob": "_project/scripts/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": ".gitignore"}, {"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_blog/building-benchbox/outlines/06-extracting-textcharts.md"}, {"kind": "only_modify", "path_glob": "_project/DATAFRAME_CONSOLIDATION_SUMMARY.md"}, {"kind": "only_modify", "path_glob": "_project/DATAFRAME_SUPPORT_MASTER.md"}, {"kind": "only_modify", "path_glob": "_project/DESIGN-integrated-task-management.md"}, {"kind": "only_modify", "path_glob": "_project/FORMAT_CONVERTER_COMPLETION_SUMMARY.md"}, {"kind": "only_modify", "path_glob": "_project/FORMAT_CONVERTER_IMPLEMENTATION_STATUS.md"}, {"kind": "only_modify", "path_glob": "_project/PLATFORM_INTEGRATION_ROADMAP.md"}, {"kind": "only_modify", "path_glob": "_project/PRE_RELEASE_CHECKLIST.md"}, {"kind": "only_modify", "path_glob": "_project/PROJECT_DONE.yaml"}, {"kind": "only_modify", "path_glob": "_project/PROJECT_TODO.yaml"}, {"kind": "only_modify", "path_glob": "_project/PROJECT_TODO.yaml.before-fix"}, {"kind": "only_modify", "path_glob": "_project/RELEASE_PRIORITIZATION_v1.md"}, {"kind": "only_modify", "path_glob": "_project/_archive/"}, {"kind": "only_modify", "path_glob": "_project/adr/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}, {"kind": "only_modify", "path_glob": "_project/coverage.json"}, {"kind": "only_modify", "path_glob": "_project/dataframe_poc.py"}, {"kind": "only_modify", "path_glob": "_project/format-converter-adversarial-review-and-fixes.md"}, {"kind": "only_modify", "path_glob": "_project/handoffs/"}, {"kind": "only_modify", "path_glob": "_project/query_plan_capture_overview.md"}, {"kind": "only_modify", "path_glob": "_project/query_plan_workstream_summary.md"}, {"kind": "only_modify", "path_glob": "_project/reclassification.yaml"}, {"kind": "only_modify", "path_glob": "_project/specs/"}, {"kind": "only_modify", "path_glob": "_project/timing_report.txt"}, {"kind": "only_modify", "path_glob": "_project/timing_report_all.txt"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/"}, {"kind": "only_modify", "path_glob": "docs/development/adr/"}, {"kind": "only_modify", "path_glob": "docs/development/benchbox-results-platform-strategy.md"}, {"kind": "only_modify", "path_glob": "docs/operations/results-phase-2-runbook.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "results-data/README.md"}], "state": "done", "title": "Relocate canonical assets out of _project/ scratch area to proper homes", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py stats", "description": "TODO system still works after all moves", "expected": "Item counts match pre-cleanup totals; no errors about missing schema or config files", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -r '_project/adr' . --include='*.md' --include='*.rst' --include='*.yaml'", "description": "No stale references to moved ADR paths", "expected": "Zero matches", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -r '_project/analysis/hosted-results-contract\\|_project/analysis/threat-model\\|_project/analysis/benchbox-results-platform-strategy' . --include='*.md'", "description": "No stale references to moved analysis docs", "expected": "Zero matches", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 3}, {"command": "ls _project/PROJECT_TODO.yaml _project/PROJECT_DONE.yaml 2>&1", "description": "Archived monolithic YAMLs are gone from _project/ root", "expected": "ls: cannot access (files no longer at root)", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 4}, {"command": "ls _project/coverage.json 2>&1", "description": "coverage.json absent from _project/ root", "expected": "ls: cannot access", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests still pass", "expected": "All tests pass, no failures", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph validation passes", "expected": "No cycles or dangling dependency edges", "item_id": "cleanup-project-scratch-directory", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "VERDICT Q1 - _project/specs/ \u2192 OPTION C: ARCHIVE. docs/benchmarks/ and\ndocs/platforms/ are comprehensive (25+ and 50+ files respectively) and cover\nall the same benchmarks and platforms. specs/ has zero external references\noutside _project/. Decision: git mv _project/specs/ _project/_archive/specs/\n\nVERDICT Q2 - DESIGN-integrated-task-management.md \u2192 OPTION B: MOVE.\nZero external references found (grep confirmed no hits outside the TODO file).\nStandalone architecture doc describing task system design. Move to\ndocs/development/task-management-design.md.\n\nVERDICT Q3 - Roadmap/quality docs \u2192 OPTION C: ARCHIVE. PLATFORM_INTEGRATION_ROADMAP.md\nlast commit Nov 2024. PRE_RELEASE_CHECKLIST.md and RELEASE_PRIORITIZATION_v1.md only\nreferenced in completed DONE items (PROJECT_DONE.yaml, _project/DONE/). All describe\ncompleted pre-release planning. Archive to _project/_archive/.\n\nVERDICT Q4 - DATAFRAME_SUPPORT_MASTER.md + related \u2192 OPTION A: ARCHIVE. No open\nTODO depends on these files. Only self-referential _project/ links. No active\nimplementation work references them. Archive to _project/_archive/.\n\nVERDICT Q5 - _project/handoffs/ \u2192 OPTION B: PARTIAL ARCHIVE. Most recent file\n(results-publishing-handoff-2026-04-04.md) was updated 2026-04-12 describing active\nPhase 2 results-publishing work (PR #53). Keep it active. Archive the other 3 older\nhandoffs to _project/_archive/handoffs/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research gate: read and classify the 5 ambiguous asset groups", "wid": "w1"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "No research gate required - these are unambiguously clutter.\n\nARCHIVE with git mv (preserve history):\n git mv _project/PROJECT_TODO.yaml _project/_archive/PROJECT_TODO.yaml\n git mv _project/PROJECT_DONE.yaml _project/_archive/PROJECT_DONE.yaml\n git mv _project/PROJECT_TODO.yaml.before-fix _project/_archive/PROJECT_TODO.yaml.before-fix\n git mv _project/timing_report.txt _project/_archive/timing_report.txt\n git mv _project/timing_report_all.txt _project/_archive/timing_report_all.txt\n git mv _project/dataframe_poc.py _project/_archive/dataframe_poc.py\n git mv _project/reclassification.yaml _project/_archive/reclassification.yaml\n\nDELETE (generated, reproducible, not worth archiving):\n git rm _project/coverage.json\n\nGITIGNORE: check if _project/coverage.json is in .gitignore.\n grep \"coverage.json\" .gitignore\n If absent, add the line: _project/coverage.json\n\nPOST-MOVE VERIFICATION:\n uv run --project _project/scripts -- python _project/scripts/todo_cli.py stats\n Expected: same item counts as before - confirms no TODO path breakage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Archive superseded monolithic YAMLs and delete generated artifacts", "wid": "w2"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "Source files (confirmed no production code dependency on these paths):\n _project/adr/0001-unified-platform-registration.md\n _project/adr/adr-textcharts-extraction.md\n\nSteps:\n mkdir -p docs/development/adr/\n git mv _project/adr/0001-unified-platform-registration.md docs/development/adr/0001-unified-platform-registration.md\n git mv _project/adr/adr-textcharts-extraction.md docs/development/adr/adr-textcharts-extraction.md\n\nUPDATE REFERENCES (exactly one external reference confirmed):\n File: _blog/building-benchbox/outlines/06-extracting-textcharts.md\n Search: grep -n \"_project/adr\" _blog/building-benchbox/outlines/06-extracting-textcharts.md\n Update the path from `_project/adr/adr-textcharts-extraction.md`\n to `docs/development/adr/adr-textcharts-extraction.md`.\n\nAlso scan for any other references:\n grep -r \"_project/adr\" . --include=\"*.md\" --include=\"*.rst\" --include=\"*.yaml\"\n Expected after fix: zero matches.\n\nConsider adding docs/development/adr/README.md (one line: \"Architecture Decision Records\")\nto make the directory self-documenting if it doesn't already have one.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move ADRs from _project/adr/ to docs/development/adr/", "wid": "w3"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": [], "notes": "These 3 files are actively linked from outside _project/ and belong in docs/.\nConfirmed external references exist to all three.\n\nMOVE A - hosted-results-contract.md:\n git mv _project/analysis/hosted-results-contract.md docs/reference/hosted-results-contract.md\n Update references:\n grep -rn \"hosted-results-contract\" . --include=\"*.md\" --include=\"*.rst\"\n Files confirmed to reference it:\n - results-data/README.md\n - docs/operations/results-phase-2-runbook.md\n - _project/analysis/benchbox-results-platform-strategy.md (update in-place after move B)\n - _project/handoffs/ (do NOT update - historical records, broken links acceptable)\n\nMOVE B - threat-model.md:\n git mv _project/analysis/threat-model.md docs/reference/threat-model.md\n Update references:\n grep -rn \"threat-model\" . --include=\"*.md\" --include=\"*.rst\"\n Update any hits outside _project/handoffs/\n\nMOVE C - benchbox-results-platform-strategy.md:\n git mv _project/analysis/benchbox-results-platform-strategy.md docs/development/benchbox-results-platform-strategy.md\n Update references:\n grep -rn \"benchbox-results-platform-strategy\" . --include=\"*.md\"\n Update any hits outside _project/handoffs/\n\nFINAL VERIFICATION:\n grep -r \"_project/analysis/hosted-results-contract\\|_project/analysis/threat-model\\|_project/analysis/benchbox-results-platform-strategy\" \\\n . --include=\"*.md\" --include=\"*.rst\"\n Expected: zero matches (handoff internal links excluded from this check).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move 3 canonical analysis docs to proper docs/ locations", "wid": "w4"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": ["w1"], "notes": "Apply one of the classified actions per research gate outcome. The exact moves\ndepend entirely on w1 findings - the options are specified here:\n\nFOR _project/specs/ - pick ONE:\n Option A (authoritative content not in docs/):\n For each unique spec file: git mv _project/specs/{name}.md docs/benchmarks/{name}.md\n OR git mv _project/specs/platforms/{name}.md docs/platforms/{name}.md\n Archive anything already covered: git mv _project/specs/{dup}.md _project/_archive/specs/{dup}.md\n Option B (internal design notes only):\n Write _project/specs/README.md: \"Internal design notes - not published. See docs/ for authoritative content.\"\n No moves needed.\n Option C (superseded by public docs):\n git mv _project/specs/ _project/_archive/specs/\n\nFOR _project/DESIGN-integrated-task-management.md - pick ONE:\n Option A (agent navigation reference \u2192 stay in _project/): no action.\n Option B (publishable architecture doc \u2192 move):\n git mv _project/DESIGN-integrated-task-management.md docs/development/task-management-design.md\n grep -rn \"DESIGN-integrated-task-management\" . --include=\"*.md\" --include=\"*.py\" --include=\"*.yaml\"\n Update any hits.\n\nFOR roadmap/quality docs (PLATFORM_INTEGRATION_ROADMAP.md, RELEASE_PRIORITIZATION_v1.md, PRE_RELEASE_CHECKLIST.md) - pick ONE:\n Option A (agent-facing action lists \u2192 stay in _project/): no action.\n Option B (publishable \u2192 merge/move to docs/development/):\n git mv each file to appropriate docs/ path.\n Option C (completed milestones \u2192 archive):\n git mv each file to _project/_archive/\n\nFOR _project/DATAFRAME_SUPPORT_MASTER.md + FORMAT_CONVERTER_*.md + query_plan_*.md - pick ONE per file:\n Option A (not consulted in >90 days, no open TODO dependency): git mv to _project/_archive/\n Option B (active reference \u2192 condense): Do NOT bulk-move the 67 KB master doc.\n Create a condensed docs/development/dataframe-support-architecture.md (key decisions only)\n and archive the master doc. This is non-trivial - only do this if clearly warranted.\n\nFOR _project/handoffs/ - pick ONE:\n Option A (all historical): git mv _project/handoffs/ _project/_archive/handoffs/\n Option B (most recent is active): git mv older handoffs individually to _project/_archive/handoffs/\n Leave the most recent 1-2 in _project/handoffs/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Execute research-gate decisions for specs, design docs, roadmaps, and handoffs", "wid": "w5"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": ["w3", "w4", "w5"], "notes": "Scan for any paths moved in w3, w4, w5 that still have stale references:\n\n grep -r \"_project/adr\" . --include=\"*.md\" --include=\"*.rst\" --include=\"*.py\" --include=\"*.yaml\"\n grep -r \"_project/analysis/hosted-results-contract\\|_project/analysis/threat-model\\|_project/analysis/benchbox-results-platform-strategy\" \\\n . --include=\"*.md\" --include=\"*.rst\"\n\nIf w5 moved specs/ or design docs, also run:\n grep -r \"_project/specs\\|_project/DESIGN-integrated-task-management\\|_project/PLATFORM_INTEGRATION_ROADMAP\\|_project/RELEASE_PRIORITIZATION\\|_project/PRE_RELEASE_CHECKLIST\" \\\n . --include=\"*.md\" --include=\"*.rst\" --include=\"*.yaml\"\n\nFix every match that falls outside _project/handoffs/ (historical docs - broken links there are acceptable).\nIf AGENTS.md references any moved file, update it. The timing_wall_clock_allowlist.json reference\nin AGENTS.md line 64 MUST NOT be changed (that file is not moving).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update AGENTS.md and any remaining stale cross-references", "wid": "w6"}, {"evidence": null, "item_id": "cleanup-project-scratch-directory", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": "Run full validation suite:\n uv run --project _project/scripts -- python _project/scripts/validate_todo.py --all\n uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph\n uv run --project _project/scripts -- python _project/scripts/generate_indexes.py\n\nRun fast tests to confirm nothing is broken:\n uv run -- python -m pytest -m fast -q\n\nCommit with explicit staging - DO NOT use git add -A:\n git add \n git commit -m \"chore(cleanup): relocate canonical assets out of _project/ scratch area\"\n\nCommit message should list the major moves: ADRs, analysis docs, archived YAMLs,\nand note which research-gate decisions were made.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Validate TODO system integrity and commit all changes", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "\"Technical Debt & Refactoring\"", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Add CLI options for selective data generation and database loading phases without running queries, providing fine-grained control over benchmark preparation workflows\"\n- \"**Context**: Users need separate control over data generation and database loading phases for data preparation, storage testing, file format validation, and multi-stage benchmark workflows\"\n- \"**Flag Definitions**:\"\n- \"**Implementation Completed**:\"\n- \"**Files Modified**: `benchbox/cli/main.py`, `benchbox/cli/orchestrator.py`, `benchbox/cli/dryrun.py`, `benchbox/core/ssb/benchmark.py`, `tests/unit/cli/test_cli_data_load_modes.py`\"\n- \"**Impact**: Enables efficient data preparation workflows, supports file-based data generation for external tools, reduces execution time for data generation testing, and provides granular control over benchmark phases\"\n- \"**Manual Testing Verified**: All modes working correctly with proper validation and result reporting\"", "id": "cli-data-only-and-load-only-execution-options", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "CLI Data-Only and Load-Only Execution Options", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Data Quality and Compatibility Issues", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"`uv run -- python -m pytest tests/unit/core/test_runner_manifest_reuse.py`\"\n- \"`uv run -- python -m pytest tests/unit/generators/test_tpcds_generator.py::TestTPCDSDataGenerator::test_data_generation_workflow tests/unit/generators/test_tpcds_generator.py::TestGeneratorExtended::test_file_format_consistency_without_compression`\"", "id": "cli-detect-and-skip-unneeded-data-generation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"\u2705 CLI: Detect and Skip Unneeded Data Generation\"", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- Implemented manifest-based detection to skip redundant data generation without scanning or decompressing data files.\n- \"Generators (TPC-DS and TPC-H) now write `_datagen_manifest.json` containing per-table file entries with `path`, `size_bytes`, and `row_count`, plus global metadata (benchmark, scale, compression, parallel, timestamp).\"\n- \"Orchestrator preflight validates the manifest strictly: confirms benchmark/scale, verifies each listed file exists and its `size_bytes` matches. When valid, it reuses data and constructs `benchmark.tables` mapping directly from the manifest.\"\n- \"Added CLI flags `--no-regenerate` and `--force-regenerate` to control regeneration.\"\n- Added unit tests for orchestrator flag behavior and manifest creation/consumption.\n- \"Impact: Deterministic, fast detection for reuse (local and cloud), improved UX and run predictability.\"", "id": "cli-detect-and-skip-unneeded-data-generation-2", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"CLI: Detect and Skip Unneeded Data Generation \u2705\"", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "New Feature Requests", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Enhance CLI guided run capability with intelligent system-aware configuration and comprehensive guidance features\"\n- \"**Tasks**:\"\n- \"**Files Affected**: `benchbox/cli/benchmarks.py`, `benchbox/cli/database.py`, `benchbox/cli/main.py`, `benchbox/cli/orchestrator.py`, all CLI tests, comprehensive documentation\"\n- \"**Impact**: \u2705 Provides intelligent system-aware guidance that automatically detects hardware capabilities, recommends optimal configurations, prevents resource exhaustion, and guides users through benchmark setup with professional-grade recommendations. Reduces setup complexity by 90% while ensuring optimal performance.\"", "id": "cli-guided-run-enhancements-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "CLI Guided Run Enhancements \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "keep it as a --platform-option key", "dont": "add --driver-version as a standalone Click option", "why": "driver_version is intentionally a platform option (not universal)"}, {"INSTEAD": "only update the docstring", "dont": "change the MCP run_benchmark() function signature", "why": "that would be a breaking API change"}, {"INSTEAD": "only document it for Athena Spark", "dont": "document engine_version as user-settable for Snowflake/Redshift/ClickHouse/ Databricks/Firebolt/Trino/MotherDuck", "why": "on those platforms it is auto-probed via SQL and cannot be targeted"}], "approach": "Entry point for w2: benchbox/cli/commands/run.py ~line 480 where --platform-option\nis defined (hidden=True, advanced option). Extend the help string to enumerate the\nthree well-known keys with brief descriptions.\n\nEntry point for w3: benchbox/cli/help.py - locate EXAMPLES_BY_PLATFORM dict and\nadd examples to appropriate sections:\n- DuckDB or a generic section: --platform-option driver_version=1.2.0\n --platform-option driver_auto_install=true\n- Athena Spark section (or create one): --platform-option engine_version=\"PySpark engine version 3\"\nAlso fix any stale option descriptions identified in w1.\n\nEntry point for w4: benchbox/mcp/tools/benchmark.py run_benchmark() docstring\n(~line 151). Add an \"Available --platform-option keys\" note in the docstring.\nThen update CLAUDE.md's CLI Options Summary table to add a row or note for\ndriver_version, driver_auto_install, and engine_version (Athena Spark only).\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-22T00:00:00Z", "deferrals": [], "deps": [], "description": "Three user-facing arguments are missing from the CLI help text and MCP docstrings:\n\n1. driver_version - valid --platform-option key for all platforms, registered via\n _register_driver_version_options() in benchbox/cli/platform_defaults.py.\n Allows pinning the Python driver package to a specific version (e.g. '1.2.0').\n\n2. driver_auto_install - valid --platform-option key for all platforms (same\n registration). When true, automatically installs the requested driver version\n via uv if missing.\n\n3. engine_version - valid --platform-option key for Athena Spark specifically.\n Read at adapter construction from config.get(\"engine_version\") in\n benchbox/platforms/aws/athena_spark_adapter.py:707. Selects the Spark engine\n version (default: \"PySpark engine version 3\"). For all other cloud platforms\n (Snowflake, Redshift, ClickHouse, Databricks, Firebolt, Trino, MotherDuck),\n engine_version is AUTO-PROBED via SQL queries - it is NOT user-settable on\n those platforms.\n\nCurrently the --platform-option advanced option is described only as \"Platform\noption in KEY=VALUE form (repeatable)\" with no indication of what keys are\navailable. The MCP run_benchmark docstring similarly has no mention of driver or\nengine version options.\n\nAdditionally, the existing help text should be audited for consistency with the\ncurrent code - option descriptions, examples, and the CLAUDE.md quick reference\ntable should all reflect the actual CLI and MCP behaviour.", "id": "cli-help-driver-version-documentation", "preserves": ["--help and --help-topic all continue to render without errors (exit code 0)", "All existing --platform-option KEY=VALUE pairs still parse and apply correctly", "BenchBoxCommand.format_options() Core/Options/Advanced grouping is unchanged", "MCP run_benchmark() function signature unchanged - only the docstring is updated"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "CLAUDE.md"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}], "state": "done", "title": "Document driver_version, driver_auto_install, and engine_version in CLI/MCP help", "verifications": [{"command": "uv run benchbox run --help", "description": "Help text renders correctly for default --help", "expected": "output is well-formed, exit 0, existing options unchanged", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run benchbox run --help-topic all", "description": "Help topic all shows driver_version, driver_auto_install, engine_version", "expected": "output contains 'driver_version', 'driver_auto_install', and 'engine_version'", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run benchbox run --help-topic examples", "description": "Help topic examples shows driver version and Athena Spark engine_version usage", "expected": "output contains at least one --platform-option driver_version example", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests still pass (no regressions)", "expected": "all tests pass, 0 failures", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit existing help text in run.py and help.py against actual option list - note any stale, missing, or inaccurate descriptions to fix in w2-w4", "wid": "w1"}, {"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Improve --platform-option help string in run.py (~line 480) to list driver_version, driver_auto_install, and engine_version (Athena Spark) as well-known keys with one-line descriptions", "wid": "w2"}, {"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add usage examples to help.py: driver version pinning example and Athena Spark engine_version example; fix any stale descriptions found in w1", "wid": "w3"}, {"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update MCP run_benchmark docstring in benchmark.py to mention driver_version and engine_version (Athena Spark) as supported --platform-option keys; update CLAUDE.md CLI Options Summary table", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add --driver-version as a standalone Click option", "instead": "keep it as a --platform-option key", "why": "driver_version is intentionally a platform option (not universal)"}, {"dont": "change the MCP run_benchmark() function signature", "instead": "only update the docstring", "why": "that would be a breaking API change"}, {"dont": "document engine_version as user-settable for Snowflake/Redshift/ClickHouse/ Databricks/Firebolt/Trino/MotherDuck", "instead": "only document it for Athena Spark", "why": "on those platforms it is auto-probed via SQL and cannot be targeted"}], "approach": "Entry point for w2: benchbox/cli/commands/run.py ~line 480 where --platform-option\nis defined (hidden=True, advanced option). Extend the help string to enumerate the\nthree well-known keys with brief descriptions.\n\nEntry point for w3: benchbox/cli/help.py - locate EXAMPLES_BY_PLATFORM dict and\nadd examples to appropriate sections:\n- DuckDB or a generic section: --platform-option driver_version=1.2.0\n --platform-option driver_auto_install=true\n- Athena Spark section (or create one): --platform-option engine_version=\"PySpark engine version 3\"\nAlso fix any stale option descriptions identified in w1.\n\nEntry point for w4: benchbox/mcp/tools/benchmark.py run_benchmark() docstring\n(~line 151). Add an \"Available --platform-option keys\" note in the docstring.\nThen update CLAUDE.md's CLI Options Summary table to add a row or note for\ndriver_version, driver_auto_install, and engine_version (Athena Spark only).\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-22T00:00:00Z", "deferrals": [], "deps": [], "description": "Three user-facing arguments are missing from the CLI help text and MCP docstrings:\n\n1. driver_version - valid --platform-option key for all platforms, registered via\n _register_driver_version_options() in benchbox/cli/platform_defaults.py.\n Allows pinning the Python driver package to a specific version (e.g. '1.2.0').\n\n2. driver_auto_install - valid --platform-option key for all platforms (same\n registration). When true, automatically installs the requested driver version\n via uv if missing.\n\n3. engine_version - valid --platform-option key for Athena Spark specifically.\n Read at adapter construction from config.get(\"engine_version\") in\n benchbox/platforms/aws/athena_spark_adapter.py:707. Selects the Spark engine\n version (default: \"PySpark engine version 3\"). For all other cloud platforms\n (Snowflake, Redshift, ClickHouse, Databricks, Firebolt, Trino, MotherDuck),\n engine_version is AUTO-PROBED via SQL queries - it is NOT user-settable on\n those platforms.\n\nCurrently the --platform-option advanced option is described only as \"Platform\noption in KEY=VALUE form (repeatable)\" with no indication of what keys are\navailable. The MCP run_benchmark docstring similarly has no mention of driver or\nengine version options.\n\nAdditionally, the existing help text should be audited for consistency with the\ncurrent code - option descriptions, examples, and the CLAUDE.md quick reference\ntable should all reflect the actual CLI and MCP behaviour.", "id": "cli-help-driver-version-documentation", "preserves": ["--help and --help-topic all continue to render without errors (exit code 0)", "All existing --platform-option KEY=VALUE pairs still parse and apply correctly", "BenchBoxCommand.format_options() Core/Options/Advanced grouping is unchanged", "MCP run_benchmark() function signature unchanged - only the docstring is updated"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "CLAUDE.md"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}], "state": "done", "title": "Document driver_version, driver_auto_install, and engine_version in CLI/MCP help", "verifications": [{"command": "uv run benchbox run --help", "description": "Help text renders correctly for default --help", "expected": "output is well-formed, exit 0, existing options unchanged", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run benchbox run --help-topic all", "description": "Help topic all shows driver_version, driver_auto_install, engine_version", "expected": "output contains 'driver_version', 'driver_auto_install', and 'engine_version'", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run benchbox run --help-topic examples", "description": "Help topic examples shows driver version and Athena Spark engine_version usage", "expected": "output contains at least one --platform-option driver_version example", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests still pass (no regressions)", "expected": "all tests pass, 0 failures", "item_id": "cli-help-driver-version-documentation", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit existing help text in run.py and help.py against actual option list - note any stale, missing, or inaccurate descriptions to fix in w2-w4", "wid": "w1"}, {"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Improve --platform-option help string in run.py (~line 480) to list driver_version, driver_auto_install, and engine_version (Athena Spark) as well-known keys with one-line descriptions", "wid": "w2"}, {"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add usage examples to help.py: driver version pinning example and Athena Spark engine_version example; fix any stale descriptions found in w1", "wid": "w3"}, {"evidence": null, "item_id": "cli-help-driver-version-documentation", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update MCP run_benchmark docstring in benchmark.py to mention driver_version and engine_version (Athena Spark) as supported --platform-option keys; update CLAUDE.md CLI Options Summary table", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation and User Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Expand the CLI --help examples system with platform-specific and use-case\nspecific examples.\n\nCurrent state: 'benchbox run --help examples' provides categorized usage\nexamples. These could be expanded with:\n- Platform-specific examples (Snowflake, Databricks, etc.)\n- Use-case specific examples (comparison, tuning, data-only)\n- Error recovery examples\n\nThe goal is to make --help a complete reference that reduces need for\nexternal documentation.", "id": "cli-help-examples-expansion", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "CLI Help Examples Expansion", "verifications": [], "work": [], "worktree": "docs-experience"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation and User Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-13T00:00:00Z", "deferrals": [], "deps": [], "description": "Add CLI help examples for platforms currently missing from `benchbox run --help examples`.\n\nMissing platforms: Firebolt, Presto, Trino, Athena, PostgreSQL, Azure Synapse, Fabric Warehouse\n\nEach platform needs 3-5 practical examples showing:\n- Basic connection and authentication\n- Platform-specific options (catalog, warehouse, workgroup, etc.)\n- Common use cases (with comments explaining what each does)", "id": "cli-help-examples-missing-platforms", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "CLI Help Examples for Missing Platforms", "verifications": [], "work": [], "worktree": "docs-experience"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add --warmup-iterations in this change", "why": "out of scope"}, {"INSTEAD": "(not specified)", "dont": "change the default in constants.py", "why": "only the CLI wiring changes"}, {"INSTEAD": "(not specified)", "dont": "update the MCP tool", "why": "it hardcodes phases=power and is out of scope"}, {"INSTEAD": "(not specified)", "dont": "use @click.option instead of @advanced_option", "why": "--iterations should be hidden from --help to avoid parameter bloat"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-09T00:00:00Z", "deferrals": [], "deps": [], "description": "The power test runs a fixed 3 measurement iterations (GENERIC_POWER_DEFAULT_MEASUREMENT_ITERATIONS\nin benchbox/core/constants.py). Users have no way to control this from the CLI, which makes\nisolation testing and statistical analysis cumbersome - the only workaround is looping the\nCLI command manually.\n\nThe orchestrator and RunConfig already fully support variable iteration counts:\n- benchbox/cli/orchestrator.py:427 reads options.get(\"power_iterations\", 3)\n- benchbox/core/schemas.py:122 carries RunConfig.iterations through to every platform adapter\n- benchbox/core/results/schema.py derives run.iterations from actual query data automatically\n\nOnly the CLI wiring is missing.", "id": "cli-iterations-parameter", "preserves": ["All existing CLI options and their behaviour unchanged", "Default behaviour: --iterations omitted \u2192 3 measurement iterations (unchanged)", "Orchestrator and RunConfig require no changes"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/constants.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/schemas.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}], "state": "done", "title": "Add --iterations CLI parameter for power test measurement count", "verifications": [{"command": "benchbox run --platform spark --benchmark tpcds --scale 1 --queries 37 --phases power", "description": "Default still produces 3 iterations", "expected": "run.iterations == 3 in result JSON", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 1}, {"command": "benchbox run --platform spark --benchmark tpcds --scale 1 --queries 37 --phases power --iterations 10", "description": "10 iterations produces correct result", "expected": "run.iterations == 10 in result JSON", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 2}, {"command": "benchbox run --help | grep -c iterations; benchbox run --help-topic all | grep -c iterations", "description": "Flag hidden from --help, visible in --help-topic all", "expected": "0 and 1 respectively", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests pass", "expected": "All tests pass", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "cli-iterations-parameter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add @advanced_option decorator, run() signature parameter, and phase-compat warning", "wid": "w1"}, {"evidence": null, "item_id": "cli-iterations-parameter", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Propagate to all four BenchmarkConfig options dicts", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a second native-error classifier", "why": "reuse platform_registry.py:863."}, {"INSTEAD": "(not specified)", "dont": "fix the leak by removing functionality or dropping a dependency from a path that needs it at call time", "why": "defer the import, do not delete the feature."}, {"INSTEAD": "(not specified)", "dont": "rely on an in-process CliRunner for the --help isolation assertion", "why": "it shares the test session's sys.modules and will give a false PASS; invoke a fresh interpreter/subprocess."}], "approach": null, "blocked_reason": "imported from YAML status: Blocked", "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cli-lean-import-guarantee", "id": 8, "reason": "After the in-scope lazy refactor, tests/unit/test_import_isolation.py is down to\nboto3/google/cloudpathlib failures only. The verified remaining path is\nbenchbox/__init__.py eagerly importing FlightData, which imports\nbenchbox/utils/datagen_manifest.py, whose top-level cloudpathlib import loads\ncloudpathlib plus boto3/google provider SDKs. Both benchbox/__init__.py and\nbenchbox/utils/datagen_manifest.py are outside this TODO's scope_limit.only_modify,\nso the contract cannot be made green without widening scope.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Deferral describes exactly the remaining scope of the still-open, critical-priority source item itself (cloud-path lazy import of datagen_manifest.py/cloudpathlib). (tracked by cli-lean-import-guarantee)", "summary": "Make the remaining cloud-path import path lazy outside this TODO's file allowlist."}], "deps": [], "description": "v0.3.0 shipped a broken wheel to PyPI because an optional dependency loaded on\na CLI startup path that nothing tested. The contract test the team believed\nguarded this (tests/unit/test_package_dependencies.py) does NOT exist on\ndevelop (no git history at that path); the surviving startup test\n(tests/unit/cli/test_cli_startup_imports.py) only asserts that runner/monitoring\nand textcharts setup-help modules are absent - it makes NO assertion about heavy\noptional deps.\n\nA live import trace on develop @ cb49cecf5 shows the leak is real and\nentry-point-specific:\n - `import benchbox` eagerly loads duckdb, psycopg, boto3, google, cloudpathlib\n (NOT pandas/polars/pyarrow - drop-pandas-from-core worked).\n - `benchbox --help` additionally loads pandas, polars, pyarrow.\nVerified culprits (first benchbox frame that triggers each):\n - duckdb / psycopg <- benchbox/platforms/__init__.py:252 (eager\n _load_optional_adapter of concrete adapters)\n - boto3/google/cloudpathlib <- benchbox/core/benchmark_result_validation.py:8\n -> benchbox/utils/cloud_storage.py:22 (top-level\n cloudpathlib import)\n - pandas <- benchbox/utils/format_converters/__init__.py:16\n -> parquet_converter.py:15\n - pyarrow <- benchbox/utils/format_converters/__init__.py:7\n -> format_converters/base.py:15\n - polars <- benchbox/platforms/dataframe/__init__.py:60\n -> polars_df.py:36\n - the CLI command surface <- benchbox/cli/main.py:8 re-exports every command;\n benchbox/cli/commands/__init__.py:9 eagerly imports\n all command modules; BenchBoxGroup.format_commands\n (benchbox/cli/help.py:383) calls get_command() per\n subcommand, re-importing bodies during top-level help.\n\nThe common root cause is the same throughout: package __init__ modules (and the\ncli.main compat shim) eagerly import concrete heavy submodules. One consistent\ntechnique - defer to lazy resolution - fixes every hub. This item pins the\ncontract with a test FIRST, then makes each hub lazy, then wires the test (and a\ntwo-mode pre-publish wheel smoke) into CI so the regression cannot recur.\n", "id": "cli-lean-import-guarantee", "preserves": ["Bare installs without an optional extra still degrade gracefully (clear 'platform unavailable' message), never crash, for both ImportError and OSError.", "Every CLI command and its help text remain available and unchanged; benchbox --help and command execution paths keep working.", "No behavior change to command logic; this is an import-time/laziness refactor only."], "prior_art": [{"concept": "{'ref': '.github/workflows/nightly.yml:287', 'note': 'Existing isolated-wheel pandas-absence smoke (uv run --isolated --no-project --with wheel). Reuse its mechanism for the pre-publish two-mode smoke; today it checks only pandas and runs post-build, not as a publish predecessor.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '.github/workflows/release.yml:177', 'note': 'test-installation runs AFTER publish (needs: publish) and checks only pandas - detect not prevent. Reorder to a publish predecessor and broaden the assertion.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/platform_registry.py:863', 'note': 'Existing native-import-error classifier (_NATIVE_IMPORT_ERROR_MARKERS, OSError handling) - route the normalized optional-import handling (w6) through it; do not add a parallel classifier.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/cli/test_cli_startup_imports.py:24', 'note': 'Existing (too-narrow) startup-isolation test; extend its sys.modules pattern, do not invent a new harness. It only checks runner/monitoring/textcharts.'}", "decision": "extend", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/test_release_infrastructure.py:104', 'note': 'test_import_benchbox_does_not_pull_pandas - the single-dep precedent to generalize to the full heavy set across three entry points.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/cloud_storage.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/format_converters/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "active", "title": "Restore and enforce the lean-import guarantee for import benchbox, import benchbox.cli.main, and benchbox --help", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_import_isolation.py -q", "description": "All three entry points are lean in a clean interpreter", "expected": "passes; none of pandas/polars/pyarrow/duckdb/psycopg/boto3/google/cloudpathlib in sys.modules after import benchbox, import benchbox.cli.main, benchbox --help", "item_id": "cli-lean-import-guarantee", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- benchbox --help && uv run -- benchbox run --help", "description": "Command surface and help unchanged", "expected": "full command list rendered; exit 0; help text identical to pre-change snapshot", "item_id": "cli-lean-import-guarantee", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": [], "notes": "New tests/unit/test_import_isolation.py, subprocess-based (one clean\ninterpreter per case so the test session's own imports do not pollute\nsys.modules), parametrized per dep so failures are individually legible.\nAssert NONE of {pandas, polars, pyarrow, duckdb, psycopg, boto3, google,\ncloudpathlib} appear in sys.modules after each of: `import benchbox`,\n`import benchbox.cli.main`, and `benchbox --help` (via subprocess invocation\nof the console entry, not an in-process CliRunner that shares sys.modules).\nMark it fast. It supersedes the narrow assertions in\ntests/unit/cli/test_cli_startup_imports.py (keep that file's runner/monitoring\nchecks; do not duplicate). This pins the contract before any refactor.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write the import-isolation contract test across all three entry points (expected RED until w1-w6 land)", "wid": "w0"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "benchbox/platforms/__init__.py:252 eagerly calls _load_optional_adapter for\nconcrete adapters. The registry already supports lazy resolution (cudf is\nregistered lazily). Convert the eager concrete-adapter loads to string-based\nlazy registration (module_path, class_name) resolved on first get_adapter()/\ninstantiation. Preserve the current `except ImportError -> None` degradation\nand assert the missing-extra error message text in a test so it does not\nregress.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Lazy platform-adapter registration (fixes duckdb, psycopg)", "wid": "w1"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "Move the top-level cloudpathlib import in benchbox/utils/cloud_storage.py:22\nto function scope inside the cloud-path helpers, so importing the module (via\ncore/benchmark_result_validation.py:8 on the core path) does not pull the\ncloud SDKs. Re-verify cloud-path round-trip helpers still work.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Defer cloud SDK imports (fixes boto3, google, cloudpathlib)", "wid": "w2"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "benchbox/utils/format_converters/__init__.py eagerly imports base (pyarrow,\n:7) and parquet_converter (pandas, :16). Convert to PEP 562 module-level\n__getattr__ lazy exports (or a string registry) so importing the package\ndoes not pull pandas/pyarrow. Keep FormatConversionOrchestrator resolvable\n(the existing test_cli_startup_imports.py:30 assertion must stay green).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Lazy format-converter loading (fixes pandas, pyarrow)", "wid": "w3"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "benchbox/platforms/dataframe/__init__.py:60 eagerly imports polars_df.\nConvert to lazy registration consistent with w1. Keep DataFrame backend\nselection working.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Lazy DataFrame-backend registration (fixes polars)", "wid": "w4"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "A lazy Click group alone does NOT fix `import benchbox.cli.main`; all three\nparts are required or the CLI path stays heavy:\n (i) Entrypoint: move the console script from benchbox.cli.main:main\n (pyproject.toml:370) to benchbox.cli.app:main, and reduce\n benchbox/cli/main.py to a compat shim that re-exports cli/main/command\n symbols lazily via module-level __getattr__ (PEP 562) so importing it\n no longer eagerly binds command objects.\n (ii) Command fan-out: benchbox/cli/commands/__init__.py:9 imports every\n command module. Convert to lazy per-command resolution (string\n \"module:callback\" registry resolved on first use).\n (iii) Static help: BenchBoxGroup.format_commands (benchbox/cli/help.py:383)\n calls self.get_command() per subcommand, re-importing bodies during\n top-level help. Render from static command metadata (name + short help\n held in the registry) so `benchbox --help` never imports command\n bodies.\nAcceptance is BOTH that w0's benchbox.cli.main + --help cases pass AND that\nthe full command list and help text are unchanged (snapshot/list test), and\n`benchbox --help` plus execution paths still work.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make benchbox.cli.main itself lean - entrypoint, command fan-out, and static help (three parts)", "wid": "w5"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "Inconsistent catches let a native .so/GLIBC OSError crash instead of\ndegrading to \"platform unavailable\". Standardize on (ImportError, OSError),\nrouted through the existing native classifier (platform_registry.py:863), at:\n - benchbox/platforms/__init__.py:198, :225, :253 (catch ImportError only;\n :165 already catches both - match it)\n - benchbox/core/platform_registry.py:515 (_detect_library - ImportError\n only; :577 too)\nAdd a unit test that simulates an optional adapter import raising OSError ->\nreported unavailable, not a crash; ImportError behavior unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Normalize optional-import failure handling to (ImportError, OSError) at every lazy-load site", "wid": "w6"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": "1. Add tests/unit/test_import_isolation.py to the required code-lint/code-test\n lane in .github/workflows/pr.yml so the contract blocks merge.\n2. Pre-publish wheel smoke must verify TWO distinct properties, in two jobs,\n BOTH as predecessors of the publish job (not post-publish as\n release.yml:177 currently is):\n (a) minimal no-extras install: install the bare wheel in a clean env;\n `benchbox --help` must NOT crash.\n (b) full-env import isolation: in an env where heavy deps ARE installed,\n assert none appear in sys.modules after `import benchbox`,\n `import benchbox.cli.main`, and `benchbox --help` (reuse w0's set).\nReuse the nightly.yml:287 isolated-wheel mechanism; broaden its single-pandas\nassertion to the full heavy set.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire the contract test into required CI and split the wheel smoke into two pre-publish checks", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add --warmup-iterations in this change", "instead": "(not specified)", "why": "out of scope"}, {"dont": "change the default in constants.py", "instead": "(not specified)", "why": "only the CLI wiring changes"}, {"dont": "update the MCP tool", "instead": "(not specified)", "why": "it hardcodes phases=power and is out of scope"}, {"dont": "use @click.option instead of @advanced_option", "instead": "(not specified)", "why": "--iterations should be hidden from --help to avoid parameter bloat"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-09T00:00:00Z", "deferrals": [], "deps": [], "description": "The power test runs a fixed 3 measurement iterations (GENERIC_POWER_DEFAULT_MEASUREMENT_ITERATIONS\nin benchbox/core/constants.py). Users have no way to control this from the CLI, which makes\nisolation testing and statistical analysis cumbersome - the only workaround is looping the\nCLI command manually.\n\nThe orchestrator and RunConfig already fully support variable iteration counts:\n- benchbox/cli/orchestrator.py:427 reads options.get(\"power_iterations\", 3)\n- benchbox/core/schemas.py:122 carries RunConfig.iterations through to every platform adapter\n- benchbox/core/results/schema.py derives run.iterations from actual query data automatically\n\nOnly the CLI wiring is missing.", "id": "cli-iterations-parameter", "preserves": ["All existing CLI options and their behaviour unchanged", "Default behaviour: --iterations omitted \u2192 3 measurement iterations (unchanged)", "Orchestrator and RunConfig require no changes"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/constants.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/schemas.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}], "state": "done", "title": "Add --iterations CLI parameter for power test measurement count", "verifications": [{"command": "benchbox run --platform spark --benchmark tpcds --scale 1 --queries 37 --phases power", "description": "Default still produces 3 iterations", "expected": "run.iterations == 3 in result JSON", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 1}, {"command": "benchbox run --platform spark --benchmark tpcds --scale 1 --queries 37 --phases power --iterations 10", "description": "10 iterations produces correct result", "expected": "run.iterations == 10 in result JSON", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 2}, {"command": "benchbox run --help | grep -c iterations; benchbox run --help-topic all | grep -c iterations", "description": "Flag hidden from --help, visible in --help-topic all", "expected": "0 and 1 respectively", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast tests pass", "expected": "All tests pass", "item_id": "cli-iterations-parameter", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "cli-iterations-parameter", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add @advanced_option decorator, run() signature parameter, and phase-compat warning", "wid": "w1"}, {"evidence": null, "item_id": "cli-iterations-parameter", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Propagate to all four BenchmarkConfig options dicts", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add a second native-error classifier", "instead": "(not specified)", "why": "reuse platform_registry.py:863."}, {"dont": "fix the leak by removing functionality or dropping a dependency from a path that needs it at call time", "instead": "(not specified)", "why": "defer the import, do not delete the feature."}, {"dont": "rely on an in-process CliRunner for the --help isolation assertion", "instead": "(not specified)", "why": "it shares the test session's sys.modules and will give a false PASS; invoke a fresh interpreter/subprocess."}], "approach": null, "blocked_reason": "imported from YAML status: Blocked", "category": "Release Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cli-lean-import-guarantee", "id": 8, "reason": "After the in-scope lazy refactor, tests/unit/test_import_isolation.py is down to\nboto3/google/cloudpathlib failures only. The verified remaining path is\nbenchbox/__init__.py eagerly importing FlightData, which imports\nbenchbox/utils/datagen_manifest.py, whose top-level cloudpathlib import loads\ncloudpathlib plus boto3/google provider SDKs. Both benchbox/__init__.py and\nbenchbox/utils/datagen_manifest.py are outside this TODO's scope_limit.only_modify,\nso the contract cannot be made green without widening scope.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Deferral describes exactly the remaining scope of the still-open, critical-priority source item itself (cloud-path lazy import of datagen_manifest.py/cloudpathlib). (tracked by cli-lean-import-guarantee)", "summary": "Make the remaining cloud-path import path lazy outside this TODO's file allowlist."}], "deps": [], "description": "v0.3.0 shipped a broken wheel to PyPI because an optional dependency loaded on\na CLI startup path that nothing tested. The contract test the team believed\nguarded this (tests/unit/test_package_dependencies.py) does NOT exist on\ndevelop (no git history at that path); the surviving startup test\n(tests/unit/cli/test_cli_startup_imports.py) only asserts that runner/monitoring\nand textcharts setup-help modules are absent - it makes NO assertion about heavy\noptional deps.\n\nA live import trace on develop @ cb49cecf5 shows the leak is real and\nentry-point-specific:\n - `import benchbox` eagerly loads duckdb, psycopg, boto3, google, cloudpathlib\n (NOT pandas/polars/pyarrow - drop-pandas-from-core worked).\n - `benchbox --help` additionally loads pandas, polars, pyarrow.\nVerified culprits (first benchbox frame that triggers each):\n - duckdb / psycopg <- benchbox/platforms/__init__.py:252 (eager\n _load_optional_adapter of concrete adapters)\n - boto3/google/cloudpathlib <- benchbox/core/benchmark_result_validation.py:8\n -> benchbox/utils/cloud_storage.py:22 (top-level\n cloudpathlib import)\n - pandas <- benchbox/utils/format_converters/__init__.py:16\n -> parquet_converter.py:15\n - pyarrow <- benchbox/utils/format_converters/__init__.py:7\n -> format_converters/base.py:15\n - polars <- benchbox/platforms/dataframe/__init__.py:60\n -> polars_df.py:36\n - the CLI command surface <- benchbox/cli/main.py:8 re-exports every command;\n benchbox/cli/commands/__init__.py:9 eagerly imports\n all command modules; BenchBoxGroup.format_commands\n (benchbox/cli/help.py:383) calls get_command() per\n subcommand, re-importing bodies during top-level help.\n\nThe common root cause is the same throughout: package __init__ modules (and the\ncli.main compat shim) eagerly import concrete heavy submodules. One consistent\ntechnique - defer to lazy resolution - fixes every hub. This item pins the\ncontract with a test FIRST, then makes each hub lazy, then wires the test (and a\ntwo-mode pre-publish wheel smoke) into CI so the regression cannot recur.\n", "id": "cli-lean-import-guarantee", "preserves": ["Bare installs without an optional extra still degrade gracefully (clear 'platform unavailable' message), never crash, for both ImportError and OSError.", "Every CLI command and its help text remain available and unchanged; benchbox --help and command execution paths keep working.", "No behavior change to command logic; this is an import-time/laziness refactor only."], "prior_art": [{"concept": "{'ref': '.github/workflows/nightly.yml:287', 'note': 'Existing isolated-wheel pandas-absence smoke (uv run --isolated --no-project --with wheel). Reuse its mechanism for the pre-publish two-mode smoke; today it checks only pandas and runs post-build, not as a publish predecessor.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '.github/workflows/release.yml:177', 'note': 'test-installation runs AFTER publish (needs: publish) and checks only pandas - detect not prevent. Reorder to a publish predecessor and broaden the assertion.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/platform_registry.py:863', 'note': 'Existing native-import-error classifier (_NATIVE_IMPORT_ERROR_MARKERS, OSError handling) - route the normalized optional-import handling (w6) through it; do not add a parallel classifier.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/cli/test_cli_startup_imports.py:24', 'note': 'Existing (too-narrow) startup-isolation test; extend its sys.modules pattern, do not invent a new harness. It only checks runner/monitoring/textcharts.'}", "decision": "extend", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/test_release_infrastructure.py:104', 'note': 'test_import_benchbox_does_not_pull_pandas - the single-dep precedent to generalize to the full heavy set across three entry points.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/cloud_storage.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/format_converters/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "active", "title": "Restore and enforce the lean-import guarantee for import benchbox, import benchbox.cli.main, and benchbox --help", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_import_isolation.py -q", "description": "All three entry points are lean in a clean interpreter", "expected": "passes; none of pandas/polars/pyarrow/duckdb/psycopg/boto3/google/cloudpathlib in sys.modules after import benchbox, import benchbox.cli.main, benchbox --help", "item_id": "cli-lean-import-guarantee", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- benchbox --help && uv run -- benchbox run --help", "description": "Command surface and help unchanged", "expected": "full command list rendered; exit 0; help text identical to pre-change snapshot", "item_id": "cli-lean-import-guarantee", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": [], "notes": "New tests/unit/test_import_isolation.py, subprocess-based (one clean\ninterpreter per case so the test session's own imports do not pollute\nsys.modules), parametrized per dep so failures are individually legible.\nAssert NONE of {pandas, polars, pyarrow, duckdb, psycopg, boto3, google,\ncloudpathlib} appear in sys.modules after each of: `import benchbox`,\n`import benchbox.cli.main`, and `benchbox --help` (via subprocess invocation\nof the console entry, not an in-process CliRunner that shares sys.modules).\nMark it fast. It supersedes the narrow assertions in\ntests/unit/cli/test_cli_startup_imports.py (keep that file's runner/monitoring\nchecks; do not duplicate). This pins the contract before any refactor.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write the import-isolation contract test across all three entry points (expected RED until w1-w6 land)", "wid": "w0"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "benchbox/platforms/__init__.py:252 eagerly calls _load_optional_adapter for\nconcrete adapters. The registry already supports lazy resolution (cudf is\nregistered lazily). Convert the eager concrete-adapter loads to string-based\nlazy registration (module_path, class_name) resolved on first get_adapter()/\ninstantiation. Preserve the current `except ImportError -> None` degradation\nand assert the missing-extra error message text in a test so it does not\nregress.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Lazy platform-adapter registration (fixes duckdb, psycopg)", "wid": "w1"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "Move the top-level cloudpathlib import in benchbox/utils/cloud_storage.py:22\nto function scope inside the cloud-path helpers, so importing the module (via\ncore/benchmark_result_validation.py:8 on the core path) does not pull the\ncloud SDKs. Re-verify cloud-path round-trip helpers still work.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Defer cloud SDK imports (fixes boto3, google, cloudpathlib)", "wid": "w2"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "benchbox/utils/format_converters/__init__.py eagerly imports base (pyarrow,\n:7) and parquet_converter (pandas, :16). Convert to PEP 562 module-level\n__getattr__ lazy exports (or a string registry) so importing the package\ndoes not pull pandas/pyarrow. Keep FormatConversionOrchestrator resolvable\n(the existing test_cli_startup_imports.py:30 assertion must stay green).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Lazy format-converter loading (fixes pandas, pyarrow)", "wid": "w3"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "benchbox/platforms/dataframe/__init__.py:60 eagerly imports polars_df.\nConvert to lazy registration consistent with w1. Keep DataFrame backend\nselection working.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Lazy DataFrame-backend registration (fixes polars)", "wid": "w4"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "A lazy Click group alone does NOT fix `import benchbox.cli.main`; all three\nparts are required or the CLI path stays heavy:\n (i) Entrypoint: move the console script from benchbox.cli.main:main\n (pyproject.toml:370) to benchbox.cli.app:main, and reduce\n benchbox/cli/main.py to a compat shim that re-exports cli/main/command\n symbols lazily via module-level __getattr__ (PEP 562) so importing it\n no longer eagerly binds command objects.\n (ii) Command fan-out: benchbox/cli/commands/__init__.py:9 imports every\n command module. Convert to lazy per-command resolution (string\n \"module:callback\" registry resolved on first use).\n (iii) Static help: BenchBoxGroup.format_commands (benchbox/cli/help.py:383)\n calls self.get_command() per subcommand, re-importing bodies during\n top-level help. Render from static command metadata (name + short help\n held in the registry) so `benchbox --help` never imports command\n bodies.\nAcceptance is BOTH that w0's benchbox.cli.main + --help cases pass AND that\nthe full command list and help text are unchanged (snapshot/list test), and\n`benchbox --help` plus execution paths still work.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make benchbox.cli.main itself lean - entrypoint, command fan-out, and static help (three parts)", "wid": "w5"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w0"], "notes": "Inconsistent catches let a native .so/GLIBC OSError crash instead of\ndegrading to \"platform unavailable\". Standardize on (ImportError, OSError),\nrouted through the existing native classifier (platform_registry.py:863), at:\n - benchbox/platforms/__init__.py:198, :225, :253 (catch ImportError only;\n :165 already catches both - match it)\n - benchbox/core/platform_registry.py:515 (_detect_library - ImportError\n only; :577 too)\nAdd a unit test that simulates an optional adapter import raising OSError ->\nreported unavailable, not a crash; ImportError behavior unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Normalize optional-import failure handling to (ImportError, OSError) at every lazy-load site", "wid": "w6"}, {"evidence": null, "item_id": "cli-lean-import-guarantee", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": "1. Add tests/unit/test_import_isolation.py to the required code-lint/code-test\n lane in .github/workflows/pr.yml so the contract blocks merge.\n2. Pre-publish wheel smoke must verify TWO distinct properties, in two jobs,\n BOTH as predecessors of the publish job (not post-publish as\n release.yml:177 currently is):\n (a) minimal no-extras install: install the bare wheel in a clean env;\n `benchbox --help` must NOT crash.\n (b) full-env import isolation: in an env where heavy deps ARE installed,\n assert none appear in sys.modules after `import benchbox`,\n `import benchbox.cli.main`, and `benchbox --help` (reuse w0's set).\nReuse the nightly.yml:287 isolated-wheel mechanism; broaden its single-pandas\nassertion to the full heavy set.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire the contract test into required CI and split the wheel smoke into two pre-publish checks", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Architecture", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Archived DONE item for: \u2705 CLI Parameter Consistency with Unified Runner", "id": "cli-parameter-consistency-with-unified-runner", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 CLI Parameter Consistency with Unified Runner", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "\"Technical Debt & Refactoring\"", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Files Affected**:\"\n- \"**\u2705 Results Achieved**:\"\n- \"**Impact**: \u2705 ACHIEVED - Dramatically improves user experience for multi-platform benchmarking, reduces configuration errors, provides clear guidance for library installation, enables better environment management, and makes BenchBox more accessible to users working with multiple database platforms\"", "id": "cli-platform-adapter-management-feature", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 CLI Platform Adapter Management Feature", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Comprehensive test coverage improvement plan for CLI modules to achieve 80% coverage target, addressing critical gaps in core CLI functionality\"\n- \"**Current Coverage Analysis** (Final status):\"\n- \"**Implementation Strategy Completed**: Fixed all failing tests, created comprehensive test files for missing modules\"\n- \"**Tasks**:\"\n- \"**Implementation Summary**: \u2705 **COMPLETED** - Successfully addressed all critical CLI test coverage gaps and fixed all failing tests. Created 1,200+ lines of comprehensive test code across 3 new test files and fixed 4 existing test files.\"\n- \"**Key Achievements**:\"\n- \"**Files Modified**:\"\n- \"**Test Coverage Improvement**: Significantly expanded CLI test coverage from critical gaps to comprehensive coverage\"\n- \"**Impact**: \u2705 **COMPLETED** - Ensured CLI reliability after recent consolidation changes, provided comprehensive validation of user-facing functionality, enabled confident CLI refactoring, improved user experience with robust test coverage\"", "id": "cli-test-coverage-enhancement-to-80", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"CLI Test Coverage Enhancement to 80%\"", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Comprehensive verbose logging mode implementation for CLI debugging and troubleshooting\"\n- \"**Tasks**:\"\n- \"**Files Affected**: `benchbox/cli/main.py`, `tests/unit/cli/test_cli_verbose.py`, `_project/cli_verbose_logging_reference.md`, logging configuration throughout the project\"\n- \"**Implementation Details**:\"\n- \"**Test Coverage**: \u2705 16 comprehensive tests covering verbose/non-verbose modes, CLI flag handling, third-party logger suppression, and integration scenarios - all passing\"\n- \"**Documentation**: \u2705 Comprehensive CLI reference guide with practical examples, troubleshooting workflows, and advanced usage patterns\"\n- \"**Impact**: \u2705 Enables users to diagnose CLI issues effectively, provides detailed insight into benchmark execution flow, helps identify configuration problems, and aids in troubleshooting platform-specific issues. Users can now debug complex benchmark setup and execution problems with comprehensive logging output. Complete implementation with full test coverage and documentation.\"", "id": "cli-verbose-logging-mode-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "CLI Verbose Logging Mode \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:35:52Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #370 of dataframe-test-coverage.\nDeferred: ClickBench dataframe queries (446 stmts, 30.5%)\nReason deferred: ClickBench uses a single wide table with specific data patterns. Testing requires the ClickBench dataset or representative fixtures. Lower priority - push after main targets are hit.", "id": "clickbench-dataframe-queries-446-stmts-30-5", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "ClickBench dataframe queries (446 stmts, 30.5%)", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 **COMPLETED**\"\n- \"**Completion Date**: 2025-09-04\"\n- \"**Description**: ClickBench data generator creates empty strings for fields that should be empty, but constraint violation suggests NULL values may be generated instead\"\n- \"**Root Cause Analysis**: The ClickBench schema correctly defines all fields as `NOT NULL` and queries use `<> '` filtering patterns, indicating empty strings are the expected representation for missing data. However, the CLI fails with \\\"NOT NULL constraint failed: hits.Referer\\\" suggesting actual NULL values are being generated somewhere in the data pipeline.\"\n- \"**Final Root Cause**: DuckDB CSV loading was using default NULL handling (`nullstr=''`), which converts empty CSV fields to NULL values, violating ClickBench's NOT NULL constraints.\"\n- \"**Solution Implemented**:\"\n- \"**Tasks Completed**:\"\n- \"**Files Modified**:\"\n- \"**Technical Details**:\"\n- \"**Validation Results**:\"\n- \"**Impact**:\"", "id": "clickbench-empty-string-vs-null-constraint-issue-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 ClickBench Empty String vs NULL Constraint Issue (COMPLETED)", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "backfill historical result artifacts in this TODO", "why": "scope it to alias removal only"}, {"INSTEAD": "(not specified)", "dont": "remove the alias without the helpful error message", "why": "silent 'platform not found' errors are harder to diagnose"}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "clickhouse-alias-removal-post-deprecation", "id": 309, "reason": "Mutating stored result artifacts requires a separate decision about historical record integrity; file separately if needed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Historical-artifact backfill explicitly deferred pending a separate decision on mutating stored records; no such decision made.", "summary": "Backfill historical result artifacts: rename old clickhouse labels to clickhouse-local or clickhouse-server"}], "deps": [], "description": "When ClickHouse support was split into `clickhouse-local` and\n`clickhouse-server` platforms (`first-class-clickhouse-local-server-cloud-platforms`,\nDONE 2026-04-11), a set of temporary compatibility aliases were added so\nthat existing `platform: clickhouse` references continue to work during a\ndeprecation window.\n\nThe DONE file defers two cleanup tasks:\n(a) Remove the temporary `clickhouse` compatibility aliases entirely after\n one deprecation window.\n(b) Backfill or rewrite historical result artifacts so old `clickhouse`\n platform labels are normalized to `clickhouse-local` or `clickhouse-server`.\n\nThis TODO tracks (a). Artifact backfill (b) is a separate larger operation\nthat depends on a decision about whether old result artifacts should be\nmutated; it should be filed separately if needed.", "id": "clickhouse-alias-removal-post-deprecation", "preserves": ["Error message for the removed 'clickhouse' alias names the replacements explicitly", "clickhouse-local and clickhouse-server platforms continue to work correctly after alias removal"], "prior_art": [{"concept": "CHANGELOG \u2014 confirm the deprecation start date", "decision": "reuse", "path": "CHANGELOG"}, {"concept": "benchbox/platforms/ \u2014 location of platform registration and alias maps", "decision": "reuse", "path": "benchbox/platforms/"}, {"concept": "first-class-clickhouse-local-server-cloud-platforms DONE file \u2014 source of the alias and deprecation intent", "decision": "reuse", "path": "first-class-clickhouse-local-server-cloud-platforms"}], "priority": "low", "scope": [], "state": "done", "title": "Remove temporary clickhouse\u2192clickhouse-local/server compatibility aliases after deprecation window", "verifications": [{"command": "benchbox run --platform clickhouse --benchmark tpch 2>&1 | head -5", "description": "benchbox run --platform clickhouse produces a clear error message", "expected": "error message mentioning 'clickhouse-local' and 'clickhouse-server'", "item_id": "clickhouse-alias-removal-post-deprecation", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -rn \"'clickhouse'\" benchbox/platforms/ | grep -v 'clickhouse-' | grep -v '#'", "description": "No clickhouse (unqualified) alias remains in platform registration", "expected": "empty (no unqualified alias registrations remain)", "item_id": "clickhouse-alias-removal-post-deprecation", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": [], "notes": "grep -rn 'clickhouse[^-]' benchbox/ tests/ docs/ --include='*.py' --include='*.yaml' --include='*.md'\nIdentify: adapter registration aliases, platform-name normalisation maps,\ndocs references, test fixtures that use 'clickhouse' without a suffix.\nConfirm the deprecation window started at the release that landed the\nfirst-class-clickhouse TODO (check CHANGELOG for the release date).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory all current clickhouse alias sites and confirm the deprecation window has passed", "wid": "w0"}, {"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": ["w0"], "notes": "Remove the platform alias mapping entries from benchbox/platforms/ (likely\na PLATFORM_ALIASES or similar dict). After removal, benchbox run --platform\nclickhouse should fail with a clear error message naming the replacements\n(clickhouse-local, clickhouse-server).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the compatibility alias registrations", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": ["w1"], "notes": "When platform='clickhouse' is passed after alias removal, raise a clear\nerror:\n \"Platform 'clickhouse' has been removed. Use 'clickhouse-local' or\n 'clickhouse-server' instead.\"\nThis prevents silent breakage for callers who missed the deprecation notice.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a helpful error message for the removed alias", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": ["w2"], "notes": "Update docs/platforms/clickhouse.md, README, and any runbook references\nto use the qualified names. Remove any test fixtures that used the\nunqualified alias.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs and remove any remaining clickhouse (unqualified) references", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "add cloud mode as a new code path alongside existing modes", "dont": "break existing ClickHouse self-hosted modes (server, chdb)", "why": "self-hosted users must not be affected"}, {"INSTEAD": "add cloud authentication as an additional branch", "dont": "refactor existing ClickHouse connection logic", "why": "server and chdb modes are stable"}, {"INSTEAD": "gate cloud config behind deployment mode checks", "dont": "require cloud credentials for non-cloud modes", "why": "self-hosted users should not need cloud configuration"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-16T00:00:00Z", "completed_pr": null, "created_at": "2026-01-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add ClickHouse Cloud deployment mode to the existing ClickHouse adapter,\nenabling benchmarking of ClickHouse's managed cloud service via the\nclickhouse:cloud syntax.\n\nThis completes the ClickHouse platform family: local (chDB), server\n(self-hosted), and cloud (managed).", "id": "clickhouse-cloud-deployment", "preserves": ["All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "Existing ClickHouse unit and integration tests continue to pass", "Existing clickhouse:server and chdb modes unchanged", "Platform registry autodiscovery continues working for all existing platforms"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/clickhouse_cloud.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_clickhouse_adapter.py"}], "state": "done", "title": "ClickHouse Cloud Deployment Mode", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.credentials.clickhouse_cloud import *; print('import OK')\"", "description": "Cloud module imports correctly", "expected": "import OK", "item_id": "clickhouse-cloud-deployment", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py -v --tb=short", "description": "ClickHouse adapter unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "clickhouse-cloud-deployment", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "clickhouse-cloud-deployment", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'cloud' deployment mode to ClickHouse registry metadata", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement OAuth token authentication", "wid": "w10"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add credential setup wizard for ClickHouse Cloud", "wid": "w11"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support environment variables", "wid": "w12"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "CLICKHOUSE_CLOUD_ENDPOINT", "wid": "w13"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "CLICKHOUSE_CLOUD_API_KEY", "wid": "w14"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement cloud endpoint connection", "wid": "w15"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement cloud storage upload for data loading", "wid": "w16"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support S3 staging", "wid": "w17"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support GCS staging", "wid": "w18"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement COPY FROM for cloud data", "wid": "w19"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set requires_credentials=True", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Configure result caching behavior", "wid": "w20"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for cloud authentication", "wid": "w21"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration tests with ClickHouse Cloud (requires account)", "wid": "w22"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-H benchmark at scale 0.1", "wid": "w23"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Compare results with self-hosted ClickHouse", "wid": "w24"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set requires_cloud_storage=True", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define auth_methods=['api_key', 'oauth']", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cloud-specific configuration options", "wid": "w5"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud endpoint URL", "wid": "w6"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Organization/service ID", "wid": "w7"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "API key or OAuth token", "wid": "w8"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement API key authentication", "wid": "w9"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "create clickhouse_sql/ overrides instead", "dont": "modify base query files", "why": "other platforms depend on standard SQL FILTER/EXTRACT"}, {"INSTEAD": "wrap in toDateTime() first", "dont": "use toHour() directly on TIME columns", "why": "ClickHouse toHour() rejects bare TIME values"}], "approach": "Follow existing override pattern. For QSA4, mechanically replace\nSUM(x) FILTER (WHERE cond) with sumIf(x, cond). For QTM1, wrap\norder_time in toDateTime() before passing to toHour().\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "6 query failures on ClickHouse in coffeeshop benchmark:\n- QSA4 \u00d7 3: SUM(x) FILTER (WHERE cond) \u2192 ClickHouse needs sumIf(x, cond)\n- QTM1 \u00d7 3: EXTRACT(HOUR FROM order_time) where order_time is TIME type \u2192\n toHour() doesn't accept bare TIME values, needs toHour(toDateTime(order_time))\n\nPart of Pattern B (EXTRACT) and Pattern D (FILTER aggregate).", "id": "clickhouse-coffeeshop-dialect-overrides", "preserves": ["Base coffeeshop QSA4/QTM1 queries unchanged - other platforms use standard FILTER and EXTRACT", "Override queries return identical column names and semantics"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Add ClickHouse dialect overrides for coffeeshop QSA4 (sumIf) and QTM1 (TIME type)", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'coffeeshop and clickhouse'", "description": "ClickHouse coffeeshop overrides load with sumIf and toDateTime wrappers", "expected": "Tests pass confirming sumIf/toDateTime overrides for QSA4/QTM1", "item_id": "clickhouse-coffeeshop-dialect-overrides", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-coffeeshop-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for SA4 - actual issue is nested window aggregate SUM(SUM()) OVER (), replaced with CROSS JOIN subquery. TODO description was incorrect about FILTER clause.", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-coffeeshop-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for TM1 using toHour(toDateTime(order_time)) for TIME-type column", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-coffeeshop-dialect-overrides", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests: load SA4/TM1 for ClickHouse and assert SQL avoids nested window aggregate and uses toDateTime() respectively", "wid": "w3"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "backfill historical result artifacts in this TODO", "instead": "(not specified)", "why": "scope it to alias removal only"}, {"dont": "remove the alias without the helpful error message", "instead": "(not specified)", "why": "silent 'platform not found' errors are harder to diagnose"}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "clickhouse-alias-removal-post-deprecation", "id": 309, "reason": "Mutating stored result artifacts requires a separate decision about historical record integrity; file separately if needed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Historical-artifact backfill explicitly deferred pending a separate decision on mutating stored records; no such decision made.", "summary": "Backfill historical result artifacts: rename old clickhouse labels to clickhouse-local or clickhouse-server"}], "deps": [], "description": "When ClickHouse support was split into `clickhouse-local` and\n`clickhouse-server` platforms (`first-class-clickhouse-local-server-cloud-platforms`,\nDONE 2026-04-11), a set of temporary compatibility aliases were added so\nthat existing `platform: clickhouse` references continue to work during a\ndeprecation window.\n\nThe DONE file defers two cleanup tasks:\n(a) Remove the temporary `clickhouse` compatibility aliases entirely after\n one deprecation window.\n(b) Backfill or rewrite historical result artifacts so old `clickhouse`\n platform labels are normalized to `clickhouse-local` or `clickhouse-server`.\n\nThis TODO tracks (a). Artifact backfill (b) is a separate larger operation\nthat depends on a decision about whether old result artifacts should be\nmutated; it should be filed separately if needed.", "id": "clickhouse-alias-removal-post-deprecation", "preserves": ["Error message for the removed 'clickhouse' alias names the replacements explicitly", "clickhouse-local and clickhouse-server platforms continue to work correctly after alias removal"], "prior_art": [{"concept": "CHANGELOG \u2014 confirm the deprecation start date", "decision": "reuse", "path": "CHANGELOG"}, {"concept": "benchbox/platforms/ \u2014 location of platform registration and alias maps", "decision": "reuse", "path": "benchbox/platforms/"}, {"concept": "first-class-clickhouse-local-server-cloud-platforms DONE file \u2014 source of the alias and deprecation intent", "decision": "reuse", "path": "first-class-clickhouse-local-server-cloud-platforms"}], "priority": "low", "scope": [], "state": "done", "title": "Remove temporary clickhouse\u2192clickhouse-local/server compatibility aliases after deprecation window", "verifications": [{"command": "benchbox run --platform clickhouse --benchmark tpch 2>&1 | head -5", "description": "benchbox run --platform clickhouse produces a clear error message", "expected": "error message mentioning 'clickhouse-local' and 'clickhouse-server'", "item_id": "clickhouse-alias-removal-post-deprecation", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -rn \"'clickhouse'\" benchbox/platforms/ | grep -v 'clickhouse-' | grep -v '#'", "description": "No clickhouse (unqualified) alias remains in platform registration", "expected": "empty (no unqualified alias registrations remain)", "item_id": "clickhouse-alias-removal-post-deprecation", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": [], "notes": "grep -rn 'clickhouse[^-]' benchbox/ tests/ docs/ --include='*.py' --include='*.yaml' --include='*.md'\nIdentify: adapter registration aliases, platform-name normalisation maps,\ndocs references, test fixtures that use 'clickhouse' without a suffix.\nConfirm the deprecation window started at the release that landed the\nfirst-class-clickhouse TODO (check CHANGELOG for the release date).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory all current clickhouse alias sites and confirm the deprecation window has passed", "wid": "w0"}, {"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": ["w0"], "notes": "Remove the platform alias mapping entries from benchbox/platforms/ (likely\na PLATFORM_ALIASES or similar dict). After removal, benchbox run --platform\nclickhouse should fail with a clear error message naming the replacements\n(clickhouse-local, clickhouse-server).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the compatibility alias registrations", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": ["w1"], "notes": "When platform='clickhouse' is passed after alias removal, raise a clear\nerror:\n \"Platform 'clickhouse' has been removed. Use 'clickhouse-local' or\n 'clickhouse-server' instead.\"\nThis prevents silent breakage for callers who missed the deprecation notice.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a helpful error message for the removed alias", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-alias-removal-post-deprecation", "needs": ["w2"], "notes": "Update docs/platforms/clickhouse.md, README, and any runbook references\nto use the qualified names. Remove any test fixtures that used the\nunqualified alias.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs and remove any remaining clickhouse (unqualified) references", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "break existing ClickHouse self-hosted modes (server, chdb)", "instead": "add cloud mode as a new code path alongside existing modes", "why": "self-hosted users must not be affected"}, {"dont": "refactor existing ClickHouse connection logic", "instead": "add cloud authentication as an additional branch", "why": "server and chdb modes are stable"}, {"dont": "require cloud credentials for non-cloud modes", "instead": "gate cloud config behind deployment mode checks", "why": "self-hosted users should not need cloud configuration"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-16T00:00:00Z", "completed_pr": null, "created_at": "2026-01-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Add ClickHouse Cloud deployment mode to the existing ClickHouse adapter,\nenabling benchmarking of ClickHouse's managed cloud service via the\nclickhouse:cloud syntax.\n\nThis completes the ClickHouse platform family: local (chDB), server\n(self-hosted), and cloud (managed).", "id": "clickhouse-cloud-deployment", "preserves": ["All existing platform adapter tests pass: uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "Existing ClickHouse unit and integration tests continue to pass", "Existing clickhouse:server and chdb modes unchanged", "Platform registry autodiscovery continues working for all existing platforms"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/clickhouse_cloud.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_clickhouse_adapter.py"}], "state": "done", "title": "ClickHouse Cloud Deployment Mode", "verifications": [{"command": "uv run -- python -c \"from benchbox.platforms.credentials.clickhouse_cloud import *; print('import OK')\"", "description": "Cloud module imports correctly", "expected": "import OK", "item_id": "clickhouse-cloud-deployment", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py -v --tb=short", "description": "ClickHouse adapter unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "clickhouse-cloud-deployment", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "all tests pass, 0 failures", "item_id": "clickhouse-cloud-deployment", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'cloud' deployment mode to ClickHouse registry metadata", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement OAuth token authentication", "wid": "w10"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add credential setup wizard for ClickHouse Cloud", "wid": "w11"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support environment variables", "wid": "w12"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "CLICKHOUSE_CLOUD_ENDPOINT", "wid": "w13"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "CLICKHOUSE_CLOUD_API_KEY", "wid": "w14"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement cloud endpoint connection", "wid": "w15"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement cloud storage upload for data loading", "wid": "w16"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support S3 staging", "wid": "w17"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support GCS staging", "wid": "w18"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w16"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement COPY FROM for cloud data", "wid": "w19"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set requires_credentials=True", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Configure result caching behavior", "wid": "w20"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for cloud authentication", "wid": "w21"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration tests with ClickHouse Cloud (requires account)", "wid": "w22"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-H benchmark at scale 0.1", "wid": "w23"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Compare results with self-hosted ClickHouse", "wid": "w24"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set requires_cloud_storage=True", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define auth_methods=['api_key', 'oauth']", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add cloud-specific configuration options", "wid": "w5"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud endpoint URL", "wid": "w6"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Organization/service ID", "wid": "w7"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "API key or OAuth token", "wid": "w8"}, {"evidence": null, "item_id": "clickhouse-cloud-deployment", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement API key authentication", "wid": "w9"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "modify base query files", "instead": "create clickhouse_sql/ overrides instead", "why": "other platforms depend on standard SQL FILTER/EXTRACT"}, {"dont": "use toHour() directly on TIME columns", "instead": "wrap in toDateTime() first", "why": "ClickHouse toHour() rejects bare TIME values"}], "approach": "Follow existing override pattern. For QSA4, mechanically replace\nSUM(x) FILTER (WHERE cond) with sumIf(x, cond). For QTM1, wrap\norder_time in toDateTime() before passing to toHour().\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "6 query failures on ClickHouse in coffeeshop benchmark:\n- QSA4 \u00d7 3: SUM(x) FILTER (WHERE cond) \u2192 ClickHouse needs sumIf(x, cond)\n- QTM1 \u00d7 3: EXTRACT(HOUR FROM order_time) where order_time is TIME type \u2192\n toHour() doesn't accept bare TIME values, needs toHour(toDateTime(order_time))\n\nPart of Pattern B (EXTRACT) and Pattern D (FILTER aggregate).", "id": "clickhouse-coffeeshop-dialect-overrides", "preserves": ["Base coffeeshop QSA4/QTM1 queries unchanged - other platforms use standard FILTER and EXTRACT", "Override queries return identical column names and semantics"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Add ClickHouse dialect overrides for coffeeshop QSA4 (sumIf) and QTM1 (TIME type)", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'coffeeshop and clickhouse'", "description": "ClickHouse coffeeshop overrides load with sumIf and toDateTime wrappers", "expected": "Tests pass confirming sumIf/toDateTime overrides for QSA4/QTM1", "item_id": "clickhouse-coffeeshop-dialect-overrides", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-coffeeshop-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for SA4 - actual issue is nested window aggregate SUM(SUM()) OVER (), replaced with CROSS JOIN subquery. TODO description was incorrect about FILTER clause.", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-coffeeshop-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for TM1 using toHour(toDateTime(order_time)) for TIME-type column", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-coffeeshop-dialect-overrides", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests: load SA4/TM1 for ClickHouse and assert SQL avoids nested window aggregate and uses toDateTime() respectively", "wid": "w3"}], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:58Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #574 of snowflake-clickhouse-open-table-format-loading.\nDeferred: ClickHouse Delta support via conversion or external tooling\nReason deferred: ClickHouse lacks native Delta support; would need Parquet conversion layer", "id": "clickhouse-delta-support-via-conversion-or-external-tooling", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "ClickHouse Delta support via conversion or external tooling", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "always capture the message", "dont": "catch exceptions without storing str(e) in the result error field", "why": "swallowed errors force manual re-runs for diagnosis"}, {"INSTEAD": "only add error message capture to existing except blocks", "dont": "change query execution flow or retry logic", "why": "this is a diagnostics-only fix"}], "approach": "Search the ClickHouse adapter for try/except blocks in query execution methods.\nLook for patterns where the exception is caught to set status=FAILED but the\nerror message is not stored. Ensure the error field in QueryResult is populated.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "The ClickHouse adapter captures failure status but swallows the exception message.\nread_primitives and tpcdi result JSONs show \"status\": \"FAILED\" with no error string,\nmaking diagnosis of query failures much harder.\n\nThe adapter's query execution error handler should always capture str(exception)\ninto the result object. This is Pattern G from the SF1 failure analysis.", "id": "clickhouse-error-message-capture", "preserves": ["Query execution flow and retry logic unchanged", "Successful queries unaffected"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Fix ClickHouse adapter to capture error messages in query results", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'clickhouse and (error_capture or error_message)'", "description": "Error messages appear in query results when queries fail", "expected": "Tests pass confirming str(exception) is stored in result.error for failed queries", "item_id": "clickhouse-error-message-capture", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-error-message-capture", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Find the ClickHouse adapter's query execution error handler - locate where exceptions are caught but messages discarded", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-error-message-capture", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ensure str(exception) is always written to the result error field", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-error-message-capture", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit test verifying error messages are captured in query results", "wid": "w3"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "create clickhouse_sql/ override instead", "dont": "modify base queries.py or default SQL files", "why": "other platforms depend on standard SQL PERCENTILE_CONT"}], "approach": "Follow existing ClickHouse override patterns in other benchmarks. Check how\nquery overrides are loaded (likely a clickhouse_sql/ subdirectory convention).\nThe SQL change is straightforward: replace PERCENTILE_CONT(p) WITHIN GROUP\n(ORDER BY col) with quantile(p)(col).\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "h2odb Q9 fails on ClickHouse with SYNTAX_ERROR because it uses standard SQL\nPERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY fare_amount). ClickHouse doesn't\nimplement ordered-set aggregate syntax.\n\nFix: Add a ClickHouse-specific Q9 override using quantile(0.5)(fare_amount) and\nquantile(0.9)(fare_amount). This is a 3-line SQL change.\n\nPart of Pattern C (PERCENTILE_CONT rewrite) which also affects read_primitives.", "id": "clickhouse-h2odb-percentile-override", "preserves": ["Base h2odb Q9 query unchanged - other platforms use standard PERCENTILE_CONT syntax", "ClickHouse override returns identical column names and semantics as the base query"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Add ClickHouse dialect override for h2odb Q9 PERCENTILE_CONT \u2192 quantile()", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'h2odb and clickhouse'", "description": "ClickHouse h2odb override loads and uses quantile() syntax", "expected": "Tests pass confirming quantile() override for Q9", "item_id": "clickhouse-h2odb-percentile-override", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-h2odb-percentile-override", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse-specific Q9 override file using quantile(0.5)(fare_amount) / quantile(0.9)(fare_amount)", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-h2odb-percentile-override", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit test that loads h2odb Q9 for ClickHouse and asserts SQL contains 'quantile(' not 'PERCENTILE_CONT'", "wid": "w2"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "keep at 50% minimum", "dont": "lower max_bytes_in_join below 0.5 \u00d7 max_memory_usage", "why": "SF1 Q5 multi-table join needs 2.72 GiB and 25% cap caused OOM"}, {"INSTEAD": "verify it takes effect", "dont": "remove grace_hash join_algorithm", "why": "grace_hash spills to disk and is the safety net when memory limits are hit"}], "approach": "Simple parameter change in tuning.py. Check the max_bytes_in_join calculation\nand restore to 50% multiplier. Then grep for join_algorithm across the ClickHouse\nadapter to verify grace_hash isn't overridden by session-level settings.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "ClickHouse SF1 Q5 fails across TPC-H, tpchavoc, and tpch_skew with:\n\"Limit for JOIN exceeded, max bytes: 2.00 GiB, current bytes: 2.72 GiB\"\n\nRoot cause: benchbox/platforms/clickhouse/tuning.py ~line 91 sets max_bytes_in_join\nto 25% of max_memory_usage. With a 2 GB default, that's 512 MB - too low for Q5's\nmulti-table join at SF1 which needs 2.72 GB.\n\nAdditionally, join_algorithm should be set to \"grace_hash\" (~line 105) which spills\nto disk and should avoid this limit entirely. Need to verify grace_hash is actually\ntaking effect and not being overridden.\n\nSingle fix resolves 9 failures across 3 benchmarks.", "id": "clickhouse-join-memory-tuning", "preserves": ["All other tuning parameters (max_memory_usage, max_threads, etc.) retain current values", "Tuning applies correctly for both SF0.01 and SF1 scales", "grace_hash join algorithm setting is not overridden by adapter session defaults"], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Fix ClickHouse JOIN memory limit - restore max_bytes_in_join to 50% and verify grace_hash", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'clickhouse and tuning and (join or memory)'", "description": "Tuning tests pass with updated max_bytes_in_join multiplier", "expected": "Tests pass confirming 0.5 multiplier and grace_hash settings", "item_id": "clickhouse-join-memory-tuning", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-join-memory-tuning", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In tuning.py, change max_bytes_in_join multiplier from 0.25 to 0.5 of max_memory_usage", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-join-memory-tuning", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify join_algorithm = 'grace_hash' is being applied - check for overrides in adapter or session settings", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-join-memory-tuning", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add/update unit tests for tuning parameter values", "wid": "w3"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "catch exceptions without storing str(e) in the result error field", "instead": "always capture the message", "why": "swallowed errors force manual re-runs for diagnosis"}, {"dont": "change query execution flow or retry logic", "instead": "only add error message capture to existing except blocks", "why": "this is a diagnostics-only fix"}], "approach": "Search the ClickHouse adapter for try/except blocks in query execution methods.\nLook for patterns where the exception is caught to set status=FAILED but the\nerror message is not stored. Ensure the error field in QueryResult is populated.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "The ClickHouse adapter captures failure status but swallows the exception message.\nread_primitives and tpcdi result JSONs show \"status\": \"FAILED\" with no error string,\nmaking diagnosis of query failures much harder.\n\nThe adapter's query execution error handler should always capture str(exception)\ninto the result object. This is Pattern G from the SF1 failure analysis.", "id": "clickhouse-error-message-capture", "preserves": ["Query execution flow and retry logic unchanged", "Successful queries unaffected"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Fix ClickHouse adapter to capture error messages in query results", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'clickhouse and (error_capture or error_message)'", "description": "Error messages appear in query results when queries fail", "expected": "Tests pass confirming str(exception) is stored in result.error for failed queries", "item_id": "clickhouse-error-message-capture", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-error-message-capture", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Find the ClickHouse adapter's query execution error handler - locate where exceptions are caught but messages discarded", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-error-message-capture", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ensure str(exception) is always written to the result error field", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-error-message-capture", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit test verifying error messages are captured in query results", "wid": "w3"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "modify base queries.py or default SQL files", "instead": "create clickhouse_sql/ override instead", "why": "other platforms depend on standard SQL PERCENTILE_CONT"}], "approach": "Follow existing ClickHouse override patterns in other benchmarks. Check how\nquery overrides are loaded (likely a clickhouse_sql/ subdirectory convention).\nThe SQL change is straightforward: replace PERCENTILE_CONT(p) WITHIN GROUP\n(ORDER BY col) with quantile(p)(col).\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "h2odb Q9 fails on ClickHouse with SYNTAX_ERROR because it uses standard SQL\nPERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY fare_amount). ClickHouse doesn't\nimplement ordered-set aggregate syntax.\n\nFix: Add a ClickHouse-specific Q9 override using quantile(0.5)(fare_amount) and\nquantile(0.9)(fare_amount). This is a 3-line SQL change.\n\nPart of Pattern C (PERCENTILE_CONT rewrite) which also affects read_primitives.", "id": "clickhouse-h2odb-percentile-override", "preserves": ["Base h2odb Q9 query unchanged - other platforms use standard PERCENTILE_CONT syntax", "ClickHouse override returns identical column names and semantics as the base query"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Add ClickHouse dialect override for h2odb Q9 PERCENTILE_CONT \u2192 quantile()", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'h2odb and clickhouse'", "description": "ClickHouse h2odb override loads and uses quantile() syntax", "expected": "Tests pass confirming quantile() override for Q9", "item_id": "clickhouse-h2odb-percentile-override", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-h2odb-percentile-override", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse-specific Q9 override file using quantile(0.5)(fare_amount) / quantile(0.9)(fare_amount)", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-h2odb-percentile-override", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit test that loads h2odb Q9 for ClickHouse and asserts SQL contains 'quantile(' not 'PERCENTILE_CONT'", "wid": "w2"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "lower max_bytes_in_join below 0.5 \u00d7 max_memory_usage", "instead": "keep at 50% minimum", "why": "SF1 Q5 multi-table join needs 2.72 GiB and 25% cap caused OOM"}, {"dont": "remove grace_hash join_algorithm", "instead": "verify it takes effect", "why": "grace_hash spills to disk and is the safety net when memory limits are hit"}], "approach": "Simple parameter change in tuning.py. Check the max_bytes_in_join calculation\nand restore to 50% multiplier. Then grep for join_algorithm across the ClickHouse\nadapter to verify grace_hash isn't overridden by session-level settings.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "ClickHouse SF1 Q5 fails across TPC-H, tpchavoc, and tpch_skew with:\n\"Limit for JOIN exceeded, max bytes: 2.00 GiB, current bytes: 2.72 GiB\"\n\nRoot cause: benchbox/platforms/clickhouse/tuning.py ~line 91 sets max_bytes_in_join\nto 25% of max_memory_usage. With a 2 GB default, that's 512 MB - too low for Q5's\nmulti-table join at SF1 which needs 2.72 GB.\n\nAdditionally, join_algorithm should be set to \"grace_hash\" (~line 105) which spills\nto disk and should avoid this limit entirely. Need to verify grace_hash is actually\ntaking effect and not being overridden.\n\nSingle fix resolves 9 failures across 3 benchmarks.", "id": "clickhouse-join-memory-tuning", "preserves": ["All other tuning parameters (max_memory_usage, max_threads, etc.) retain current values", "Tuning applies correctly for both SF0.01 and SF1 scales", "grace_hash join algorithm setting is not overridden by adapter session defaults"], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Fix ClickHouse JOIN memory limit - restore max_bytes_in_join to 50% and verify grace_hash", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'clickhouse and tuning and (join or memory)'", "description": "Tuning tests pass with updated max_bytes_in_join multiplier", "expected": "Tests pass confirming 0.5 multiplier and grace_hash settings", "item_id": "clickhouse-join-memory-tuning", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-join-memory-tuning", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In tuning.py, change max_bytes_in_join multiplier from 0.25 to 0.5 of max_memory_usage", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-join-memory-tuning", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify join_algorithm = 'grace_hash' is being applied - check for overrides in adapter or session settings", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-join-memory-tuning", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add/update unit tests for tuning parameter values", "wid": "w3"}], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:54Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #561 of write-primitives-sketch-persistence-category.\nDeferred: ClickHouse-native sketch variants using -State / -Merge combinators\nReason deferred: ClickHouse uses its own serialization (uniqState / quantileTDigestState / topKState) that is not binary-compatible with DataSketches. Adding ClickHouse-native variants is valuable but is a scope expansion that should land as a follow-up TODO once the DataSketches-portable core works.", "id": "clickhouse-native-sketch-variants-using-state-merge-combinat", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "ClickHouse-native sketch variants using -State / -Merge combinators", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "create clickhouse_sql/ overrides instead", "dont": "modify base query files", "why": "other platforms depend on standard SQL EXTRACT"}, {"INSTEAD": "apply % 7 adjustment", "dont": "use toDayOfWeek() directly as DOW replacement", "why": "ClickHouse returns 1=Monday while SQL standard DOW is 0=Sunday"}], "approach": "Check the existing nyctaxi query structure for override patterns. Create\nclickhouse_sql/ subdirectory with 3 query files using ClickHouse-native\ndate/time functions. The SQL logic stays the same; only the function calls change.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "12 query failures across nyctaxi on ClickHouse due to unsupported EXTRACT variants:\n- rush-hour-analysis: EXTRACT(HOUR FROM ts) and EXTRACT(EPOCH FROM interval)\n- weekday-weekend-comparison: EXTRACT(DOW FROM ts)\n- trips-by-day-of-week: EXTRACT(DOW FROM ts)\n\nClickHouse equivalents:\n- EXTRACT(HOUR FROM ts) \u2192 toHour(ts)\n- EXTRACT(DOW FROM ts) \u2192 toDayOfWeek(ts) % 7\n- EXTRACT(EPOCH FROM (ts2 - ts1)) \u2192 dateDiff('second', ts1, ts2)\n\nPart of Pattern B (EXTRACT rewrite) which also affects coffeeshop.", "id": "clickhouse-nyctaxi-extract-overrides", "preserves": ["Base nyctaxi queries unchanged - other platforms use standard EXTRACT syntax", "Override queries return identical column names, types, and row semantics"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Add ClickHouse dialect overrides for nyctaxi EXTRACT functions", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'nyctaxi and clickhouse'", "description": "ClickHouse nyctaxi overrides load and use native date functions", "expected": "Tests pass confirming toHour/toDayOfWeek/dateDiff overrides", "item_id": "clickhouse-nyctaxi-extract-overrides", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for rush-hour-analysis query using toHour() and dateDiff('second', ...)", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for weekday-weekend-comparison using toDayOfWeek() % 7", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for trips-by-day-of-week using toDayOfWeek() % 7", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify all 3 overrides are loaded correctly by the query manager", "wid": "w4"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "mark fixable queries as expected_failure", "why": "dialect overrides are preferred over skipping"}, {"INSTEAD": "create clickhouse_sql/ overrides instead", "dont": "modify base query files", "why": "other platforms depend on standard SQL"}, {"INSTEAD": "(not specified)", "dont": "use skip_on for runtime-only timeout risk", "why": "reserve skip_on for queries that are genuinely unsupported on that dialect"}], "approach": "Categorize each of the 22 failing queries into: fixable (dialect override),\nexpected_failure (platform limitation), or needs-investigation. Work through\nfixable queries first using ClickHouse override patterns. Window function and\nQUALIFY fixes may require subquery rewrites rather than simple function substitutions.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:10Z", "from_item": "clickhouse-read-primitives-dialect-fixes", "id": 577, "reason": "Tracked separately in clickhouse-error-message-capture TODO", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Target TODO _project/DONE/platform-expansion/planning/clickhouse-error-message-capture.yaml already exists and is DONE.", "summary": "Fix ClickHouse adapter error message capture (Pattern G)"}], "deps": [], "description": "64/450 query failures across 22 unique queries in read_primitives on ClickHouse.\nThese break into several categories:\n\nDialect fixes needed:\n- any_value_simple, any_value_with_filter: WHERE x = ANY(subquery) \u2192 WHERE x IN (subquery) (Pattern F)\n- array_length: LENGTH(array_col) mistranslated to lengthUTF8() \u2192 should use length()\n- statistical_percentiles, intrinsic_appx_median: PERCENTILE_CONT WITHIN GROUP \u2192 quantile() (Pattern C)\n- statistical_correlation: likely CORR() function differences\n- timeseries_trend_analysis, window_moving_frame, window_multiple_orderings: window function syntax differences\n- qualify_cume_dist: QUALIFY clause not supported in ClickHouse\n\nExpected failures (not fixable as dialect issues):\n- aggregation_groupby_large: Timeout (907s vs 300s limit)\n- fulltext_boolean_search, fulltext_phrase_search, fulltext_simple_search: full-text search syntax not portable\n\nSecondary bug: The result JSON shows no error messages for most failures. The ClickHouse\nadapter is capturing failure status but swallowing the exception message (Pattern G).", "id": "clickhouse-read-primitives-dialect-fixes", "preserves": ["Base read_primitives query files unchanged - other platforms use standard SQL", "Existing ClickHouse read_primitives results for passing queries unaffected", "Override queries return identical column names, types, and row semantics"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Fix ClickHouse read_primitives failures - dialect overrides, expected_failures, and error capture", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'read_primitives and clickhouse'", "description": "ClickHouse read_primitives overrides load and pass", "expected": "Tests pass confirming IN-subquery, length(), quantile(), window, and QUALIFY overrides", "item_id": "clickhouse-read-primitives-dialect-fixes", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/read_primitives/test_benchmark_variants.py -k aggregation_groupby_large", "description": "Timeout-only ClickHouse query remains selectable", "expected": "aggregation_groupby_large still appears in clickhouse get_queries() output", "item_id": "clickhouse-read-primitives-dialect-fixes", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix ANY(subquery) \u2192 IN (subquery) for any_value_simple and any_value_with_filter queries", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix array_length LENGTH() \u2192 length() (not lengthUTF8) for array columns", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add PERCENTILE_CONT \u2192 quantile() overrides for statistical_percentiles and intrinsic_appx_median", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep working fulltext_* ClickHouse variants enabled and document aggregation_groupby_large as a timeout-risk query without hard-skipping it", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix window function syntax failures: timeseries_trend_analysis, window_moving_frame, window_multiple_orderings - adjust frame specs and ORDER BY for ClickHouse", "wid": "w5"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix QUALIFY clause failures: qualify_cume_dist - rewrite as subquery with WHERE on window result", "wid": "w6"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix statistical function failures: statistical_correlation - investigate CORR() syntax differences", "wid": "w7"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify all overrides and expected_failure markers", "wid": "w8"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "apply after SQLGlot", "dont": "apply JULIANDAY\u2192dateDiff before SQLGlot translation", "why": "SQLGlot reorders dateDiff() arguments"}, {"INSTEAD": "create ClickHouse overrides instead", "dont": "modify the base query_analytics.py queries", "why": "other platforms use them correctly"}, {"INSTEAD": "use dateDiff('day', ...) instead", "dont": "use JULIANDAY() in ClickHouse overrides", "why": "it is a SQLite-only function"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "21/114 query failures on ClickHouse in TPC-DI benchmark across 7 unique queries:\n- AQ6: SUM(SUM(x)) OVER () nested window aggregate unsupported in ClickHouse\n- AQ7, AQ8, AQ10: JULIANDAY() SQLite function not portable\n- EQ7: FROM (VALUES (0)) not supported in ClickHouse FROM clause\n- EQ3: Double-nested COUNT scalar subquery (always returns 1, not actual count)\n\nFixes applied:\n- AQ6: Full query override with CROSS JOIN pre-computed grand total\n- AQ7/AQ8/AQ10: JULIANDAY(x)-JULIANDAY(y) \u2192 dateDiff('day', y, x), DATE('now') \u2192 today()\n- EQ7: FROM VALUES (N) AS alias \u2192 FROM (SELECT N) AS alias (post-SQLGlot regex)\n- EQ3: Flatten double-nested COUNT(*) FROM (SELECT COUNT(*) ...) (pre-SQLGlot regex)\n\nKey insight: JULIANDAY replacements applied AFTER SQLGlot translation because SQLGlot\nreorders dateDiff() argument order when parsing from netezza dialect.", "id": "clickhouse-tpcdi-dialect-overrides", "preserves": ["Base query_analytics.py queries unchanged - other platforms use standard SQL JULIANDAY/NULLIF/FILTER syntax", "Override queries return identical column names, types, and row semantics"], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add ClickHouse dialect overrides for TPC-DI analytics queries (JULIANDAY, NULLIF, FILTER, subqueries)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_tpcdi_clickhouse_overrides.py -v", "description": "ClickHouse TPC-DI overrides load and use native syntax", "expected": "11 passed", "item_id": "clickhouse-tpcdi-dialect-overrides", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse overrides for AQ7, AQ8, AQ10 - replace JULIANDAY/DATE('now') with dateDiff/today()", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for AQ5 - rewrite NULLIF inside aggregates to if() expressions", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for AQ6 - replace SUM(SUM()) OVER () with CROSS JOIN total", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Investigate and create ClickHouse overrides for EQ3, EQ7 - VALUES and double-nested COUNT", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify all overrides load and pass syntax validation", "wid": "w5"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "catch and swallow all exceptions in query generation", "why": "individual query failures are acceptable but total failure must be surfaced"}, {"INSTEAD": "raise a clear error instead", "dont": "silently return empty results", "why": "this masks dsqgen failures and produces misleading 'COMPLETED' status"}], "approach": "Start with diagnostic (w1) to understand whether dsqgen works locally on DuckDB.\nIf DuckDB succeeds, the issue is ClickHouse-adapter-specific query setup.\nIf DuckDB also fails, the dsqgen binary is the root cause.\n\nFor the code fix, add a guard in power_test.py's _run_power_test() method:\nafter query generation, check len(queries) == 0 and raise a descriptive error\nrather than iterating over an empty list. In queries.py, track per-query failures\nand raise when all queries fail.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "ClickHouse SF1 runs for tpcds and tpcds_obt show power_test status \"COMPLETED\" with\nduration 0ms and an empty queries list. The phase silently swallows a query generation\nfailure, reporting success when zero queries actually executed.\n\nTwo candidate root causes:\n\nPath A (most likely): dsqgen binary missing/failing silently. TPCDSQueryManager.get_all_queries()\nin benchbox/core/tpcds/queries.py catches dsqgen exceptions per-query and skips them. If all 99\nfail, it returns an empty dict. The power test runner iterates over zero queries and reports\nCOMPLETED.\n\nPath B: _preflight_validate_generation() in power_test.py raises RuntimeError, caught at the\nphase level, returning an empty result list.\n\nVerification showed tpcds_obt did not share the same root cause. Base tpcds was fixed by\nsurfacing zero-query dsqgen failures, while tpcds_obt required adapter routing changes so it\nruns through the generic OBT power-test path instead of the specialized TPC-DS stream runner.\n\nThis is the highest-priority fix because it blocks ~195 queries across both benchmarks and\nmasks the actual root cause (likely dsqgen binary availability on the ClickHouse Cloud host).", "id": "clickhouse-tpcds-silent-empty-query-bug", "preserves": ["Power test phase still completes normally when queries are present", "TPCDSQueryManager.get_all_queries() continues to work when dsqgen succeeds for all queries", "tpcds_obt executes its OBT query set via the generic adapter path instead of the base TPC-DS runner"], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "Fix TPC-DS silent empty-query bug - surface FAILED when power test returns 0 queries", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcds/ -v -k 'zero_quer'", "description": "Zero-query generation raises error instead of silent completion", "expected": "Tests pass showing FAILED status when 0 queries generated", "item_id": "clickhouse-tpcds-silent-empty-query-bug", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Diagnose: run `benchbox run --platform duckdb --benchmark tpcds --scale 0.01 --phases power` to determine if dsqgen works locally", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In benchbox/core/tpcds/queries.py, add warning logging when dsqgen fails for individual queries, and raise error when total success count is zero", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In power_test.py, detect 0-query generation and set phase status to FAILED with descriptive error message. Also fixed schema.py to derive power_test phase status from real executions.", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for zero-query detection in both queries.py and power_test.py", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run ClickHouse tpcds and tpcds_obt at SF1 to verify fix surfaces the actual error", "wid": "w5"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "only transform function names and syntax", "dont": "apply numeric normalization that changes the string representation of valid SQL literals", "why": "downstream engines may have strict parsing"}, {"INSTEAD": "preserve '0.06' form", "dont": "strip leading zeros from decimal literals", "why": "ClickHouse rejects bare '.06' as invalid syntax"}], "approach": "Search the ClickHouse adapter for QueryTransformer or query rewriting logic.\nThe bug likely involves a regex or string replacement that inadvertently strips\nleading zeros from decimal numbers like 0.06 \u2192 .06. Check query_transformer.py\nfirst, then adapter.py for any pre-execution SQL munging.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "tpchavoc Q6 fails on ClickHouse with syntax error on \".06 - 0.01\" - the source\nin benchbox/core/tpchavoc/variant_sets/q06.py contains \"0.06\" (with leading zero),\nbut a query transformation step strips the leading zero, producing \".06\" which is\ninvalid ClickHouse syntax.\n\nRoot cause: fix_type_casts Patterns 2/3/4 use (0\\.0*)\\b which backtracks to\nmatch \"0.\" from \"0.06\" via a word boundary between \".\" (non-word) and the next\ndigit. This leaves the trailing digit(s) dangling after the CAST replacement,\ne.g. \"col + 0.06\" \u2192 \"col + CAST(0 AS Decimal(15,2))6\".\n\nFix: replaced \\b(?!\\)) with (?![0-9\\)]) in Pattern 2, and \\b with (?![0-9]) in\nPatterns 3/4, to prevent matching when the zero literal is followed by a digit.", "id": "clickhouse-tpchavoc-q6-decimal-bug", "preserves": ["Other numeric literal transformations continue to work correctly", "Queries that don't use small decimals remain unaffected"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Fix ClickHouse QueryTransformer stripping leading zeros from decimal literals (tpchavoc Q6)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py::TestClickHouseQueryTransformerDecimalLiterals -v", "description": "Decimal literal 0.06 preserved through transformation", "expected": "6 passed", "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Find the ClickHouse QueryTransformer code that handles numeric literals - grep for decimal/numeric transformation logic", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the leading-zero stripping bug in the transformer", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit test with decimal literals like 0.06 to prevent regression", "wid": "w3"}], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "keep it explicitly unsupported or explicitly out of the default wheel.", "dont": "add `benchbox.experimental` to public exports or benchmark registries to make the packaging policy feel simpler", "why": "that would silently expand the supported Beta surface"}, {"INSTEAD": "either change packaging or change docs, but not one without the other.", "dont": "claim DuckDB is part of the default install unless `[project.dependencies]` actually says so", "why": "users will hit import/runtime mismatches"}, {"INSTEAD": "ratchet the release-critical entrypoints first and open follow-up work for the rest.", "dont": "try to zero the entire repository-wide `ty` backlog inside this item", "why": "that turns a focused release blocker into an unbounded cleanup"}, {"INSTEAD": "update all canonical current-state docs in the same pass.", "dont": "update only `README.md`", "why": "the FAQ, installation guide, and version-reporting surfaces will drift again"}, {"INSTEAD": "update `pyproject.toml` packaging config instead.", "dont": "use `MANIFEST.in` as the mechanism for wheel package exclusion", "why": "wheels are controlled by setuptools package discovery/package-data"}], "approach": "Use the 2026-04-01 review findings as the input set, but ground every change\nin existing local decisions:\n- `move-duckdb-to-optional-extras`\n- `isolate-experimental-core-subsystems`\n- `remove-release-tooling-from-wheel`\n\nResolve policy/documentation questions first, then implement any packaging\nboundary changes, then clean the targeted `ty` diagnostics, then ratchet\nregression tests. Prefer extending existing release/version/package tests over\ninventing one-off scripts.\n", "blocked_reason": null, "category": "Release Readiness", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "close-pre-beta-release-blockers", "id": 118, "reason": "Useful follow-up work, but larger than a Beta-release blocker once the critical gate is ratcheted.", "resolution": "promoted", "resolved_item": "burn-down-the-full-repository-wide-ty-backlog-outside-releas", "resolved_reason": null, "summary": "Burn down the full repository-wide `ty` backlog outside release-critical entrypoints"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "close-pre-beta-release-blockers", "id": 119, "reason": "Potential long-term architecture change; only do this after the immediate Beta packaging policy is stabilized.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on the default-wheel boundary 'remaining contentious'; Beta packaging policy has since stabilized (item done).", "summary": "Extract benchbox.experimental into a separate distribution if the default-wheel boundary remains contentious"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "close-pre-beta-release-blockers", "id": 120, "reason": "Current-state docs and package artifacts are the immediate Beta blocker; historical content can follow.", "resolution": "promoted", "resolved_item": "audit-blog-posts-and-historical-release-summaries-for-outdat", "resolved_reason": null, "summary": "Audit blog posts and historical release summaries for outdated Alpha/install language"}], "deps": [], "description": "The 2026-04-01 whole-project pre-Beta review found four release-surface gaps\nthat should be closed before BenchBox is presented as Beta software:\n\n1. Package metadata already says Beta, but public docs still say Alpha.\n2. Install docs contradict the actual dependency model after DuckDB moved to\n optional extras.\n3. The default wheel still ships the unsupported `benchbox.experimental`\n namespace without a fully explicit Beta packaging/support policy.\n4. `uv run ty check` has too much standing noise to act as a meaningful\n release gate, including warnings on the final execution path in\n `benchbox/cli/commands/run.py`.\n\nThis item is about release coherence, not runtime break/fix work. Lint, fast\ntests, the wheel/sdist build, and CLI help/version commands are already\npassing. The remaining blockers are policy drift, packaging ambiguity, and\nrelease-critical static-analysis noise.\n\nThe work must reconcile two completed historical decisions with the current\nBeta goal:\n\n- `move-duckdb-to-optional-extras` intentionally made DuckDB non-default.\n- `isolate-experimental-core-subsystems` intentionally moved prototypes into\n `benchbox.experimental` but kept them importable from the source tree.\n\nThe outcome of this item must be a single coherent Beta support contract\nacross `pyproject.toml`, canonical docs, built artifacts, and release\nverification commands.", "id": "close-pre-beta-release-blockers", "preserves": ["The completed `move-duckdb-to-optional-extras` rationale is not accidentally reversed by docs-only edits; packaging and docs must agree.", "`benchbox.experimental` stays out of `benchbox.__all__` and benchmark registries regardless of whether it remains in or leaves the default wheel.", "`benchbox.release` remains excluded from the default wheel.", "`uv run -- benchbox --help`, `uv run -- benchbox --version`, and `uv run -- benchbox run --help` continue to exit 0 and expose the current command set.", "`uv run -- python -m pytest -m fast -q` remains green after any release-surface cleanup."], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "Benchmark query corpora under `benchbox/core/*/queries*` and `benchbox/core/*/dataframe_queries*`"}, {"kind": "do_not_modify", "path_glob": "Benchmark result schema and exported result format contracts"}, {"kind": "do_not_modify", "path_glob": "Platform adapter behavior unrelated to packaging/status/install/typecheck cleanup"}, {"kind": "do_not_modify", "path_glob": "Supported benchmark registry membership for non-experimental benchmarks"}, {"kind": "only_modify", "path_glob": "README.md"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/experimental/"}, {"kind": "only_modify", "path_glob": "docs/README.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/duckdb.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/platform-selection-guide.md"}, {"kind": "only_modify", "path_glob": "docs/usage/faq.md"}, {"kind": "only_modify", "path_glob": "docs/usage/getting-started.md"}, {"kind": "only_modify", "path_glob": "docs/usage/installation.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_package_exports.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_version_command.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_exclusion_invariants.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_infrastructure.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_version_management.py"}, {"kind": "only_modify", "path_glob": "tests/unit/utils/test_version_markers.py"}], "state": "done", "title": "Close pre-Beta release blockers in status messaging, install docs, packaging policy, and typecheck gating", "verifications": [{"command": "rg -n \"Status-Alpha|alpha software|Alpha Software|works with DuckDB out of the box|automatically installed as a dependency\" README.md docs/README.md docs/usage/faq.md docs/usage/installation.md docs/platforms/platform-selection-guide.md docs/platforms/duckdb.md", "description": "Current-state docs no longer contain stale Alpha/default-DuckDB claims", "expected": "No matches for stale current-state status/install claims", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run ty check benchbox/cli/commands/run.py benchbox/base.py benchbox/__init__.py", "description": "Release-critical entrypoints are clean under ty", "expected": "Found 0 diagnostics", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/cli/test_cli_package_exports.py tests/unit/cli/test_cli_version_command.py tests/unit/test_version_management.py tests/unit/utils/test_version_markers.py tests/unit/test_release_exclusion_invariants.py tests/unit/test_release_infrastructure.py -q", "description": "Version/package/release regression tests pass", "expected": "All selected tests pass", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv build --sdist --wheel && unzip -l dist/benchbox-*.whl | rg 'benchbox/(experimental|release)/'", "description": "Default distribution contents match the documented Beta support policy", "expected": "`benchbox/release` absent; `benchbox/experimental` presence or absence matches the documented policy and its tests", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -m fast -q && uv run -- benchbox --help && uv run -- benchbox --version && uv run -- benchbox run --help", "description": "Overall smoke gate remains green", "expected": "Fast suite passes and all CLI help/version commands exit 0", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": [], "notes": "Inventory every current-state claim in:\n- `pyproject.toml`\n- `README.md`\n- `docs/README.md`\n- `docs/usage/faq.md`\n- `docs/usage/installation.md`\n- `docs/platforms/platform-selection-guide.md`\n- `docs/platforms/duckdb.md`\n- `uv run -- benchbox --version`\n- `uv run -- benchbox --help`\n\nCapture, in one working matrix, the current answers to:\n- Is BenchBox Alpha or Beta right now?\n- What benchmark/platform counts are publicly claimed?\n- Does plain `uv add benchbox` imply DuckDB support?\n- Is `benchbox.experimental` part of the default wheel contract?\n- Which existing tests already guard these claims?\n\nThis is the source-of-truth step. No subsequent work unit should invent\npolicy independently.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a canonical Beta release matrix for status, install surface, scope counts, and wheel boundaries", "wid": "w1"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w8", "w9"], "notes": "Run and capture the final gate:\n- `uv run ruff check .`\n- `uv run ty check benchbox/cli/commands/run.py benchbox/base.py benchbox/__init__.py`\n- `uv run -- python -m pytest -m fast -q`\n- `uv build --sdist --wheel`\n- wheel inspection for `benchbox/release` and `benchbox/experimental`\n- `uv run -- benchbox --help`\n- `uv run -- benchbox --version`\n- `uv run -- benchbox run --help`\n\nClose this item only when public docs, metadata, wheel contents, and\ntargeted type checking all agree with the chosen Beta support policy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the final pre-Beta verification matrix and record the release decision", "wid": "w10"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Align release status and scope wording across:\n- `pyproject.toml` classifier(s)\n- README badge, status section, and contribution/release language\n- `docs/README.md`\n- `docs/usage/faq.md`\n- any other current-state entry docs found in w1\n\nRemove stale Alpha wording and outdated benchmark/platform count claims.\nLimit follow-on doc edits to canonical current-state entry points, not\nhistorical release notes, blog posts, or archived announcements.\nIf the repo should stay Alpha until the final verification pass, revert\nall surfaces together first and flip all surfaces together at the end.\nMixed Alpha/Beta state is not acceptable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Resolve alpha/beta and benchmark/platform-count drift across metadata and canonical docs", "wid": "w2"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Reconcile all install guidance with the completed\n`move-duckdb-to-optional-extras` decision.\n\nAudit and fix contradictory wording in:\n- `README.md`\n- `docs/usage/faq.md`\n- `docs/usage/installation.md`\n- `docs/platforms/platform-selection-guide.md`\n- `docs/platforms/duckdb.md`\n- quick-start/getting-started pages surfaced by w1 if they are canonical\n current-state install entry points under `docs/` or `README.md`\n\nDecide explicitly between:\n- keeping DuckDB optional and documenting `--extra duckdb` clearly, or\n- intentionally restoring DuckDB to the default dependency surface and\n updating packaging/tests accordingly.\n\nDo not ship hybrid wording such as \"works out of the box\" while\n`pyproject.toml` still keeps DuckDB optional.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Repair install documentation to match the extras-based DuckDB model or intentionally change the dependency surface", "wid": "w3"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Revisit the current outcome of\n`isolate-experimental-core-subsystems` and choose one Beta-safe contract:\n\n- Option A: `benchbox.experimental` remains in the default wheel as an\n explicitly unsupported namespace, documented as outside the supported\n product surface.\n- Option B: `benchbox.experimental` is removed from the default wheel or\n otherwise moved behind an explicit opt-in distribution/install path.\n\nThe chosen contract must cover source-tree imports, wheel installs,\npackage exports, benchmark registries, docs, and regression tests. This\nwork unit is the decision point only: record the chosen contract, why the\nother option was rejected, and the concrete implementation surfaces that\nmust change next.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose the Beta packaging and support contract for benchbox.experimental", "wid": "w4"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w4"], "notes": "Update only the canonical release surfaces implicated by the w4 decision\nso the support boundary is explicit before packaging/test changes land.\n\nCandidate surfaces:\n- `README.md`\n- `docs/README.md`\n- `docs/usage/faq.md`\n- `pyproject.toml` classifiers or package metadata comments if they are\n the canonical place to express the support contract\n\nDo not widen this into a repo-wide prose sweep. The goal is a documented,\ncurrent-state Beta contract for the shipped package.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the chosen benchbox.experimental contract on canonical release surfaces", "wid": "w5"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w4"], "notes": "If Option B is chosen, update setuptools package discovery in\n`pyproject.toml` and any wheel/package tests. Do not use `MANIFEST.in`\nfor wheel exclusion; wheel content is controlled by setuptools package\ndiscovery/package-data, not MANIFEST pruning.\n\nIf Option A is chosen, make the support boundary explicit in docs and\ntests so the shipped namespace is clearly unsupported but intentionally\npresent.\n\nIn both cases:\n- keep `benchbox.release` excluded from the default wheel\n- ensure `benchbox.experimental` does not leak into `benchbox.__all__`\n or benchmark registries\n- verify wheel contents match the documented policy\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the experimental-boundary packaging decision and add wheel-level invariants", "wid": "w6"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Make `ty` actionable on release-critical source files by cleaning the\ndiagnostics in:\n- `benchbox/cli/commands/run.py`\n- `benchbox/base.py`\n- `benchbox/__init__.py`\n\nRequired outcomes:\n- no possibly-unresolved `database_config` / `benchmark_config` flow in\n the final orchestration/export/save path\n- no unused blanket `type: ignore` suppressions in these files\n- control flow in `run.py` is statically obvious enough to support a\n targeted release gate\n\nThis is not a mandate to fix all 1200+ repository diagnostics in one\nitem. The immediate goal is zero noise on release-critical entrypoints.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reduce ty diagnostics to zero for Beta-critical entrypoints", "wid": "w7"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w2", "w3", "w5", "w6"], "notes": "Extend existing tests before adding new one-off scripts.\nCandidate targets:\n- `tests/unit/cli/test_cli_package_exports.py`\n- `tests/unit/cli/test_cli_version_command.py`\n- `tests/unit/test_version_management.py`\n- `tests/unit/utils/test_version_markers.py`\n- `tests/unit/test_release_exclusion_invariants.py`\n- `tests/unit/test_release_infrastructure.py`\n\nAdd assertions for:\n- consistent status wording across metadata/canonical docs\n- install docs not claiming default DuckDB unless packaging says so\n- wheel contents matching the chosen `benchbox.experimental` policy\n- public package exports matching the documented support boundary\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ratchet regression tests so status/install/packaging drift cannot recur", "wid": "w8"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w7"], "notes": "Keep the release-critical typecheck gate small, explicit, and durable.\n\nCandidate targets:\n- extend an existing release-infrastructure test if it already shells out\n to project verification commands\n- otherwise add a narrowly scoped regression test that asserts the\n release-critical `ty` invocation stays clean on:\n - `benchbox/cli/commands/run.py`\n - `benchbox/base.py`\n - `benchbox/__init__.py`\n\nDo not expand this into a repository-wide typecheck policy change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a targeted ty regression gate for the Beta-critical entrypoints", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "collapse apply_unified_tuning or close_connection into the same PR as the load-loop extraction", "why": "each is a separate near-duplicate with its own risk surface; keep them as deferred follow-ups."}, {"INSTEAD": "(not specified)", "dont": "force bigquery/databricks onto the shared skeleton if their staging model does not actually match", "why": "a forced-fit abstraction is worse than two adapters with an honest small duplication."}, {"INSTEAD": "(not specified)", "dont": "pick the casing convention without grepping downstream consumers of table_stats first", "why": "a blind choice can silently break result reporting for whichever platform's casing changes."}], "approach": "Extract the skeleton (w1) before touching either adapter, so the interface\nis designed against both existing implementations' actual needs rather\nthan generalized from just one. Resolve the casing question (w2) with a\ngrep across the codebase for table_stats consumers before choosing, since\nthis is the one place a wrong choice silently breaks a caller. Migrate\nredshift and snowflake together in one PR (w3) so the drift is fixed\natomically rather than leaving one platform on the old pattern.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cloud-adapter-staged-load-template", "id": 13, "reason": "Same duplication pattern as the load loop but a separate method; sequence after the staged-load template proves the extraction pattern works, to avoid landing two large simultaneous refactors in one PR.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Deliberately sequenced follow-up under the same still-open staged-load-template item; not a separate untracked task. (tracked by cloud-adapter-staged-load-template)", "summary": "Deduplicate apply_unified_tuning (redshift.py:2637 vs snowflake.py:1748)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cloud-adapter-staged-load-template", "id": 14, "reason": "Smaller and lower-risk than the load loop; a good follow-up once platforms/base has an established place for shared connection-lifecycle helpers (see benchbox/platforms/base/connection_lifecycle.py, which may already be the right home).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Deferral is a note on its own from_item (cloud-adapter-staged-load-template), which is itself still open and planning this exact work.", "summary": "Deduplicate the triplicated close_connection (redshift.py:2437, snowflake.py:1592, databricks/adapter.py:2288)"}], "deps": [], "description": "Architecture review (Work Area E, structural remediation) found that\n`benchbox/platforms/redshift.py:load_data` (starts at line 1794) and\n`benchbox/platforms/snowflake.py:load_data` (starts at line 762)\nindependently implement the same staged-load orchestration loop - resolve\ndata files per table, upload/stage, run the platform load command, record\nper-table row counts, log success/failure - with behavioral drift between\nthe two copies rather than a shared abstraction:\n\n- Key casing: redshift keys `table_stats` by `table_name.lower()`\n (redshift.py:1817, 1828, 1838, 1849, 1859, 1870); snowflake keys by\n `table_name.upper()` (snowflake.py:791, 806, 816).\n- Error reporting on failure: redshift logs the full error text\n (`error_message = str(e) or repr(e) or type(e).__name__`, then\n `self.logger.error(f\"Failed to load {table_name}: {error_message}\")`);\n snowflake truncates to `str(e)[:100]`\n (`self.logger.error(f\"Failed to load {table_name}: {str(e)[:100]}...\")`).\n Same failure class, different diagnostic fidelity depending on which\n platform hit it.\n\nSimilar near-identical duplication exists elsewhere in the same two files\nplus databricks: `apply_unified_tuning` (redshift.py:2637 vs\nsnowflake.py:1748) and `close_connection`, which is actually triplicated\n(redshift.py:2437, snowflake.py:1592,\n`benchbox/platforms/databricks/adapter.py:2288`). This TODO focuses on the\nstaged-load loop first since it has observable behavioral drift (the key\ncasing difference could already be causing lookup bugs for any caller that\nassumes a fixed case); the other duplications are cheaper copy-align fixes\nonce the load-loop template exists to model them on.\n\nThis is a targeted refactor, not a shrink-campaign item -\n`_project/goal-shrink-core-code.md` explicitly scopes \"dual-impl collapses\nthat trade correctness risk for lines\" out of the autonomous shrink loop;\nthis work is done deliberately with adapter test coverage as the safety\nnet, not for line-count credit.\n", "id": "cloud-adapter-staged-load-template", "preserves": ["Existing result-capture and reporting code that reads table_stats keys continues to find its expected keys after the casing normalization in w2.", "Per-platform load behavior (COPY vs PUT+COPY INTO, S3 staging vs Snowflake stage, file-format setup) verified by existing adapter tests continues to work identically after migration.", "table_stats total row counts and the load_data return contract (tuple[dict[str, int], float, dict[str, Any] | None]) from benchbox/platforms/base/adapter.py:323 are unchanged."], "prior_art": [{"concept": "benchbox/platforms/base/adapter.py:323 load_data abstract method - the existing contract both adapters implement; the new skeleton must satisfy this same signature/return contract.", "decision": "reuse", "path": "benchbox/platforms/base/adapter.py:323"}, {"concept": "benchbox/platforms/base/connection_lifecycle.py - likely existing home for the deferred close_connection deduplication; check before creating a new module for that follow-up.", "decision": "reuse", "path": "benchbox/platforms/base/connection_lifecycle.py"}, {"concept": "benchbox/platforms/base/data_loading.py - existing shared data-loading helpers (COPY/CTAS loaders with trailing-delimiter handling per datafusion-tbl-trailing-delimiter-load) already used as a common base by other adapters; extend this file's pattern rather than starting a parallel shared-helpers module.", "decision": "extend", "path": "benchbox/platforms/base/data_loading.py"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/staged_load.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/bigquery.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/redshift.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/snowflake.py"}], "state": "planning", "title": "Extract a shared staged-load template method for redshift/snowflake (and evaluate bigquery/databricks)", "verifications": [{"command": "uv run -- python -m pytest tests -k 'redshift or snowflake' -m 'not slow and not stress' --tb=short", "description": "Redshift and Snowflake adapter load-data tests pass after migration to the shared skeleton", "expected": "all redshift/snowflake adapter tests pass", "item_id": "cloud-adapter-staged-load-template", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'table_stats\\[' benchbox/platforms/redshift.py benchbox/platforms/snowflake.py", "description": "table_stats casing convention is consistent and documented", "expected": "both files use the same .lower() or .upper() convention on table_stats keys", "item_id": "cloud-adapter-staged-load-template", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": [], "notes": "Add a shared skeleton to `benchbox/platforms/base/data_loading.py` (or\na new `benchbox/platforms/base/staged_load.py` if data_loading.py is\nalready large) implementing: resolve data files via\n`_resolve_data_files` -> per-table hook (upload/stage + platform-specific\nload command, e.g. COPY/PUT+COPY INTO) -> collect stats -> aggregate\ntiming. The per-table load command and any platform-specific setup\n(Redshift's S3 client + external schema, Snowflake's file formats +\nquery tag) stay as abstract hooks the subclass implements; only the\nloop/bookkeeping/logging shape is shared.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extract a template-method staged-load skeleton in platforms/base", "wid": "w1"}, {"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": ["w1"], "notes": "Pick one casing convention (lower or upper) for the shared skeleton's\n`table_stats` dict keys. Check every caller of `load_data`'s returned\n`table_stats` (result capture, reporting, validation) for any\ncase-sensitive lookup before picking, since changing either platform's\ncasing could silently break a caller that currently works only because\nof that platform's specific case. Document the chosen convention in\nthe base class docstring so future adapters do not reintroduce drift.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Normalize table_stats key casing and document the choice", "wid": "w2"}, {"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": ["w2"], "notes": "Reimplement `RedshiftAdapter.load_data` and `SnowflakeAdapter.load_data`\nas thin per-table hooks over the new base skeleton, removing the\nduplicated loop/logging code. Normalize the error-logging behavior to\none policy (recommend: log full error text like redshift currently\ndoes, since truncating to 100 chars as snowflake does can hide the\nactionable part of a load error) - state the choice and rationale in\nthe PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Migrate redshift.py and snowflake.py to the shared skeleton", "wid": "w3"}, {"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": ["w3"], "notes": "After redshift+snowflake prove the template out, check whether\n`benchbox/platforms/bigquery.py` and\n`benchbox/platforms/databricks/adapter.py` have a load_data loop\nsimilar enough to migrate, or whether their staging model (e.g.\nDatabricks' different ingestion path) genuinely differs enough that\nforcing them onto the same skeleton would hurt readability. Migrate\nonly if it is a clean fit; otherwise record why they are excluded.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Evaluate bigquery/databricks for the same template", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "require platform-specific assertions", "dont": "mark a platform as supported without adapter-level tests", "why": "dry-run regressions would be invisible"}, {"INSTEAD": "add a dedicated sorted-ingestion strategy control", "dont": "overload existing CLUSTERING/Z_ORDERING semantics", "why": "users need separate A/B dimensions"}, {"INSTEAD": "require explicit opt-in", "dont": "silently switch default cloud behavior to deep sort", "why": "it can materially change runtime/cost"}], "approach": "Build on PlatformAdapter.apply_ctas_sort and existing cloud adapter tuning hooks,\nbut add an explicit strategy layer so sorted ingestion can be requested even on\nplatforms previously marked as CTAS-sort unsupported.\n\nStart with strategy selection and capability gating, then implement platform\nexecution paths one adapter at a time. Record selected strategy and overhead\nmetrics in results for downstream comparison analysis.\n\nKeep all cloud-platform behavior opt-in for safety and to preserve current defaults.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [], "deps": ["ctas-sort-cloud-platforms", "ctas-sort-dataloader-connectivity"], "description": "Add explicit sorted data ingestion support for cloud data warehouse platforms as\na first-class benchmark strategy, even when platform-native tuning features\n(clustering keys, Z-ordering, partitioning, sort keys) already exist.\n\nRationale:\n- Deep sorting is an alternative physical layout strategy, not only a fallback.\n- Users need apples-to-apples performance and cost comparisons between:\n 1) deep sorted ingestion,\n 2) platform-native tuning features,\n 3) combined strategies where safe/valid.\n- Current cloud CTAS-sort decisions default to \"unsupported/no-op\" for several\n platforms, which blocks this experiment design.\n\nScope:\n- Add an opt-in sorted ingestion mode for cloud platforms.\n- Implement per-platform sorted ingestion execution paths with clear capability\n gates and documented behavior.\n- Capture benchmark metadata so result analysis can quantify value/cost trade-offs.", "id": "cloud-dw-sorted-data-ingestion", "preserves": ["Default behavior for existing runs remains unchanged when sorted-ingestion mode is not enabled", "Dry-run mode continues to capture all emitted SQL/actions without requiring live credentials", "Platform-native tuning behavior (Snowflake clustering, Databricks Z-ORDER, BigQuery CLUSTER BY) remains available and testable", "benchbox compare remains compatible with result schema changes"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}], "state": "done", "title": "Add sorted data ingestion support for cloud data warehouses to compare against clustering/Z-order/partitioning", "verifications": [{"command": "uv run _project/scripts/validate_todo.py _project/DONE/physical-tuning/active/cloud-dw-sorted-data-ingestion.yaml", "description": "New TODO validates against schema", "expected": "Validation passed", "item_id": "cloud-dw-sorted-data-ingestion", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_cloud_sorted_ingestion_strategy.py -v --tb=short", "description": "Cloud strategy tests pass", "expected": "all tests pass, 0 failures", "item_id": "cloud-dw-sorted-data-ingestion", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_cloud_platform_ctas_sort.py -v --tb=short", "description": "Existing cloud CTAS sort tests stay green", "expected": "all tests pass, 0 failures", "item_id": "cloud-dw-sorted-data-ingestion", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define sorted-ingestion strategy model and CLI/tuning config surface (force/auto/off, method selection)", "wid": "w1"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement capability matrix + guardrails for cloud platforms (Snowflake, Databricks, BigQuery, Redshift, Athena, Synapse, ClickHouse Cloud)", "wid": "w2"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Snowflake/Databricks deep-sort ingestion paths with safe defaults and explicit method markers", "wid": "w3"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement BigQuery/Redshift deep-sort ingestion paths or explicit actionable errors", "wid": "w4"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Athena/Synapse deep-sort ingestion paths or explicit actionable errors", "wid": "w5"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ClickHouse Cloud deep-sort ingestion path or explicit actionable error", "wid": "w6"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add result metadata for ingestion strategy, runtime overhead, and cost-impact comparability", "wid": "w7"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w3", "w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit/integration test coverage for strategy selection and SQL/action generation in dry-run mode", "wid": "w8"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Documentation and examples: deep-sort vs native-tuning comparison workflow", "wid": "w9"}], "worktree": "physical-tuning"} +{"anti_patterns": [{"dont": "modify base query files", "instead": "create clickhouse_sql/ overrides instead", "why": "other platforms depend on standard SQL EXTRACT"}, {"dont": "use toDayOfWeek() directly as DOW replacement", "instead": "apply % 7 adjustment", "why": "ClickHouse returns 1=Monday while SQL standard DOW is 0=Sunday"}], "approach": "Check the existing nyctaxi query structure for override patterns. Create\nclickhouse_sql/ subdirectory with 3 query files using ClickHouse-native\ndate/time functions. The SQL logic stays the same; only the function calls change.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "12 query failures across nyctaxi on ClickHouse due to unsupported EXTRACT variants:\n- rush-hour-analysis: EXTRACT(HOUR FROM ts) and EXTRACT(EPOCH FROM interval)\n- weekday-weekend-comparison: EXTRACT(DOW FROM ts)\n- trips-by-day-of-week: EXTRACT(DOW FROM ts)\n\nClickHouse equivalents:\n- EXTRACT(HOUR FROM ts) \u2192 toHour(ts)\n- EXTRACT(DOW FROM ts) \u2192 toDayOfWeek(ts) % 7\n- EXTRACT(EPOCH FROM (ts2 - ts1)) \u2192 dateDiff('second', ts1, ts2)\n\nPart of Pattern B (EXTRACT rewrite) which also affects coffeeshop.", "id": "clickhouse-nyctaxi-extract-overrides", "preserves": ["Base nyctaxi queries unchanged - other platforms use standard EXTRACT syntax", "Override queries return identical column names, types, and row semantics"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Add ClickHouse dialect overrides for nyctaxi EXTRACT functions", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'nyctaxi and clickhouse'", "description": "ClickHouse nyctaxi overrides load and use native date functions", "expected": "Tests pass confirming toHour/toDayOfWeek/dateDiff overrides", "item_id": "clickhouse-nyctaxi-extract-overrides", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for rush-hour-analysis query using toHour() and dateDiff('second', ...)", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for weekday-weekend-comparison using toDayOfWeek() % 7", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for trips-by-day-of-week using toDayOfWeek() % 7", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-nyctaxi-extract-overrides", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify all 3 overrides are loaded correctly by the query manager", "wid": "w4"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "mark fixable queries as expected_failure", "instead": "(not specified)", "why": "dialect overrides are preferred over skipping"}, {"dont": "modify base query files", "instead": "create clickhouse_sql/ overrides instead", "why": "other platforms depend on standard SQL"}, {"dont": "use skip_on for runtime-only timeout risk", "instead": "(not specified)", "why": "reserve skip_on for queries that are genuinely unsupported on that dialect"}], "approach": "Categorize each of the 22 failing queries into: fixable (dialect override),\nexpected_failure (platform limitation), or needs-investigation. Work through\nfixable queries first using ClickHouse override patterns. Window function and\nQUALIFY fixes may require subquery rewrites rather than simple function substitutions.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:10Z", "from_item": "clickhouse-read-primitives-dialect-fixes", "id": 577, "reason": "Tracked separately in clickhouse-error-message-capture TODO", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Target TODO _project/DONE/platform-expansion/planning/clickhouse-error-message-capture.yaml already exists and is DONE.", "summary": "Fix ClickHouse adapter error message capture (Pattern G)"}], "deps": [], "description": "64/450 query failures across 22 unique queries in read_primitives on ClickHouse.\nThese break into several categories:\n\nDialect fixes needed:\n- any_value_simple, any_value_with_filter: WHERE x = ANY(subquery) \u2192 WHERE x IN (subquery) (Pattern F)\n- array_length: LENGTH(array_col) mistranslated to lengthUTF8() \u2192 should use length()\n- statistical_percentiles, intrinsic_appx_median: PERCENTILE_CONT WITHIN GROUP \u2192 quantile() (Pattern C)\n- statistical_correlation: likely CORR() function differences\n- timeseries_trend_analysis, window_moving_frame, window_multiple_orderings: window function syntax differences\n- qualify_cume_dist: QUALIFY clause not supported in ClickHouse\n\nExpected failures (not fixable as dialect issues):\n- aggregation_groupby_large: Timeout (907s vs 300s limit)\n- fulltext_boolean_search, fulltext_phrase_search, fulltext_simple_search: full-text search syntax not portable\n\nSecondary bug: The result JSON shows no error messages for most failures. The ClickHouse\nadapter is capturing failure status but swallowing the exception message (Pattern G).", "id": "clickhouse-read-primitives-dialect-fixes", "preserves": ["Base read_primitives query files unchanged - other platforms use standard SQL", "Existing ClickHouse read_primitives results for passing queries unaffected", "Override queries return identical column names, types, and row semantics"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Fix ClickHouse read_primitives failures - dialect overrides, expected_failures, and error capture", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'read_primitives and clickhouse'", "description": "ClickHouse read_primitives overrides load and pass", "expected": "Tests pass confirming IN-subquery, length(), quantile(), window, and QUALIFY overrides", "item_id": "clickhouse-read-primitives-dialect-fixes", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/read_primitives/test_benchmark_variants.py -k aggregation_groupby_large", "description": "Timeout-only ClickHouse query remains selectable", "expected": "aggregation_groupby_large still appears in clickhouse get_queries() output", "item_id": "clickhouse-read-primitives-dialect-fixes", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix ANY(subquery) \u2192 IN (subquery) for any_value_simple and any_value_with_filter queries", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix array_length LENGTH() \u2192 length() (not lengthUTF8) for array columns", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add PERCENTILE_CONT \u2192 quantile() overrides for statistical_percentiles and intrinsic_appx_median", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep working fulltext_* ClickHouse variants enabled and document aggregation_groupby_large as a timeout-risk query without hard-skipping it", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix window function syntax failures: timeseries_trend_analysis, window_moving_frame, window_multiple_orderings - adjust frame specs and ORDER BY for ClickHouse", "wid": "w5"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix QUALIFY clause failures: qualify_cume_dist - rewrite as subquery with WHERE on window result", "wid": "w6"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix statistical function failures: statistical_correlation - investigate CORR() syntax differences", "wid": "w7"}, {"evidence": null, "item_id": "clickhouse-read-primitives-dialect-fixes", "needs": ["w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify all overrides and expected_failure markers", "wid": "w8"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "apply JULIANDAY\u2192dateDiff before SQLGlot translation", "instead": "apply after SQLGlot", "why": "SQLGlot reorders dateDiff() arguments"}, {"dont": "modify the base query_analytics.py queries", "instead": "create ClickHouse overrides instead", "why": "other platforms use them correctly"}, {"dont": "use JULIANDAY() in ClickHouse overrides", "instead": "use dateDiff('day', ...) instead", "why": "it is a SQLite-only function"}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "21/114 query failures on ClickHouse in TPC-DI benchmark across 7 unique queries:\n- AQ6: SUM(SUM(x)) OVER () nested window aggregate unsupported in ClickHouse\n- AQ7, AQ8, AQ10: JULIANDAY() SQLite function not portable\n- EQ7: FROM (VALUES (0)) not supported in ClickHouse FROM clause\n- EQ3: Double-nested COUNT scalar subquery (always returns 1, not actual count)\n\nFixes applied:\n- AQ6: Full query override with CROSS JOIN pre-computed grand total\n- AQ7/AQ8/AQ10: JULIANDAY(x)-JULIANDAY(y) \u2192 dateDiff('day', y, x), DATE('now') \u2192 today()\n- EQ7: FROM VALUES (N) AS alias \u2192 FROM (SELECT N) AS alias (post-SQLGlot regex)\n- EQ3: Flatten double-nested COUNT(*) FROM (SELECT COUNT(*) ...) (pre-SQLGlot regex)\n\nKey insight: JULIANDAY replacements applied AFTER SQLGlot translation because SQLGlot\nreorders dateDiff() argument order when parsing from netezza dialect.", "id": "clickhouse-tpcdi-dialect-overrides", "preserves": ["Base query_analytics.py queries unchanged - other platforms use standard SQL JULIANDAY/NULLIF/FILTER syntax", "Override queries return identical column names, types, and row semantics"], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add ClickHouse dialect overrides for TPC-DI analytics queries (JULIANDAY, NULLIF, FILTER, subqueries)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_tpcdi_clickhouse_overrides.py -v", "description": "ClickHouse TPC-DI overrides load and use native syntax", "expected": "11 passed", "item_id": "clickhouse-tpcdi-dialect-overrides", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse overrides for AQ7, AQ8, AQ10 - replace JULIANDAY/DATE('now') with dateDiff/today()", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for AQ5 - rewrite NULLIF inside aggregates to if() expressions", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ClickHouse override for AQ6 - replace SUM(SUM()) OVER () with CROSS JOIN total", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Investigate and create ClickHouse overrides for EQ3, EQ7 - VALUES and double-nested COUNT", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-tpcdi-dialect-overrides", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify all overrides load and pass syntax validation", "wid": "w5"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "catch and swallow all exceptions in query generation", "instead": "(not specified)", "why": "individual query failures are acceptable but total failure must be surfaced"}, {"dont": "silently return empty results", "instead": "raise a clear error instead", "why": "this masks dsqgen failures and produces misleading 'COMPLETED' status"}], "approach": "Start with diagnostic (w1) to understand whether dsqgen works locally on DuckDB.\nIf DuckDB succeeds, the issue is ClickHouse-adapter-specific query setup.\nIf DuckDB also fails, the dsqgen binary is the root cause.\n\nFor the code fix, add a guard in power_test.py's _run_power_test() method:\nafter query generation, check len(queries) == 0 and raise a descriptive error\nrather than iterating over an empty list. In queries.py, track per-query failures\nand raise when all queries fail.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "ClickHouse SF1 runs for tpcds and tpcds_obt show power_test status \"COMPLETED\" with\nduration 0ms and an empty queries list. The phase silently swallows a query generation\nfailure, reporting success when zero queries actually executed.\n\nTwo candidate root causes:\n\nPath A (most likely): dsqgen binary missing/failing silently. TPCDSQueryManager.get_all_queries()\nin benchbox/core/tpcds/queries.py catches dsqgen exceptions per-query and skips them. If all 99\nfail, it returns an empty dict. The power test runner iterates over zero queries and reports\nCOMPLETED.\n\nPath B: _preflight_validate_generation() in power_test.py raises RuntimeError, caught at the\nphase level, returning an empty result list.\n\nVerification showed tpcds_obt did not share the same root cause. Base tpcds was fixed by\nsurfacing zero-query dsqgen failures, while tpcds_obt required adapter routing changes so it\nruns through the generic OBT power-test path instead of the specialized TPC-DS stream runner.\n\nThis is the highest-priority fix because it blocks ~195 queries across both benchmarks and\nmasks the actual root cause (likely dsqgen binary availability on the ClickHouse Cloud host).", "id": "clickhouse-tpcds-silent-empty-query-bug", "preserves": ["Power test phase still completes normally when queries are present", "TPCDSQueryManager.get_all_queries() continues to work when dsqgen succeeds for all queries", "tpcds_obt executes its OBT query set via the generic adapter path instead of the base TPC-DS runner"], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "Fix TPC-DS silent empty-query bug - surface FAILED when power test returns 0 queries", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcds/ -v -k 'zero_quer'", "description": "Zero-query generation raises error instead of silent completion", "expected": "Tests pass showing FAILED status when 0 queries generated", "item_id": "clickhouse-tpcds-silent-empty-query-bug", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Diagnose: run `benchbox run --platform duckdb --benchmark tpcds --scale 0.01 --phases power` to determine if dsqgen works locally", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In benchbox/core/tpcds/queries.py, add warning logging when dsqgen fails for individual queries, and raise error when total success count is zero", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In power_test.py, detect 0-query generation and set phase status to FAILED with descriptive error message. Also fixed schema.py to derive power_test phase status from real executions.", "wid": "w3"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for zero-query detection in both queries.py and power_test.py", "wid": "w4"}, {"evidence": null, "item_id": "clickhouse-tpcds-silent-empty-query-bug", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run ClickHouse tpcds and tpcds_obt at SF1 to verify fix surfaces the actual error", "wid": "w5"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "apply numeric normalization that changes the string representation of valid SQL literals", "instead": "only transform function names and syntax", "why": "downstream engines may have strict parsing"}, {"dont": "strip leading zeros from decimal literals", "instead": "preserve '0.06' form", "why": "ClickHouse rejects bare '.06' as invalid syntax"}], "approach": "Search the ClickHouse adapter for QueryTransformer or query rewriting logic.\nThe bug likely involves a regex or string replacement that inadvertently strips\nleading zeros from decimal numbers like 0.06 \u2192 .06. Check query_transformer.py\nfirst, then adapter.py for any pre-execution SQL munging.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-08T00:00:00Z", "completed_pr": null, "created_at": "2026-04-08T00:00:00Z", "deferrals": [], "deps": [], "description": "tpchavoc Q6 fails on ClickHouse with syntax error on \".06 - 0.01\" - the source\nin benchbox/core/tpchavoc/variant_sets/q06.py contains \"0.06\" (with leading zero),\nbut a query transformation step strips the leading zero, producing \".06\" which is\ninvalid ClickHouse syntax.\n\nRoot cause: fix_type_casts Patterns 2/3/4 use (0\\.0*)\\b which backtracks to\nmatch \"0.\" from \"0.06\" via a word boundary between \".\" (non-word) and the next\ndigit. This leaves the trailing digit(s) dangling after the CAST replacement,\ne.g. \"col + 0.06\" \u2192 \"col + CAST(0 AS Decimal(15,2))6\".\n\nFix: replaced \\b(?!\\)) with (?![0-9\\)]) in Pattern 2, and \\b with (?![0-9]) in\nPatterns 3/4, to prevent matching when the zero literal is followed by a digit.", "id": "clickhouse-tpchavoc-q6-decimal-bug", "preserves": ["Other numeric literal transformations continue to work correctly", "Queries that don't use small decimals remain unaffected"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Fix ClickHouse QueryTransformer stripping leading zeros from decimal literals (tpchavoc Q6)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_adapter.py::TestClickHouseQueryTransformerDecimalLiterals -v", "description": "Decimal literal 0.06 preserved through transformation", "expected": "6 passed", "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Find the ClickHouse QueryTransformer code that handles numeric literals - grep for decimal/numeric transformation logic", "wid": "w1"}, {"evidence": null, "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the leading-zero stripping bug in the transformer", "wid": "w2"}, {"evidence": null, "item_id": "clickhouse-tpchavoc-q6-decimal-bug", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit test with decimal literals like 0.06 to prevent regression", "wid": "w3"}], "worktree": "platform-expansion"} +{"anti_patterns": [{"dont": "add `benchbox.experimental` to public exports or benchmark registries to make the packaging policy feel simpler", "instead": "keep it explicitly unsupported or explicitly out of the default wheel.", "why": "that would silently expand the supported Beta surface"}, {"dont": "claim DuckDB is part of the default install unless `[project.dependencies]` actually says so", "instead": "either change packaging or change docs, but not one without the other.", "why": "users will hit import/runtime mismatches"}, {"dont": "try to zero the entire repository-wide `ty` backlog inside this item", "instead": "ratchet the release-critical entrypoints first and open follow-up work for the rest.", "why": "that turns a focused release blocker into an unbounded cleanup"}, {"dont": "update only `README.md`", "instead": "update all canonical current-state docs in the same pass.", "why": "the FAQ, installation guide, and version-reporting surfaces will drift again"}, {"dont": "use `MANIFEST.in` as the mechanism for wheel package exclusion", "instead": "update `pyproject.toml` packaging config instead.", "why": "wheels are controlled by setuptools package discovery/package-data"}], "approach": "Use the 2026-04-01 review findings as the input set, but ground every change\nin existing local decisions:\n- `move-duckdb-to-optional-extras`\n- `isolate-experimental-core-subsystems`\n- `remove-release-tooling-from-wheel`\n\nResolve policy/documentation questions first, then implement any packaging\nboundary changes, then clean the targeted `ty` diagnostics, then ratchet\nregression tests. Prefer extending existing release/version/package tests over\ninventing one-off scripts.\n", "blocked_reason": null, "category": "Release Readiness", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "close-pre-beta-release-blockers", "id": 118, "reason": "Useful follow-up work, but larger than a Beta-release blocker once the critical gate is ratcheted.", "resolution": "promoted", "resolved_item": "burn-down-the-full-repository-wide-ty-backlog-outside-releas", "resolved_reason": null, "summary": "Burn down the full repository-wide `ty` backlog outside release-critical entrypoints"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "close-pre-beta-release-blockers", "id": 119, "reason": "Potential long-term architecture change; only do this after the immediate Beta packaging policy is stabilized.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on the default-wheel boundary 'remaining contentious'; Beta packaging policy has since stabilized (item done).", "summary": "Extract benchbox.experimental into a separate distribution if the default-wheel boundary remains contentious"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "close-pre-beta-release-blockers", "id": 120, "reason": "Current-state docs and package artifacts are the immediate Beta blocker; historical content can follow.", "resolution": "promoted", "resolved_item": "audit-blog-posts-and-historical-release-summaries-for-outdat", "resolved_reason": null, "summary": "Audit blog posts and historical release summaries for outdated Alpha/install language"}], "deps": [], "description": "The 2026-04-01 whole-project pre-Beta review found four release-surface gaps\nthat should be closed before BenchBox is presented as Beta software:\n\n1. Package metadata already says Beta, but public docs still say Alpha.\n2. Install docs contradict the actual dependency model after DuckDB moved to\n optional extras.\n3. The default wheel still ships the unsupported `benchbox.experimental`\n namespace without a fully explicit Beta packaging/support policy.\n4. `uv run ty check` has too much standing noise to act as a meaningful\n release gate, including warnings on the final execution path in\n `benchbox/cli/commands/run.py`.\n\nThis item is about release coherence, not runtime break/fix work. Lint, fast\ntests, the wheel/sdist build, and CLI help/version commands are already\npassing. The remaining blockers are policy drift, packaging ambiguity, and\nrelease-critical static-analysis noise.\n\nThe work must reconcile two completed historical decisions with the current\nBeta goal:\n\n- `move-duckdb-to-optional-extras` intentionally made DuckDB non-default.\n- `isolate-experimental-core-subsystems` intentionally moved prototypes into\n `benchbox.experimental` but kept them importable from the source tree.\n\nThe outcome of this item must be a single coherent Beta support contract\nacross `pyproject.toml`, canonical docs, built artifacts, and release\nverification commands.", "id": "close-pre-beta-release-blockers", "preserves": ["The completed `move-duckdb-to-optional-extras` rationale is not accidentally reversed by docs-only edits; packaging and docs must agree.", "`benchbox.experimental` stays out of `benchbox.__all__` and benchmark registries regardless of whether it remains in or leaves the default wheel.", "`benchbox.release` remains excluded from the default wheel.", "`uv run -- benchbox --help`, `uv run -- benchbox --version`, and `uv run -- benchbox run --help` continue to exit 0 and expose the current command set.", "`uv run -- python -m pytest -m fast -q` remains green after any release-surface cleanup."], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "Benchmark query corpora under `benchbox/core/*/queries*` and `benchbox/core/*/dataframe_queries*`"}, {"kind": "do_not_modify", "path_glob": "Benchmark result schema and exported result format contracts"}, {"kind": "do_not_modify", "path_glob": "Platform adapter behavior unrelated to packaging/status/install/typecheck cleanup"}, {"kind": "do_not_modify", "path_glob": "Supported benchmark registry membership for non-experimental benchmarks"}, {"kind": "only_modify", "path_glob": "README.md"}, {"kind": "only_modify", "path_glob": "benchbox/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/experimental/"}, {"kind": "only_modify", "path_glob": "docs/README.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/duckdb.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/platform-selection-guide.md"}, {"kind": "only_modify", "path_glob": "docs/usage/faq.md"}, {"kind": "only_modify", "path_glob": "docs/usage/getting-started.md"}, {"kind": "only_modify", "path_glob": "docs/usage/installation.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_package_exports.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_version_command.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_exclusion_invariants.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_infrastructure.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_version_management.py"}, {"kind": "only_modify", "path_glob": "tests/unit/utils/test_version_markers.py"}], "state": "done", "title": "Close pre-Beta release blockers in status messaging, install docs, packaging policy, and typecheck gating", "verifications": [{"command": "rg -n \"Status-Alpha|alpha software|Alpha Software|works with DuckDB out of the box|automatically installed as a dependency\" README.md docs/README.md docs/usage/faq.md docs/usage/installation.md docs/platforms/platform-selection-guide.md docs/platforms/duckdb.md", "description": "Current-state docs no longer contain stale Alpha/default-DuckDB claims", "expected": "No matches for stale current-state status/install claims", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run ty check benchbox/cli/commands/run.py benchbox/base.py benchbox/__init__.py", "description": "Release-critical entrypoints are clean under ty", "expected": "Found 0 diagnostics", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/cli/test_cli_package_exports.py tests/unit/cli/test_cli_version_command.py tests/unit/test_version_management.py tests/unit/utils/test_version_markers.py tests/unit/test_release_exclusion_invariants.py tests/unit/test_release_infrastructure.py -q", "description": "Version/package/release regression tests pass", "expected": "All selected tests pass", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv build --sdist --wheel && unzip -l dist/benchbox-*.whl | rg 'benchbox/(experimental|release)/'", "description": "Default distribution contents match the documented Beta support policy", "expected": "`benchbox/release` absent; `benchbox/experimental` presence or absence matches the documented policy and its tests", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -m fast -q && uv run -- benchbox --help && uv run -- benchbox --version && uv run -- benchbox run --help", "description": "Overall smoke gate remains green", "expected": "Fast suite passes and all CLI help/version commands exit 0", "item_id": "close-pre-beta-release-blockers", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": [], "notes": "Inventory every current-state claim in:\n- `pyproject.toml`\n- `README.md`\n- `docs/README.md`\n- `docs/usage/faq.md`\n- `docs/usage/installation.md`\n- `docs/platforms/platform-selection-guide.md`\n- `docs/platforms/duckdb.md`\n- `uv run -- benchbox --version`\n- `uv run -- benchbox --help`\n\nCapture, in one working matrix, the current answers to:\n- Is BenchBox Alpha or Beta right now?\n- What benchmark/platform counts are publicly claimed?\n- Does plain `uv add benchbox` imply DuckDB support?\n- Is `benchbox.experimental` part of the default wheel contract?\n- Which existing tests already guard these claims?\n\nThis is the source-of-truth step. No subsequent work unit should invent\npolicy independently.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a canonical Beta release matrix for status, install surface, scope counts, and wheel boundaries", "wid": "w1"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w8", "w9"], "notes": "Run and capture the final gate:\n- `uv run ruff check .`\n- `uv run ty check benchbox/cli/commands/run.py benchbox/base.py benchbox/__init__.py`\n- `uv run -- python -m pytest -m fast -q`\n- `uv build --sdist --wheel`\n- wheel inspection for `benchbox/release` and `benchbox/experimental`\n- `uv run -- benchbox --help`\n- `uv run -- benchbox --version`\n- `uv run -- benchbox run --help`\n\nClose this item only when public docs, metadata, wheel contents, and\ntargeted type checking all agree with the chosen Beta support policy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the final pre-Beta verification matrix and record the release decision", "wid": "w10"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Align release status and scope wording across:\n- `pyproject.toml` classifier(s)\n- README badge, status section, and contribution/release language\n- `docs/README.md`\n- `docs/usage/faq.md`\n- any other current-state entry docs found in w1\n\nRemove stale Alpha wording and outdated benchmark/platform count claims.\nLimit follow-on doc edits to canonical current-state entry points, not\nhistorical release notes, blog posts, or archived announcements.\nIf the repo should stay Alpha until the final verification pass, revert\nall surfaces together first and flip all surfaces together at the end.\nMixed Alpha/Beta state is not acceptable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Resolve alpha/beta and benchmark/platform-count drift across metadata and canonical docs", "wid": "w2"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Reconcile all install guidance with the completed\n`move-duckdb-to-optional-extras` decision.\n\nAudit and fix contradictory wording in:\n- `README.md`\n- `docs/usage/faq.md`\n- `docs/usage/installation.md`\n- `docs/platforms/platform-selection-guide.md`\n- `docs/platforms/duckdb.md`\n- quick-start/getting-started pages surfaced by w1 if they are canonical\n current-state install entry points under `docs/` or `README.md`\n\nDecide explicitly between:\n- keeping DuckDB optional and documenting `--extra duckdb` clearly, or\n- intentionally restoring DuckDB to the default dependency surface and\n updating packaging/tests accordingly.\n\nDo not ship hybrid wording such as \"works out of the box\" while\n`pyproject.toml` still keeps DuckDB optional.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Repair install documentation to match the extras-based DuckDB model or intentionally change the dependency surface", "wid": "w3"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Revisit the current outcome of\n`isolate-experimental-core-subsystems` and choose one Beta-safe contract:\n\n- Option A: `benchbox.experimental` remains in the default wheel as an\n explicitly unsupported namespace, documented as outside the supported\n product surface.\n- Option B: `benchbox.experimental` is removed from the default wheel or\n otherwise moved behind an explicit opt-in distribution/install path.\n\nThe chosen contract must cover source-tree imports, wheel installs,\npackage exports, benchmark registries, docs, and regression tests. This\nwork unit is the decision point only: record the chosen contract, why the\nother option was rejected, and the concrete implementation surfaces that\nmust change next.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose the Beta packaging and support contract for benchbox.experimental", "wid": "w4"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w4"], "notes": "Update only the canonical release surfaces implicated by the w4 decision\nso the support boundary is explicit before packaging/test changes land.\n\nCandidate surfaces:\n- `README.md`\n- `docs/README.md`\n- `docs/usage/faq.md`\n- `pyproject.toml` classifiers or package metadata comments if they are\n the canonical place to express the support contract\n\nDo not widen this into a repo-wide prose sweep. The goal is a documented,\ncurrent-state Beta contract for the shipped package.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the chosen benchbox.experimental contract on canonical release surfaces", "wid": "w5"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w4"], "notes": "If Option B is chosen, update setuptools package discovery in\n`pyproject.toml` and any wheel/package tests. Do not use `MANIFEST.in`\nfor wheel exclusion; wheel content is controlled by setuptools package\ndiscovery/package-data, not MANIFEST pruning.\n\nIf Option A is chosen, make the support boundary explicit in docs and\ntests so the shipped namespace is clearly unsupported but intentionally\npresent.\n\nIn both cases:\n- keep `benchbox.release` excluded from the default wheel\n- ensure `benchbox.experimental` does not leak into `benchbox.__all__`\n or benchmark registries\n- verify wheel contents match the documented policy\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the experimental-boundary packaging decision and add wheel-level invariants", "wid": "w6"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w1"], "notes": "Make `ty` actionable on release-critical source files by cleaning the\ndiagnostics in:\n- `benchbox/cli/commands/run.py`\n- `benchbox/base.py`\n- `benchbox/__init__.py`\n\nRequired outcomes:\n- no possibly-unresolved `database_config` / `benchmark_config` flow in\n the final orchestration/export/save path\n- no unused blanket `type: ignore` suppressions in these files\n- control flow in `run.py` is statically obvious enough to support a\n targeted release gate\n\nThis is not a mandate to fix all 1200+ repository diagnostics in one\nitem. The immediate goal is zero noise on release-critical entrypoints.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reduce ty diagnostics to zero for Beta-critical entrypoints", "wid": "w7"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w2", "w3", "w5", "w6"], "notes": "Extend existing tests before adding new one-off scripts.\nCandidate targets:\n- `tests/unit/cli/test_cli_package_exports.py`\n- `tests/unit/cli/test_cli_version_command.py`\n- `tests/unit/test_version_management.py`\n- `tests/unit/utils/test_version_markers.py`\n- `tests/unit/test_release_exclusion_invariants.py`\n- `tests/unit/test_release_infrastructure.py`\n\nAdd assertions for:\n- consistent status wording across metadata/canonical docs\n- install docs not claiming default DuckDB unless packaging says so\n- wheel contents matching the chosen `benchbox.experimental` policy\n- public package exports matching the documented support boundary\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ratchet regression tests so status/install/packaging drift cannot recur", "wid": "w8"}, {"evidence": null, "item_id": "close-pre-beta-release-blockers", "needs": ["w7"], "notes": "Keep the release-critical typecheck gate small, explicit, and durable.\n\nCandidate targets:\n- extend an existing release-infrastructure test if it already shells out\n to project verification commands\n- otherwise add a narrowly scoped regression test that asserts the\n release-critical `ty` invocation stays clean on:\n - `benchbox/cli/commands/run.py`\n - `benchbox/base.py`\n - `benchbox/__init__.py`\n\nDo not expand this into a repository-wide typecheck policy change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a targeted ty regression gate for the Beta-critical entrypoints", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "collapse apply_unified_tuning or close_connection into the same PR as the load-loop extraction", "instead": "(not specified)", "why": "each is a separate near-duplicate with its own risk surface; keep them as deferred follow-ups."}, {"dont": "force bigquery/databricks onto the shared skeleton if their staging model does not actually match", "instead": "(not specified)", "why": "a forced-fit abstraction is worse than two adapters with an honest small duplication."}, {"dont": "pick the casing convention without grepping downstream consumers of table_stats first", "instead": "(not specified)", "why": "a blind choice can silently break result reporting for whichever platform's casing changes."}], "approach": "Extract the skeleton (w1) before touching either adapter, so the interface\nis designed against both existing implementations' actual needs rather\nthan generalized from just one. Resolve the casing question (w2) with a\ngrep across the codebase for table_stats consumers before choosing, since\nthis is the one place a wrong choice silently breaks a caller. Migrate\nredshift and snowflake together in one PR (w3) so the drift is fixed\natomically rather than leaving one platform on the old pattern.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cloud-adapter-staged-load-template", "id": 13, "reason": "Same duplication pattern as the load loop but a separate method; sequence after the staged-load template proves the extraction pattern works, to avoid landing two large simultaneous refactors in one PR.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Deliberately sequenced follow-up under the same still-open staged-load-template item; not a separate untracked task. (tracked by cloud-adapter-staged-load-template)", "summary": "Deduplicate apply_unified_tuning (redshift.py:2637 vs snowflake.py:1748)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cloud-adapter-staged-load-template", "id": 14, "reason": "Smaller and lower-risk than the load loop; a good follow-up once platforms/base has an established place for shared connection-lifecycle helpers (see benchbox/platforms/base/connection_lifecycle.py, which may already be the right home).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Deferral is a note on its own from_item (cloud-adapter-staged-load-template), which is itself still open and planning this exact work.", "summary": "Deduplicate the triplicated close_connection (redshift.py:2437, snowflake.py:1592, databricks/adapter.py:2288)"}], "deps": [], "description": "Architecture review (Work Area E, structural remediation) found that\n`benchbox/platforms/redshift.py:load_data` (starts at line 1794) and\n`benchbox/platforms/snowflake.py:load_data` (starts at line 762)\nindependently implement the same staged-load orchestration loop - resolve\ndata files per table, upload/stage, run the platform load command, record\nper-table row counts, log success/failure - with behavioral drift between\nthe two copies rather than a shared abstraction:\n\n- Key casing: redshift keys `table_stats` by `table_name.lower()`\n (redshift.py:1817, 1828, 1838, 1849, 1859, 1870); snowflake keys by\n `table_name.upper()` (snowflake.py:791, 806, 816).\n- Error reporting on failure: redshift logs the full error text\n (`error_message = str(e) or repr(e) or type(e).__name__`, then\n `self.logger.error(f\"Failed to load {table_name}: {error_message}\")`);\n snowflake truncates to `str(e)[:100]`\n (`self.logger.error(f\"Failed to load {table_name}: {str(e)[:100]}...\")`).\n Same failure class, different diagnostic fidelity depending on which\n platform hit it.\n\nSimilar near-identical duplication exists elsewhere in the same two files\nplus databricks: `apply_unified_tuning` (redshift.py:2637 vs\nsnowflake.py:1748) and `close_connection`, which is actually triplicated\n(redshift.py:2437, snowflake.py:1592,\n`benchbox/platforms/databricks/adapter.py:2288`). This TODO focuses on the\nstaged-load loop first since it has observable behavioral drift (the key\ncasing difference could already be causing lookup bugs for any caller that\nassumes a fixed case); the other duplications are cheaper copy-align fixes\nonce the load-loop template exists to model them on.\n\nThis is a targeted refactor, not a shrink-campaign item -\n`_project/goal-shrink-core-code.md` explicitly scopes \"dual-impl collapses\nthat trade correctness risk for lines\" out of the autonomous shrink loop;\nthis work is done deliberately with adapter test coverage as the safety\nnet, not for line-count credit.\n", "id": "cloud-adapter-staged-load-template", "preserves": ["Existing result-capture and reporting code that reads table_stats keys continues to find its expected keys after the casing normalization in w2.", "Per-platform load behavior (COPY vs PUT+COPY INTO, S3 staging vs Snowflake stage, file-format setup) verified by existing adapter tests continues to work identically after migration.", "table_stats total row counts and the load_data return contract (tuple[dict[str, int], float, dict[str, Any] | None]) from benchbox/platforms/base/adapter.py:323 are unchanged."], "prior_art": [{"concept": "benchbox/platforms/base/adapter.py:323 load_data abstract method - the existing contract both adapters implement; the new skeleton must satisfy this same signature/return contract.", "decision": "reuse", "path": "benchbox/platforms/base/adapter.py:323"}, {"concept": "benchbox/platforms/base/connection_lifecycle.py - likely existing home for the deferred close_connection deduplication; check before creating a new module for that follow-up.", "decision": "reuse", "path": "benchbox/platforms/base/connection_lifecycle.py"}, {"concept": "benchbox/platforms/base/data_loading.py - existing shared data-loading helpers (COPY/CTAS loaders with trailing-delimiter handling per datafusion-tbl-trailing-delimiter-load) already used as a common base by other adapters; extend this file's pattern rather than starting a parallel shared-helpers module.", "decision": "extend", "path": "benchbox/platforms/base/data_loading.py"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/staged_load.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/bigquery.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/redshift.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/snowflake.py"}], "state": "planning", "title": "Extract a shared staged-load template method for redshift/snowflake (and evaluate bigquery/databricks)", "verifications": [{"command": "uv run -- python -m pytest tests -k 'redshift or snowflake' -m 'not slow and not stress' --tb=short", "description": "Redshift and Snowflake adapter load-data tests pass after migration to the shared skeleton", "expected": "all redshift/snowflake adapter tests pass", "item_id": "cloud-adapter-staged-load-template", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'table_stats\\[' benchbox/platforms/redshift.py benchbox/platforms/snowflake.py", "description": "table_stats casing convention is consistent and documented", "expected": "both files use the same .lower() or .upper() convention on table_stats keys", "item_id": "cloud-adapter-staged-load-template", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": [], "notes": "Add a shared skeleton to `benchbox/platforms/base/data_loading.py` (or\na new `benchbox/platforms/base/staged_load.py` if data_loading.py is\nalready large) implementing: resolve data files via\n`_resolve_data_files` -> per-table hook (upload/stage + platform-specific\nload command, e.g. COPY/PUT+COPY INTO) -> collect stats -> aggregate\ntiming. The per-table load command and any platform-specific setup\n(Redshift's S3 client + external schema, Snowflake's file formats +\nquery tag) stay as abstract hooks the subclass implements; only the\nloop/bookkeeping/logging shape is shared.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extract a template-method staged-load skeleton in platforms/base", "wid": "w1"}, {"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": ["w1"], "notes": "Pick one casing convention (lower or upper) for the shared skeleton's\n`table_stats` dict keys. Check every caller of `load_data`'s returned\n`table_stats` (result capture, reporting, validation) for any\ncase-sensitive lookup before picking, since changing either platform's\ncasing could silently break a caller that currently works only because\nof that platform's specific case. Document the chosen convention in\nthe base class docstring so future adapters do not reintroduce drift.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Normalize table_stats key casing and document the choice", "wid": "w2"}, {"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": ["w2"], "notes": "Reimplement `RedshiftAdapter.load_data` and `SnowflakeAdapter.load_data`\nas thin per-table hooks over the new base skeleton, removing the\nduplicated loop/logging code. Normalize the error-logging behavior to\none policy (recommend: log full error text like redshift currently\ndoes, since truncating to 100 chars as snowflake does can hide the\nactionable part of a load error) - state the choice and rationale in\nthe PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Migrate redshift.py and snowflake.py to the shared skeleton", "wid": "w3"}, {"evidence": null, "item_id": "cloud-adapter-staged-load-template", "needs": ["w3"], "notes": "After redshift+snowflake prove the template out, check whether\n`benchbox/platforms/bigquery.py` and\n`benchbox/platforms/databricks/adapter.py` have a load_data loop\nsimilar enough to migrate, or whether their staging model (e.g.\nDatabricks' different ingestion path) genuinely differs enough that\nforcing them onto the same skeleton would hurt readability. Migrate\nonly if it is a clean fit; otherwise record why they are excluded.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Evaluate bigquery/databricks for the same template", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "mark a platform as supported without adapter-level tests", "instead": "require platform-specific assertions", "why": "dry-run regressions would be invisible"}, {"dont": "overload existing CLUSTERING/Z_ORDERING semantics", "instead": "add a dedicated sorted-ingestion strategy control", "why": "users need separate A/B dimensions"}, {"dont": "silently switch default cloud behavior to deep sort", "instead": "require explicit opt-in", "why": "it can materially change runtime/cost"}], "approach": "Build on PlatformAdapter.apply_ctas_sort and existing cloud adapter tuning hooks,\nbut add an explicit strategy layer so sorted ingestion can be requested even on\nplatforms previously marked as CTAS-sort unsupported.\n\nStart with strategy selection and capability gating, then implement platform\nexecution paths one adapter at a time. Record selected strategy and overhead\nmetrics in results for downstream comparison analysis.\n\nKeep all cloud-platform behavior opt-in for safety and to preserve current defaults.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [], "deps": ["ctas-sort-cloud-platforms", "ctas-sort-dataloader-connectivity"], "description": "Add explicit sorted data ingestion support for cloud data warehouse platforms as\na first-class benchmark strategy, even when platform-native tuning features\n(clustering keys, Z-ordering, partitioning, sort keys) already exist.\n\nRationale:\n- Deep sorting is an alternative physical layout strategy, not only a fallback.\n- Users need apples-to-apples performance and cost comparisons between:\n 1) deep sorted ingestion,\n 2) platform-native tuning features,\n 3) combined strategies where safe/valid.\n- Current cloud CTAS-sort decisions default to \"unsupported/no-op\" for several\n platforms, which blocks this experiment design.\n\nScope:\n- Add an opt-in sorted ingestion mode for cloud platforms.\n- Implement per-platform sorted ingestion execution paths with clear capability\n gates and documented behavior.\n- Capture benchmark metadata so result analysis can quantify value/cost trade-offs.", "id": "cloud-dw-sorted-data-ingestion", "preserves": ["Default behavior for existing runs remains unchanged when sorted-ingestion mode is not enabled", "Dry-run mode continues to capture all emitted SQL/actions without requiring live credentials", "Platform-native tuning behavior (Snowflake clustering, Databricks Z-ORDER, BigQuery CLUSTER BY) remains available and testable", "benchbox compare remains compatible with result schema changes"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}], "state": "done", "title": "Add sorted data ingestion support for cloud data warehouses to compare against clustering/Z-order/partitioning", "verifications": [{"command": "uv run _project/scripts/validate_todo.py _project/DONE/physical-tuning/active/cloud-dw-sorted-data-ingestion.yaml", "description": "New TODO validates against schema", "expected": "Validation passed", "item_id": "cloud-dw-sorted-data-ingestion", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_cloud_sorted_ingestion_strategy.py -v --tb=short", "description": "Cloud strategy tests pass", "expected": "all tests pass, 0 failures", "item_id": "cloud-dw-sorted-data-ingestion", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_cloud_platform_ctas_sort.py -v --tb=short", "description": "Existing cloud CTAS sort tests stay green", "expected": "all tests pass, 0 failures", "item_id": "cloud-dw-sorted-data-ingestion", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define sorted-ingestion strategy model and CLI/tuning config surface (force/auto/off, method selection)", "wid": "w1"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement capability matrix + guardrails for cloud platforms (Snowflake, Databricks, BigQuery, Redshift, Athena, Synapse, ClickHouse Cloud)", "wid": "w2"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Snowflake/Databricks deep-sort ingestion paths with safe defaults and explicit method markers", "wid": "w3"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement BigQuery/Redshift deep-sort ingestion paths or explicit actionable errors", "wid": "w4"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Athena/Synapse deep-sort ingestion paths or explicit actionable errors", "wid": "w5"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ClickHouse Cloud deep-sort ingestion path or explicit actionable error", "wid": "w6"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add result metadata for ingestion strategy, runtime overhead, and cost-impact comparability", "wid": "w7"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w3", "w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit/integration test coverage for strategy selection and SQL/action generation in dry-run mode", "wid": "w8"}, {"evidence": null, "item_id": "cloud-dw-sorted-data-ingestion", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Documentation and examples: deep-sort vs native-tuning comparison workflow", "wid": "w9"}], "worktree": "physical-tuning"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-29T00:00:00Z", "completed_pr": null, "created_at": "2026-03-28T00:00:00Z", "deferrals": [], "deps": [], "description": "Code review of the cloud-integration-test-infrastructure implementation\n(bcccb82..877e7452d) scored 7/10. Bug fixes were committed (5f8caffb2),\nraising it to ~8.5. Four structural gaps remain:\n\n1. Test depth parity - Firebolt (4 tests), Starburst (5), MotherDuck (5)\n are significantly thinner than Snowflake (9), BigQuery (10), Databricks (9).\n Missing: SchemaManagement, DataLoading, TPC-H QueryExecution, SpecificFeatures.\n2. Scope creep - PREFER_PLATFORM_FORMAT_ORDER production code refactor\n (format_capabilities.py, data_loading.py) was bundled in a test TODO.\n Already committed; acknowledge in commit message only.\n3. Comparison test fragility - test_pg_extension_comparison.py shells out\n to CLI via subprocess. Depends on --output flag + JSON schema stability.\n Should use Python adapter API directly.\n4. S3 test fixture gap - test_s3_parquet_query assumes pre-existing\n test.parquet in S3. No setup script or fixture creates it.", "id": "cloud-integration-test-gaps", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Cloud Integration Test Infrastructure - Address Review Gaps", "verifications": [], "work": [{"evidence": null, "item_id": "cloud-integration-test-gaps", "needs": [], "notes": "In test_pg_duckdb_live.py, replace s3_test_bucket fixture with\ns3_test_parquet fixture that:\n1. Skips if BENCHBOX_S3_TEST_BUCKET / AWS creds not set\n2. Generates small parquet (100 rows) via pyarrow\n3. Uploads to s3://{bucket}/benchbox_test/test.parquet via boto3\n4. Yields {\"bucket\", \"key\", \"row_count\": 100}\n5. Deletes S3 object in teardown\nUpdate test_s3_parquet_query to use new fixture, assert exact row count.\nRemove inner try/except pytest.skip guard (fixture guarantees file exists).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "S3 test fixture - replace pre-existing file assumption with self-contained setup/teardown", "wid": "w0"}, {"evidence": null, "item_id": "cloud-integration-test-gaps", "needs": [], "notes": "Add to test_firebolt_live.py:\n- TestLiveFireboltSchemaManagement: test_create_and_drop_database\n (Firebolt uses databases not schemas)\n- TestLiveFireboltDataLoading: test_tpch_data_load\n (TPCH, dialect=\"postgres\", adapter.load_data, verify stats)\n- TestLiveFireboltQueryExecution: add test_tpch_query_1\n (get_query(1, seed=42), execute, assert results > 0, skip if no data)\n- TestLiveFireboltSpecificFeatures: test_result_cache_control, test_cleanup\nTarget: 4\u21929 tests. Add `from benchbox import TPCH`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Firebolt test depth - add SchemaManagement, DataLoading, TPC-H QueryExecution, SpecificFeatures", "wid": "w1"}, {"evidence": null, "item_id": "cloud-integration-test-gaps", "needs": [], "notes": "Add to test_motherduck_live.py:\n- TestLiveMotherDuckSchemaManagement: test_schema_creation\n (CREATE SCHEMA, verify via information_schema, cleanup)\n- TestLiveMotherDuckDataLoading: test_tpch_data_load\n (DuckDB dialect, connection.execute() pattern)\n- TestLiveMotherDuckQueryExecution: add test_tpch_query_1\n- TestLiveMotherDuckSpecificFeatures: move test_duckdb_extensions here,\n add test_cloud_databases (SHOW DATABASES), test_cleanup\nTarget: 5\u21929 tests. Add `from benchbox import TPCH`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "MotherDuck test depth - add SchemaManagement, DataLoading, TPC-H QueryExecution, SpecificFeatures", "wid": "w2"}, {"evidence": null, "item_id": "cloud-integration-test-gaps", "needs": [], "notes": "Add to test_starburst_live.py:\n- Module-level starburst_writable_catalog() fixture\n (skips if STARBURST_WRITABLE_CATALOG not set)\n- TestLiveStarburstSchemaManagement: test_schema_creation\n (in writable catalog, gated on fixture)\n- TestLiveStarburstDataLoading: test_tpch_data_load\n (in writable catalog, dialect=\"trino\")\n- TestLiveStarburstQueryExecution: add test_tpch_builtin_query\n (query tpch.tiny.lineitem with GROUP BY, guard TrinoUserError)\n- TestLiveStarburstSpecificFeatures: move test_show_catalogs here,\n add test_information_schema, test_cleanup\nTarget: 5\u21929 tests. Update module docstring for STARBURST_WRITABLE_CATALOG.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Starburst test depth - add SchemaManagement, DataLoading, TPC-H QueryExecution, SpecificFeatures", "wid": "w3"}, {"evidence": null, "item_id": "cloud-integration-test-gaps", "needs": ["w1", "w2", "w3"], "notes": "In test_pg_extension_comparison.py:\n1. Remove subprocess/json imports, remove _run_benchmark_for_platform()\n2. Add TPCH import, add adapter_class paths to PG_EXTENSIONS dict\n3. New _run_benchmark_via_adapter():\n - Instantiate adapter, TPCH(scale=0.01), generate_data()\n - adapter.create_schema(), adapter.load_data()\n - adapter.execute_query() for each query, collect timings\n - Assemble result dict matching expected structure\n - Lowercase status field (adapter returns \"SUCCESS\", test checks \"success\")\n - Cleanup in finally\n4. Update comparison_results fixture to call new function\n5. Test classes and assertions remain unchanged\nDepends on w1-w3 for familiarity with adapter API patterns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Comparison test refactor - replace subprocess CLI with Python adapter API", "wid": "w4"}, {"evidence": null, "item_id": "cloud-integration-test-gaps", "needs": ["w0", "w1", "w2", "w3", "w4"], "notes": "Include in the final commit message a note about the PREFER_PLATFORM_FORMAT_ORDER\nproduction code refactor (commit ccbd69f56) that was bundled in the original\ntest infrastructure TODO. No code changes needed - documentation only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Scope creep acknowledgment - note in commit message", "wid": "w5"}], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-28T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Consolidates deferred live integration test work from completed platform\nadapter TODOs. Each adapter was implemented with full unit test coverage but\ndeferred live cloud testing due to account provisioning requirements.\n\nPlatforms with deferred live tests:\n- Firebolt Cloud (from firebolt-cloud-deployment)\n- StarRocks (from starrocks-olap-platform)\n- Starburst Galaxy (from starburst-platform-adapter)\n- MotherDuck (from motherduck-platform-adapter)\n- pg_duckdb via MotherDuck (from pg-duckdb-platform-adapter)\n- pg_mooncake object storage (from pg-mooncake-platform-adapter)\n\nAlso consolidates deferred CI infrastructure items:\n- Docker-based PostgreSQL extension testing (from postgres-extension-benchmarking-infrastructure)\n- Automated multi-extension comparison pipeline\n\nMotherDuck adapter feature work (credential wizard, cloud upload, cache\ncontrol) has been split to motherduck-cloud-features.yaml - this TODO covers\nonly test infrastructure and smoke tests.", "id": "cloud-integration-test-infrastructure", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Cloud Platform Integration Test Infrastructure - Deferred Live Tests", "verifications": [], "work": [{"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": [], "notes": "Produce a provisioning checklist covering:\n1. Which accounts already exist (check for env vars / config files)\n2. Minimum viable setup per platform (free tier? trial? Docker alternative?)\n - StarRocks: Docker Compose (free, local) - already has docker/starrocks/\n - pg_duckdb / pg_mooncake: Docker (free, local)\n - MotherDuck: free tier available (10 GB)\n - Firebolt: trial account required\n - Starburst Galaxy: trial account required\n3. Required env vars per platform for skip-when-unavailable logic\n4. Estimated monthly cost for platforms without free tiers\n5. Recommendation: prioritize Docker-based tests (free) first, then\n free-tier cloud, then paid-tier cloud\nDecision: Docker-based tests (w7) should run in CI from the start since\nthey are free and deterministic. Cloud tests remain local-only until\ncredentials are available in CI secrets.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research - inventory cloud accounts, document provisioning steps, estimate costs", "wid": "w0"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w0"], "notes": "Register new markers (live_firebolt, live_starburst, live_motherduck,\nlive_pg_duckdb, live_pg_mooncake) in pyproject.toml.\nAdd a shared fixture or helper for the standard smoke test pattern:\nconnect \u2192 create table \u2192 load SF=0.01 \u2192 run TPC-H Q1 \u2192 validate row count.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add per-platform live test markers to pyproject.toml and create shared cloud test fixtures", "wid": "w1"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "Create unified Docker Compose config under docker/pg-extensions/ that\nbrings up PostgreSQL with all three extensions. Wire into Makefile as\nmake test-docker-up-pg-extensions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Docker-based PostgreSQL extension CI harness - Docker Compose for pg_duckdb, pg_mooncake, pg_analytics", "wid": "w10"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w10", "w6", "w8"], "notes": "Script or pytest fixture that runs the same benchmark suite against\npg_duckdb, pg_mooncake, and pg_analytics, collecting results per extension.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Multi-extension comparison orchestration - run benchmarks across pg extensions sequentially", "wid": "w11"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w11"], "notes": "Takes collected results from w11 and generates comparison report\n(ASCII table + optional HTML chart). Integrate with existing\nresult_plotter.py infrastructure.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Multi-extension comparison report generation - produce comparison tables from orchestrated results", "wid": "w12"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "External blocker: requires Firebolt Cloud trial/paid account with\nprovisioned engine and S3 credentials for data loading.\nEnv vars: FIREBOLT_HOST, FIREBOLT_USER, FIREBOLT_PASSWORD, FIREBOLT_DATABASE.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Firebolt Cloud live smoke tests - connect, create table, run TPC-H Q1, validate result", "wid": "w2"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "No external blocker - Docker Compose setup already exists at\ndocker/starrocks/docker-compose.yml. Use skip_unless_docker_service()\ntargeting localhost:19030.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "StarRocks live smoke tests - connect, create table, run TPC-H Q1, validate result", "wid": "w3"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "External blocker: requires Starburst Galaxy trial/paid account and\nrunning cluster.\nEnv vars: STARBURST_HOST, STARBURST_USER, STARBURST_PASSWORD, STARBURST_CATALOG.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Starburst Galaxy live smoke tests - connect, create table, run TPC-H Q1, validate result", "wid": "w4"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "Free tier available (10 GB). Env var: MOTHERDUCK_TOKEN.\nUse get_env_or_skip(\"MOTHERDUCK_TOKEN\", \"MotherDuck\").\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "MotherDuck live smoke tests - connect, create table, run TPC-H Q1, validate result", "wid": "w5"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "Requires Docker (pg_duckdb extension) + MOTHERDUCK_TOKEN for MotherDuck mode.\nTwo-phase skip: skip_unless_docker_service for pg, get_env_or_skip for token.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "pg_duckdb MotherDuck mode integration tests - connect via MotherDuck, run TPC-H Q1", "wid": "w6"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "External blocker: requires S3 bucket with Parquet/Iceberg test data.\nEnv vars: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BENCHBOX_S3_TEST_BUCKET.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "pg_duckdb S3 Parquet/Iceberg data lake query tests", "wid": "w7"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w1"], "notes": "External blocker: requires S3 or GCS bucket with credentials and a\nDocker PostgreSQL instance with pg_mooncake extension.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "pg_mooncake S3/GCS object storage integration tests", "wid": "w8"}, {"evidence": null, "item_id": "cloud-integration-test-infrastructure", "needs": ["w8"], "notes": "Requires pg_mooncake Docker setup from w8 plus WAL configuration.\nTests replication lag measurement and benchmark consistency under WAL load.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "pg_mooncake pg_moonlink WAL replication benchmarking tests", "wid": "w9"}], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Architecture", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- Added tests/unit/test_notebooks_content.py to validate per-notebook installation tokens and benchbox.core.runner usage\n- Existing tests/unit/test_notebooks.py validate presence, JSON validity, required sections, and run_benchmark cell", "id": "cloud-platform-benchmarking-notebooks", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Cloud Platform Benchmarking Notebooks", "verifications": [], "work": [], "worktree": "unknown"} @@ -187,12 +187,12 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-24T00:00:00Z", "deferrals": [], "deps": ["athena-spark-platform", "dataproc-serverless-platform", "synapse-spark-platform"], "description": "Add dependency groups for cloud Spark platforms to pyproject.toml.\nEnables users to install platform-specific dependencies with\n`uv add benchbox --extra cloud-spark-`.\n\n**Current State (as of 2026-01-07):**\nAll 7 cloud Spark platform adapters are COMPLETE and registered:\n- AWS: Glue, EMR Serverless, Athena Spark (all use boto3)\n- GCP: Dataproc, Dataproc Serverless (use google-cloud-dataproc + storage)\n- Azure: Synapse Spark, Fabric Spark (use azure-identity + datalake + requests)\n- Snowflake: Snowpark Connect (uses snowflake-snowpark-python)\n\n**Problem:**\nThe existing `cloud-spark` extra in pyproject.toml only includes Databricks\ndependencies and has an outdated \"Future: AWS Glue, EMR...\" comment.\nProvider-specific extras (cloud-spark-aws, cloud-spark-gcp, cloud-spark-azure)\ndon't exist.\n\n**Scope:**\nThis TODO is ONLY about pyproject.toml dependency groups, not platform\nimplementation (which is already complete).", "id": "cloud-spark-dependency-groups", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Cloud Spark Dependency Groups in pyproject.toml", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-23T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": ["spark-mixin-open-table-format-loading"], "description": "Several managed Spark adapters do not use SparkDataLoadMixin for data loading and\ntherefore are not covered by spark-mixin-open-table-format-loading.\n\nThis item covers the job/staging-based adapters whose loading is implemented through\nplatform-specific upload and remote table-registration flows:\n- emr-serverless\n- dataproc\n- dataproc-serverless\n- athena-spark\n\nThe goal is to make Delta/Iceberg/Hudi selectable and executable end-to-end for the\nadapters above, not merely to add capability-table entries.", "id": "cloud-spark-job-adapters-open-table-format-loading", "preserves": ["Capability registration must remain conservative until verified end-to-end", "Existing parquet/native loading paths for all four adapters"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/aws/athena_spark_adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/aws/emr_serverless_adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/format_capabilities.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/gcp/dataproc_adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/gcp/dataproc_serverless_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Close open-table-format loading gaps for job-based managed Spark adapters", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'emr or dataproc or athena_spark' --no-header -q", "description": "Managed Spark adapter tests pass", "expected": "All tests pass", "item_id": "cloud-spark-job-adapters-open-table-format-loading", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_format_capabilities.py -v --no-header -q", "description": "Format capability tests pass after any re-registration", "expected": "All tests pass", "item_id": "cloud-spark-job-adapters-open-table-format-loading", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cloud-spark-job-adapters-open-table-format-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit each adapter's actual load path and supported remote table-registration mechanism", "wid": "w1"}, {"evidence": null, "item_id": "cloud-spark-job-adapters-open-table-format-loading", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Delta/Iceberg/Hudi loading or explicit per-format rejection for emr-serverless", "wid": "w2"}, {"evidence": null, "item_id": "cloud-spark-job-adapters-open-table-format-loading", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Delta/Iceberg/Hudi loading or explicit per-format rejection for dataproc and dataproc-serverless", "wid": "w3"}, {"evidence": null, "item_id": "cloud-spark-job-adapters-open-table-format-loading", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Delta/Iceberg loading or explicit per-format rejection for athena-spark", "wid": "w4"}, {"evidence": null, "item_id": "cloud-spark-job-adapters-open-table-format-loading", "needs": ["w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-register only the adapters with verified end-to-end support in format_capabilities.py", "wid": "w5"}, {"evidence": null, "item_id": "cloud-spark-job-adapters-open-table-format-loading", "needs": ["w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add adapter-specific unit tests covering remote registration SQL/config generation", "wid": "w6"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-24T00:00:00Z", "deferrals": [], "deps": [], "description": "Build shared infrastructure for all cloud Spark platforms to maximize code reuse.\nThis includes CloudSparkSessionManager, CloudSparkStaging, and SparkConfigOptimizer.\n\n80% of logic is shared across Glue, Dataproc, Synapse, EMR, Fabric, and other\nmanaged Spark platforms. Building this foundation first enables rapid development\nof individual platform adapters.", "id": "cloud-spark-shared-infrastructure", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "Cloud Spark Shared Infrastructure", "verifications": [], "work": [], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "use adapter config/credentials pattern", "dont": "hardcode cloud credentials or ARNs", "why": "these vary per deployment"}, {"INSTEAD": "implement and test first", "dont": "register platforms in format_capabilities.py before loading code is tested", "why": "the fail-fast check will pass but load will fail at runtime"}], "approach": "Redshift Delta: Register Delta table in AWS Glue catalog, then create Redshift\nSpectrum external table referencing the Glue table. Requires IAM role with Glue\nand S3 permissions.\n\nBigQuery Delta: Create BigLake external table with Delta format. Requires BigLake\nconnection resource and GCS bucket access.\n\nBoth implementations are cloud-specific and require careful credential/permission\nhandling.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [{"created_at": "2026-07-18T19:50:10Z", "from_item": "cloud-sql-open-table-format-loading", "id": 566, "reason": "Requires cloud credentials and running instances", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Blocked on cloud credentials/live Redshift+BigQuery instances not currently available; no open item tracks this specific integration test.", "summary": "Integration tests with real Redshift/BigQuery instances"}, {"created_at": "2026-07-18T19:50:10Z", "from_item": "cloud-sql-open-table-format-loading", "id": 567, "reason": "Lower priority than Delta; both platforms have evolving Iceberg support", "resolution": "promoted", "resolved_item": "iceberg-support-for-redshift-via-spectrum-and-bigquery-via-b", "resolved_reason": null, "summary": "Iceberg support for Redshift (via Spectrum) and BigQuery (via BigLake)"}], "deps": [], "description": "Redshift and BigQuery both support Delta Lake on the platform but their BenchBox\nadapters only load parquet/CSV. These platforms were removed from DELTA_CAPABILITY\nin format_capabilities.py (commit 9be7d5b70) to prevent the fail-fast check from\npassing when loading would fail.\n\nPlatform mechanisms:\n- Redshift: Delta via Redshift Spectrum + AWS Lake Formation. Requires Glue catalog\n integration to register Delta tables as external tables.\n- BigQuery: Delta via BigLake external tables (GA since mid-2024). Requires BigLake\n connection + external table definition pointing to GCS Delta directory.\n\nCompleted closure:\n- BenchBox now uses mode-aware format selection so native/default flows remain\n conservative.\n- BigQuery Delta is only advertised when the run includes the required BigLake\n connection configuration.\n- Redshift Delta is only advertised when the run includes the required Spectrum/IAM\n configuration.", "id": "cloud-sql-open-table-format-loading", "preserves": ["BigQuery parquet GCS load job path must remain unchanged", "Redshift parquet/CSV COPY loading path must remain unchanged", "format_capabilities.py fail-fast check: only re-register after loading code works"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/format_capabilities.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/bigquery.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/redshift.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Finish Delta loading closure for Redshift and BigQuery adapters", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'redshift or bigquery' --no-header -q", "description": "Redshift and BigQuery adapter tests pass", "expected": "All tests pass", "item_id": "cloud-sql-open-table-format-loading", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_format_capabilities.py -v --no-header -q", "description": "Format capabilities tests pass with re-registered platforms", "expected": "All tests pass", "item_id": "cloud-sql-open-table-format-loading", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep Redshift external Delta registration path and tests green", "wid": "w1"}, {"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep BigQuery BigLake Delta registration path and tests green", "wid": "w2"}, {"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add mode-aware format selection before re-registering Redshift and BigQuery", "wid": "w3"}, {"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand tests to cover final closure criteria for format selection", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "hardcode cloud credentials or ARNs", "instead": "use adapter config/credentials pattern", "why": "these vary per deployment"}, {"dont": "register platforms in format_capabilities.py before loading code is tested", "instead": "implement and test first", "why": "the fail-fast check will pass but load will fail at runtime"}], "approach": "Redshift Delta: Register Delta table in AWS Glue catalog, then create Redshift\nSpectrum external table referencing the Glue table. Requires IAM role with Glue\nand S3 permissions.\n\nBigQuery Delta: Create BigLake external table with Delta format. Requires BigLake\nconnection resource and GCS bucket access.\n\nBoth implementations are cloud-specific and require careful credential/permission\nhandling.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [{"created_at": "2026-07-18T19:50:10Z", "from_item": "cloud-sql-open-table-format-loading", "id": 566, "reason": "Requires cloud credentials and running instances", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Blocked on cloud credentials/live Redshift+BigQuery instances not currently available; no open item tracks this specific integration test.", "summary": "Integration tests with real Redshift/BigQuery instances"}, {"created_at": "2026-07-18T19:50:10Z", "from_item": "cloud-sql-open-table-format-loading", "id": 567, "reason": "Lower priority than Delta; both platforms have evolving Iceberg support", "resolution": "promoted", "resolved_item": "iceberg-support-for-redshift-via-spectrum-and-bigquery-via-b", "resolved_reason": null, "summary": "Iceberg support for Redshift (via Spectrum) and BigQuery (via BigLake)"}], "deps": [], "description": "Redshift and BigQuery both support Delta Lake on the platform but their BenchBox\nadapters only load parquet/CSV. These platforms were removed from DELTA_CAPABILITY\nin format_capabilities.py (commit 9be7d5b70) to prevent the fail-fast check from\npassing when loading would fail.\n\nPlatform mechanisms:\n- Redshift: Delta via Redshift Spectrum + AWS Lake Formation. Requires Glue catalog\n integration to register Delta tables as external tables.\n- BigQuery: Delta via BigLake external tables (GA since mid-2024). Requires BigLake\n connection + external table definition pointing to GCS Delta directory.\n\nCompleted closure:\n- BenchBox now uses mode-aware format selection so native/default flows remain\n conservative.\n- BigQuery Delta is only advertised when the run includes the required BigLake\n connection configuration.\n- Redshift Delta is only advertised when the run includes the required Spectrum/IAM\n configuration.", "id": "cloud-sql-open-table-format-loading", "preserves": ["BigQuery parquet GCS load job path must remain unchanged", "Redshift parquet/CSV COPY loading path must remain unchanged", "format_capabilities.py fail-fast check: only re-register after loading code works"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/format_capabilities.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/bigquery.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/redshift.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Finish Delta loading closure for Redshift and BigQuery adapters", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'redshift or bigquery' --no-header -q", "description": "Redshift and BigQuery adapter tests pass", "expected": "All tests pass", "item_id": "cloud-sql-open-table-format-loading", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_format_capabilities.py -v --no-header -q", "description": "Format capabilities tests pass with re-registered platforms", "expected": "All tests pass", "item_id": "cloud-sql-open-table-format-loading", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep Redshift external Delta registration path and tests green", "wid": "w1"}, {"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep BigQuery BigLake Delta registration path and tests green", "wid": "w2"}, {"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add mode-aware format selection before re-registering Redshift and BigQuery", "wid": "w3"}, {"evidence": null, "item_id": "cloud-sql-open-table-format-loading", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand tests to cover final closure criteria for format selection", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:35:44Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #355 of benchmark-output-root-resolve-at-construction-boundary.\nDeferred: Cloud staging path (CloudStagingPath/DatabricksPath) construction-time injection edge cases\nReason deferred: Cloud roots resolve later via credentials in _apply_default_cloud_output_dir; needs its own analysis once the local-path boundary is settled.", "id": "cloud-staging-path-cloudstagingpath-databrickspath-construct", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Cloud staging path (CloudStagingPath/DatabricksPath) construction-time injection edge cases", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "the BenchBox PR ships only the TODO and triage updates.", "dont": "bundle the skill file edit into the BenchBox PR diff", "why": "the skill file lives outside the repo"}, {"INSTEAD": "keep them in their own subsection.", "dont": "generalise the proposals/spec checks into a sixth axis", "why": "the five axes are stable and broadly applicable to code, while these checks apply specifically to proposals/spec docs"}, {"INSTEAD": "bundling avoids three PRs reviewing nearly-identical changes.", "dont": "split into three separate TODOs", "why": "each finding produces a one-bullet edit to the same skill file"}], "approach": "Single bundled PR for the planning artifacts (TODO file + three\nblind-spot triage updates). The skill file edit lives in the\nuser-global `~/.claude/` directory; do that edit alongside the\nBenchBox commits but expect it not to land via the PR.\n\nPromotion via `sweep_blind_spots.py triage --action promote\n--todo-id ` only updates frontmatter `status` / `todo_id` and\nappends a `## Triage log` entry \u2014 it does not author the TODO.\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-03T00:00:00Z", "completed_pr": null, "created_at": "2026-05-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "code-review-checklist-additions-for-spec-docs", "id": 121, "reason": "Premature generalisation. The five axes are stable; these checks apply only to proposals/spec docs. Promote to a sixth axis only if the subsection grows past three or four items and starts to show structural patterns of its own.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly premature: promote to a sixth rubric axis only if the proposals/spec subsection grows past 3-4 items; no evidence that threshold was reached.", "summary": "Generalise the proposals/spec checks into a sixth axis of the rubric"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "code-review-checklist-additions-for-spec-docs", "id": 122, "reason": "Convention + reviewer attention is the proposed enforcement model, mirroring the parent UAT methodology spec's approach. Tooling is a lever to pull only if a future review shows the convention drifting.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Tooling enforcement explicitly deferred pending evidence of convention drift; no such evidence cited, no open item tracks it.", "summary": "Tooling enforcement (e.g., a /code linter that flags spec docs missing an approval protocol)"}], "deps": [], "description": "Promotes three meta blind-spot findings produced *during* the\n2026-05-03 review of the UAT methodology remediation work \u2014 findings\nabout how proposals/spec docs were reviewed, not about the source\nUAT \u2014 into a single update to the `/code` review skill.\n\nSource findings (all currently `status: open` in\n`_project/blind-spots/`):\n\n1. `2026-05-03-083859-proposals-doc-decidability-gap` \u2014 proposals\n reviewed via the five-axis rubric can pass on Correctness /\n Readability while still failing the user-facing function: each\n user-facing choice should ship with a default, a consequence, and\n a recommendation, not just a question.\n\n2. `2026-05-03-084354-stress-test-self-bias` \u2014 when the proposer also\n designs the stress-test, \"all proposals survive\" reads as\n validation but is closer to a sanity check. Reviews of stress-test\n handoffs should require a \"strongest argument against each\n proposal\" subsection (or split designer/reviewer roles).\n\n3. `2026-05-03-084923-spec-approval-ergonomics` \u2014 specs with a\n user-approval gate should declare the response shape (yes/no per\n question, defaults, partial-acceptance cascades). Reviewers should\n flag specs that ask questions without an approval protocol.\n\nImplementation surface: a single new \"Reviewing proposals/spec docs\"\nsection appended to `~/.claude/skills/references/five-axis-review.md`\nwith three checklist items. The skill file lives in the user-global\n`~/.claude/` directory and so the actual content edit ships outside\nany BenchBox PR; the BenchBox PR carries only the TODO record and\nthe three blind-spot frontmatter triage updates.", "id": "code-review-checklist-additions-for-spec-docs", "preserves": ["Existing blind-spot bodies \u2014 only `status` and `todo_id` (and the `## Triage log` entry) may change during promotion.", "The Severity Classification, Change Sizing, and Rules sections at the bottom of `five-axis-review.md`.", "The five existing axes (Correctness, Readability, Architecture, Security, Performance) and their bullets \u2014 the new section is additive."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "_project/handoffs/"}, {"kind": "do_not_modify", "path_glob": "_project/specs/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/"}, {"kind": "do_not_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/code-review-checklist-additions-for-spec-docs.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/code-review-checklist-additions-for-spec-docs.yaml"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-03-083859-proposals-doc-decidability-gap.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-03-084354-stress-test-self-bias.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-03-084923-spec-approval-ergonomics.md"}], "state": "done", "title": "Add proposal/spec-doc checks to the /code review checklist", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py list --status merged-to-todo | grep -c code-review-checklist-additions-for-spec-docs", "description": "All three blind-spots reach merged-to-todo status with todo_id set", "expected": ">= 3", "item_id": "code-review-checklist-additions-for-spec-docs", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -c 'Reviewing proposals/spec docs' ~/.claude/skills/references/five-axis-review.md", "description": "Skill file gained a 'Reviewing proposals/spec docs' subsection", "expected": ">= 1", "item_id": "code-review-checklist-additions-for-spec-docs", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "code-review-checklist-additions-for-spec-docs", "needs": [], "notes": "Append to `~/.claude/skills/references/five-axis-review.md` (under a\nnew \"Reviewing proposals/spec docs\" subsection) a bullet that says:\n\"for each user-facing choice, the doc should state a default, a\nconsequence, and a recommendation \u2014 conditional phrasing without a\ndefault pushes the decision back onto the user.\"\n\nThen promote the source blind-spot:\n uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py \\\n triage 2026-05-03-083859-proposals-doc-decidability-gap \\\n --action promote --todo-id code-review-checklist-additions-for-spec-docs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a 'decidability check' item for proposals/spec docs", "wid": "w1"}, {"evidence": null, "item_id": "code-review-checklist-additions-for-spec-docs", "needs": ["w1"], "notes": "Add a sibling bullet under the same \"Reviewing proposals/spec docs\"\nsubsection: \"when a proposals doc includes a stress-test that\nconcludes 'all proposals survive', flag the self-grading risk and\nrequire either (a) a 'strongest argument against each proposal'\nsubsection in the same doc, or (b) an independent reviewer for the\nstress-test step.\"\n\nThen promote:\n uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py \\\n triage 2026-05-03-084354-stress-test-self-bias \\\n --action promote --todo-id code-review-checklist-additions-for-spec-docs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a 'self-bias guard' item for stress-test sections", "wid": "w2"}, {"evidence": null, "item_id": "code-review-checklist-additions-for-spec-docs", "needs": ["w1"], "notes": "Add the third bullet: \"specs with a user-approval gate must declare\nthe response shape \u2014 yes/no per question, what 'accept defaults'\nselects, how partial acceptance cascades into downstream\nparameterised work \u2014 before reaching the open-questions section.\"\n\nThen promote:\n uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py \\\n triage 2026-05-03-084923-spec-approval-ergonomics \\\n --action promote --todo-id code-review-checklist-additions-for-spec-docs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an 'approval ergonomics' item for specs with user-approval gates", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "fix only the Home.tsx empty-state issue from PR #86 without also restoring strict row-count assertions in `platform-index-cold-load.spec.ts`", "why": "the loosened `count > 0` assertion will not detect partial-row cold-load regressions, which is exactly the failure mode the original test was guarding."}, {"INSTEAD": "add a login-specific prompt path instead.", "dont": "remove the hosted-submit env-token guard globally", "why": "CI and non-interactive submit flows depend on env-token precedence"}, {"INSTEAD": "keep one retry and handle only persistent mismatch differently.", "dont": "replace the results explorer retry with a generic empty state", "why": "transient DuckDB snapshot ordering can recover on retry"}, {"INSTEAD": "verify current behavior first.", "dont": "resolve or dismiss a GitHub review thread solely because it is outdated", "why": "later commits may or may not have fixed it"}, {"INSTEAD": "use POSIX shell patterns or justify a scoped shell change.", "dont": "switch the whole Makefile to bash to fix one trap", "why": "these recipes currently run under `/bin/sh` on common Linux hosts"}], "approach": "Treat each work unit as a small targeted fix. Before editing, reopen the\ncurrent `origin/develop` file and the linked GitHub review comment, then\nconfirm whether a newer commit already handled the issue. Prefer regression\ntests around each behavior rather than broad rewrites.\n\nFor Makefile work, keep shell syntax POSIX-compatible unless the target\nexplicitly sets `SHELL` and all callers/tests are updated. For DONE-item\nverification text, follow the policy in `description`: edit in place to\nkeep historical commands portable and runnable.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Audit and fix the Codex web-agent review findings that were left on merged\nPRs during the 2026-04-24 through 2026-05-01 commit window.\n\nSource scan:\n - Git range: `origin/develop` commits since `2026-04-24 00:00:00 -0400`.\n - PR-numbered squash merges scanned: 68 merged PRs, #19 through #94 where\n present in the commit subjects.\n - Codex author matched: `chatgpt-codex-connector`.\n - Inline Codex findings found: 26 comments across 17 merged PRs.\n\nNot every unresolved GitHub thread still requires a code change. Several\ncomments were fixed in later PRs but remain unresolved/outdated on GitHub.\nThis TODO keeps the active work units focused on findings that still looked\nunresolved against `origin/develop` at scan time, with enough source context\nto re-check before editing.\n\nPolicy decision (resolves prior `open_questions` ambiguity): historical\nDONE-item verification commands are kept executable. If a Codex finding\nidentifies a portability or correctness defect in a DONE-item verification\nblock, fix it in place. Rationale: those commands serve as runnable\ndocumentation of how the policy was confirmed, so anyone re-verifying the\nhistorical decision (forks, recreated rulesets, future audits) needs them\nto work. This policy applies to w11, w12, and w13 below; future sweeps\ninherit the same rule.", "id": "codex-pr-review-followups-week-2026-05-01", "preserves": ["DONE-item verification commands edited under this TODO remain executable on a clean fork (no hard-coded ruleset IDs, no repo-specific account paths).", "Environment tokens remain the normal precedence for submit/auth resolution outside the explicit `auth login` prompt path.", "Results explorer Home keeps the transient empty-results retry, but persistent snapshot mismatch must not render an endless spinner.", "TODO and DONE YAML edited by this work remains schema-valid and graph-valid.", "The path classifier continues to include deleted files via `ACDMRT` and content-only deletion PRs keep the intended cheap guard path.", "`worktree-claim` remains compatible with `/bin/sh` on Ubuntu/dash; do not reintroduce bash-only `ERR` traps.", "`worktree-release` and `worktree-pool-reset` must not discard dirty pool worktrees without an explicit force path."], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Address unresolved Codex PR review follow-ups from the week ending 2026-05-01", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/DONE/main/active/codex-pr-review-followups-week-2026-05-01.yaml", "description": "This TODO item validates cleanly", "expected": "Validation passes with no schema or DAG errors", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO dependency graph remains valid", "expected": "Graph validation passes", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/scripts/test_blind_spot_tools.py -q", "description": "w1 - blind-spot frontmatter stamping regression test", "expected": "All blind-spot tests pass, including the new block/sequence-shaped status field regression", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 3}, {"command": "tmp_home=$(mktemp -d); trap 'rm -rf \"$tmp_home\"' EXIT; mkdir -p \"$tmp_home/tmp/benchbox-lock-probe\"; touch \"$tmp_home/tmp/benchbox-lock-probe/test.lock\"; HOME=\"$tmp_home\" PATH='/usr/bin:/bin' BENCHBOX_TEST_LOCK_DIR='~/tmp/benchbox-lock-probe' make test-unlock; test ! -e \"$tmp_home/tmp/benchbox-lock-probe/test.lock\"", "description": "w2 - test-unlock honors BENCHBOX_TEST_LOCK_DIR with ~", "expected": "Resolved lock path expands ~ to $HOME and deletes the stale lock without requiring uv on PATH", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/integration/worktree/test_worktree_release_dirty_guard.py -q || echo 'test must be created by w3 implementation'", "description": "w3 - worktree-release refuses dirty trees without FORCE=1", "expected": "New test exercises a pool worktree with staged changes and confirms release exits nonzero without FORCE=1", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/integration/worktree/test_worktree_claim_benchbox_scratch.py -q || echo 'test must be created by w4 implementation'", "description": "w4 - .benchbox scratch files do not block claim", "expected": "New test seeds a free pool slot with .benchbox/foo and confirms claim succeeds", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest tests/integration/worktree/test_worktree_claim_signal_exit.py -q || echo 'test must be created by w5 implementation'", "description": "w5 - INT/TERM trap exits nonzero", "expected": "New test sends INT mid-claim under /bin/sh and confirms recipe exits nonzero with cleanup performed", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 7}, {"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml", "description": "w6 / w7 - Step 5 TODO remains schema-valid after edits", "expected": "Validation passes; last_updated is full ISO date-time; due_date is max(step_3a_merged_at, step_4_merged_at) + 30 days", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 8}, {"command": "uv run -- python -m pytest tests/unit/workflows/test_develop_post_merge_metrics.py -q || echo 'fixture test does not exist yet - w8 must create it'", "description": "w8 - develop-post-merge metrics jq fixture for two failed jobs", "expected": "Two failed jobs at different completed_at produce develop_red_detected_at equal to the earlier timestamp", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 9}, {"command": "cd results-explorer && npm run test -- src/pages/__tests__/Home.test.tsx", "description": "w9 - Home.tsx persistent-mismatch unit test", "expected": "Home tests pass; new test asserts persistent empty-results renders an actionable error/empty state, not a spinner", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 10}, {"command": "grep -E 'toHaveCount\\(' results-explorer/e2e/failures/platform-index-cold-load.spec.ts", "description": "w9 (related) - cold-load e2e restores strict row-count assertions", "expected": "Test contains explicit toHaveCount(N) assertions, not just count > 0", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 11}, {"command": "uv run -- python -m pytest tests/unit/cli/commands/test_auth.py -q", "description": "w10 - benchbox auth login prompt with env token set", "expected": "All auth tests pass; new test sets BENCHBOX_SUBMIT_TOKEN, runs auth login without --token, and confirms keyring stores prompted value", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 12}, {"command": "! grep -nE 'rulesets/[0-9]+' _project/DONE/main/active/dev-loop-step-3a-path-based-ci-skip.yaml", "description": "w11 - Step 3a ruleset verification is repo-instance-agnostic", "expected": "No matches (hard-coded numeric ruleset ID has been replaced by dynamic lookup); grep exit 1 inverted to 0", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 13}, {"command": "grep -nE 'baseRefName' _project/DONE/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml", "description": "w12 - Step 3 verification asserts develop-targeted PR base", "expected": "baseRefName guard present before gh run list ... --event pull_request", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 14}, {"command": "grep -nE 'gh pr list[^|\\\\n]*--state[^|\\\\n]*--base develop' _project/DONE/main/planning/dev-loop-step-0-verify-assumptions.yaml", "description": "w13 - Step 0 secondary probe is scoped", "expected": "Secondary probe includes --state and --base develop filters", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 15}, {"command": "test -f _project/audits/codex-thread-rescan-week-2026-05-01.md && [ \"$(grep -cE '^## ' _project/audits/codex-thread-rescan-week-2026-05-01.md)\" -ge 3 ]", "description": "w14 - rescan audit artifact exists with required sections", "expected": "File exists with at least three ## section headers (Resolved / Already-fixed / Still actionable)", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 16}], "work": [{"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #59, P2, `_project/scripts/sweep_blind_spots.py`\n - https://github.com/joeharris76/BenchBox/pull/59#discussion_r3164438197\n\nFinding: `replace_frontmatter_field()` replaces only the first line that\nstarts with `status:` or `todo_id:`. If a malformed finding has a block\nor sequence-shaped value, triage can leave orphaned continuation lines\nbehind and make the markdown file unparsable.\n\nCurrent evidence on `origin/develop`: lines 223-228 still use a\nregex-line substitution over raw frontmatter. That preserves formatting\nfor normal scalar fields, but it does not replace the whole YAML node.\n\nRecommended fix: parse the frontmatter into a YAML node tree or use a\nsafer text rewrite that can remove the full field block before inserting\nthe normalized scalar. Add a regression test that starts with\n`status:\\n - open`, stamps `dismissed`, and verifies the result parses\nand contains exactly scalar `status: dismissed`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make blind-spot frontmatter stamping replace complete YAML nodes", "wid": "w1"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #93, P2, `benchbox/cli/commands/auth.py`\n - https://github.com/joeharris76/BenchBox/pull/93#discussion_r3172049089\n\nFinding: `auth.login()` calls `refresh_submission_token()` when\n`--token` is omitted. That helper intentionally raises if\n`BENCHBOX_SUBMIT_TOKEN` or `BENCHBOX_SERVICE_TOKEN` is active, so a user\ncannot use the login prompt to bootstrap/update keyring credentials from\na shell that also has an environment token.\n\nCurrent evidence on `origin/develop`: `auth.login()` calls\n`refresh_submission_token()` for the prompt path; the helper in\n`submit_auth.py` (`refresh_submission_token` and its env-token check)\nstill rejects active env tokens before prompting.\n\nRecommended fix: keep env-token precedence and refresh safety for\nnon-login flows, but give `auth login` an explicit prompt/store path\nthat bypasses the env-token refresh guard. Add CLI coverage that sets\n`BENCHBOX_SUBMIT_TOKEN`, runs `benchbox auth login` without `--token`,\nand confirms the prompt value is saved.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Allow `benchbox auth login` to prompt even when an env token is set", "wid": "w10"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #80, P2, `_project/DONE/main/active/dev-loop-step-3a-path-based-ci-skip.yaml`\n - https://github.com/joeharris76/BenchBox/pull/80#discussion_r3170714532\n\nFinding: a DONE-item verification command calls\n`gh api repos/joeharris76/BenchBox/rulesets/15611785`, which is\nrepository-instance-specific. Forks or recreated rulesets can make this\nhistorical checklist fail even if the policy is configured correctly.\n\nCurrent evidence on `origin/develop`: line 448 still hard-codes\n`rulesets/15611785`.\n\nPer the policy decision in `description` above, edit the DONE file\nin place rather than closing as no-op.\n\nRecommended fix: replace the command with a dynamic ruleset lookup that\nfilters for develop-targeting required status checks before asserting\n`ci-required-result`. Pattern:\n`gh api repos/{owner}/{repo}/rulesets --jq '.[] | select(.target == \"branch\" and (.conditions.ref_name.include // []) | any(. == \"refs/heads/develop\")) | .id'`\nthen pipe the resolved id back into the existing `--jq` extraction.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove hard-coded repository ruleset ID from Step 3a verification", "wid": "w11"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #67, P2, `_project/TODO/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml`\n - https://github.com/joeharris76/BenchBox/pull/67#discussion_r3168098551\n\nFinding: the verification command for routine develop PR job count can\ninspect the latest `pull_request` `test.yml` run for the current branch\nwithout proving the run came from a PR targeting `develop`. A\nmain-targeted PR run could produce a false failure.\n\nCurrent evidence on `origin/develop`: the item has moved to\n`_project/DONE/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml`;\nline 239 uses `gh run list --workflow test.yml --branch \"$branch\"\n--event pull_request --limit 1` before `gh run view`.\n\nPer the policy decision in `description` above, edit the DONE file\nin place rather than closing as no-op.\n\nRecommended fix: resolve the current branch's PR and assert\n`baseRefName == develop` before selecting the run. Pattern:\n`pr_base=$(gh pr view \"$branch\" --json baseRefName --jq .baseRefName); [ \"$pr_base\" = \"develop\" ] || { echo \"PR for $branch targets $pr_base, not develop\"; exit 1; }`\nthen proceed with the existing `gh run list ... --event pull_request`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Step 3 job-count verification select develop-targeted PR runs", "wid": "w12"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment (newly tracked \u2014 was acknowledged in inventory\nbut not assigned to a work unit in the prior revision):\n - PR #67, P2, `_project/DONE/main/planning/dev-loop-step-0-verify-assumptions.yaml`\n\nFinding: the Step 0 secondary-signal probe runs\n`gh pr list --json comments` without `--state`, `--base`, or `--limit`.\nThat scans the full PR history (open + closed + merged across all base\nbranches), which is slow on busy repos, returns a different population\nthan the primary probe (which is develop-merged-recent), and can\nrate-limit the gh client mid-audit.\n\nCurrent evidence on `origin/develop`: line 83 of\n`_project/DONE/main/planning/dev-loop-step-0-verify-assumptions.yaml`\nstill has the unscoped form.\n\nPer the policy decision in `description` above, edit the DONE file\nin place rather than closing as no-op.\n\nRecommended fix: scope the secondary probe to match the primary probe's\npopulation. Pattern:\n`gh pr list --state merged --base develop --limit 50 --json number,comments`\nthen grep the comments payload for the `pr-refresh` / `make pr-refresh`\n/ `merged develop into branch` markers as today.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Scope Step 0 secondary-probe `gh pr list --json comments` query", "wid": "w13"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": ["w1", "w10", "w11", "w12", "w13", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "Re-run the PR thread scan for `chatgpt-codex-connector` on the same\nmerge window and record which comments are now fixed, stale, or still\nactionable. Do not mark a thread resolved only because it is outdated;\nverify the current code or TODO text first.\n\nRequired output artifact (resolves prior ambiguity):\n - Write findings to a new file under `_project/audits/` named\n `codex-thread-rescan-week-2026-05-01.md`.\n - File must contain three sections: (1) Resolved by w1-w13 (one row\n per thread, link to commit/PR that fixed it), (2) Already-fixed by\n earlier merges (one row per stale-but-ok thread, link to the merge\n that fixed it), (3) Still actionable (one row per thread that\n neither this TODO nor any earlier merge addressed \u2014 promote each\n to a new TODO or blind-spot).\n - Cross-check: diff `_project/blind-spots/` for entries dated within\n 2026-04-24 .. 2026-05-01 and either fold them into the audit or\n mark out-of-scope with a one-line rationale. This addresses the\n framework-gap finding in\n `_project/blind-spots/2026-05-01-130000-codex-followups-todo-misses-coverage-downgrades.md`.\n\nFindings that looked already fixed or stale during the 2026-05-01 scan:\n - PR #56 date sorting: current code uses `fmt_date(...)` before\n sorting/listing blind-spot dates.\n - PR #64 gitignore semantics: current workflow uses\n `git check-ignore -z -v --no-index --stdin`, not pathspec matching.\n - PR #64 PR-base reuse: current `pr-open` reuses PRs only with\n `gh pr list --base develop --head \"$CURRENT\"`.\n - PR #66 Step 0 merge-queue signal: current TODO explicitly says not\n to infer merge-queue availability from `.has_issues`.\n - PR #66 Step 1 lock-path command: current verification uses valid\n Python syntax.\n - PR #66 Step 3 unsupported `gh run list --json jobs`: current\n verification lists run IDs and uses `gh run view` for jobs.\n - PR #66 Step 6 machine-specific Makefile path: current TODO no longer\n references `/Users/joe/Developer/BenchBox/Makefile`.\n - PR #70 content-guard aggregator exit: current `.github/workflows/pr.yml`\n exits immediately on content-guard failure before content-only\n success messages.\n - PR #75 and PR #76 non-POSIX `ERR` trap: current claim path uses a\n POSIX `EXIT INT TERM` trap, though w5 tracks the remaining signal\n exit problem.\n - PR #75 and PR #76 PR-list cap: current pool status/sweep use\n `--limit 1000` plus per-branch fallback lookups.\n - PR #79 deleted-file classifier: current diff filter is `ACDMRT`,\n including deletions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-audit stale Codex threads after fixes and record which ones are closed", "wid": "w14"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #71, P2, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/71#discussion_r3169224322\n\nFinding: `tests/conftest.py` resolves `BENCHBOX_TEST_LOCK_DIR` with\n`Path(...).expanduser()`, but `make test-unlock` removes\n`$LOCK_DIR/test.lock` directly. If the environment variable contains a\nliteral `~/tmp`, pytest and the Make target disagree on the lock path.\n\nCurrent evidence on `origin/develop`: `test-unlock` lines 58-61 still\nset `LOCK_DIR=\"$${BENCHBOX_TEST_LOCK_DIR:-$$HOME/.benchbox}\"` and run\n`rm -f \"$$LOCK_DIR/test.lock\"` without expanduser-equivalent handling.\n\nRecommended fix: resolve the lock directory through a tiny `uv run --`\nPython one-liner or an existing helper so the Make target matches\n`tests.conftest._get_test_lock_path()`. Add or update coverage for the\n`BENCHBOX_TEST_LOCK_DIR=~/...` case.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand BENCHBOX_TEST_LOCK_DIR consistently in `make test-unlock`", "wid": "w2"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #73, P1, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/73#discussion_r3169372557\n\nFinding: `worktree-release-locked` detaches and hard-resets a pool\nworktree after checking the PR is merged, but it does not first refuse\nstaged, unstaged, or non-ignored untracked changes. A user can lose local\nedits when releasing a slot.\n\nCurrent evidence on `origin/develop`: lines 1057-1072 perform\n`git checkout --detach origin/develop` and `git reset --hard\norigin/develop` without the dirty guard that `worktree-pool-reset-locked`\nhas at lines 1177-1183.\n\nRecommended fix: before detach/reset, compute dirty state with the same\n`.benchbox` exclusion used by pool status/reset. Refuse by default and\nallow discard only with `FORCE=1`, preserving the existing PR-merged\nsafety check.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refuse `worktree-release` on dirty pool worktrees unless forced", "wid": "w3"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #73, P2, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/73#discussion_r3169372563\n\nFinding: `worktree-claim-attempt` treats any `git status --porcelain`\noutput as dirty, but pool status/reset already ignore expected\nuntracked `.benchbox/` scratch artifacts. A released slot with harmless\n`.benchbox/*` files can become unclaimable.\n\nCurrent evidence on `origin/develop`: line 1032 uses raw\n`git status --porcelain`; line 1033 requires it to be empty. Other pool\ncommands filter `^\\?\\? \\.benchbox(/|$)`.\n\nRecommended fix: reuse the same filtered dirty check in\n`worktree-claim-attempt`, while still treating\n`.benchbox/claim_in_progress` as an aborted claim marker that must not\nbe ignored.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ignore expected `.benchbox` scratch files when claiming free pool slots", "wid": "w4"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #78, P1, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/78#discussion_r3170491244\n\nFinding: the claim cleanup handler is registered for `EXIT INT TERM`,\nbut the handler does not exit when invoked by INT/TERM. In `/bin/sh`\nimplementations such as dash, the trap can swallow cancellation and let\nthe recipe continue or return success.\n\nCurrent evidence on `origin/develop`: lines 1014-1022 define\n`cleanup()` and install `trap cleanup EXIT INT TERM`; the function has no\nsignal-aware exit path.\n\nRecommended fix: split cleanup from signal handling, or have the\nINT/TERM trap run cleanup and then exit with a nonzero status. Keep the\nrecipe POSIX-sh compatible; do not reintroduce a bash-only `ERR` trap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make INT/TERM claim-trap handling exit nonzero after cleanup", "wid": "w5"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #70, P2, `_project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml`\n - https://github.com/joeharris76/BenchBox/pull/70#discussion_r3168890227\n\nFinding: Step 5 says the measurement window runs after Steps 1-4 plus\nStep 3a, but w1 still says \"post-Step-4-merge\" and the notes start the\ntimer from Step 4 alone.\n\nCurrent evidence on `origin/develop`: lines 33-42 still describe a\nStep-4-only timer. In the actual merge order Step 3a landed before\nStep 4, so the current `due_date: 2026-05-30` may still be valid;\nverify before changing dates.\n\nRecommended fix: update w1's summary and notes to define the start\ntimestamp as the later of Step 3a and Step 4 merge times. Confirm\nwhether the existing `2026-05-30` due date remains correct (compute\n`max(step_3a_merged_at, step_4_merged_at) + 30 days`); only adjust\n`due_date` if the math diverges.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct Step 5 measurement-window start description in TODO", "wid": "w6"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #83, P2, `_project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml`\n - https://github.com/joeharris76/BenchBox/pull/83#discussion_r3171491441\n\nFinding: `metadata.last_updated` is `2026-04-30`, but the TODO schema\nexpects a date-time value.\n\nCurrent evidence on `origin/develop`: line 234 has\n`last_updated: \"2026-04-30\"`. The repo convention elsewhere is full\nISO 8601 with `Z` (e.g. this TODO's own `2026-05-01T13:30:00Z`).\n\nRecommended fix: change `last_updated` to a full ISO date-time stamped\nat the time of edit. Independent of w6 \u2014 they happen to touch the same\nfile but have unrelated acceptance criteria.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Convert Step 5 `last_updated` to ISO date-time as schema expects", "wid": "w7"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #81, P2, `.github/workflows/develop-post-merge.yml`\n - https://github.com/joeharris76/BenchBox/pull/81#discussion_r3170954637\n\nFinding: the metrics job computes `develop_red_detected_at` using\n`max` over failed lint/fast-test completion timestamps. If both fail,\nthis records the last failure and understates time to revert/recover.\n\nCurrent evidence on `origin/develop`: lines 326-330 select failed\n`lint`/`fast-test` jobs and use `.completed_at] | max // null`.\n\nRecommended fix: use the earliest failed completion timestamp, e.g.\n`min`, and add a local jq regression fixture with two failed jobs at\ndifferent completion times.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record first failed post-merge job completion as red-detection time", "wid": "w8"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #86, P1, `results-explorer/src/pages/Home.tsx`\n - https://github.com/joeharris76/BenchBox/pull/86#discussion_r3171854306\n\nFinding: when `listResults()` returns `[]` while meta-leaderboard data\nis present, the Home page retries once. If the retry also returns empty,\n`hasInconsistentEmptySnapshot` remains true and the page renders\n`Loading results...` forever instead of surfacing a recoverable error or\nempty-state message.\n\nCurrent evidence on `origin/develop`: lines 60-76 implement a one-shot\nretry, while lines 159-162 still render the loading spinner whenever\n`hasInconsistentEmptySnapshot` is true.\n\nRecommended fix: after the one retry has completed, render an\nactionable error/empty state for persistent mismatch. Preserve the\ntransient retry path. Add a Home unit test in\n`results-explorer/src/pages/__tests__/Home.test.tsx`.\n\nRelated blind-spot (separate regression on the same PR):\n - `_project/blind-spots/2026-05-01-103002-explorer-cold-load-test-coverage-weakened.md`\n - PR #86 also weakened `platform-index-cold-load.spec.ts` from strict\n row counts (`toHaveCount(4)` / `toHaveCount(2)`) to `count > 0`.\n Whoever picks up w9 should restore strict assertions in the same\n PR or open a follow-up \u2014 the cold-load partial-row regression is a\n different failure mode than the Home empty-state and the loosened\n assertion will not catch it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle persistent results-explorer snapshot mismatch without an infinite spinner", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep the diff minimal and add the regression test the work unit calls for.", "dont": "add new helpers, abstractions, or refactors beyond what the work unit names", "why": "the prior sweep ran into review churn from incidental refactors"}, {"INSTEAD": "open one PR per work unit (or per `needs:`-linked pair).", "dont": "batch unrelated work units into one PR", "why": "each Codex finding lands a focused review thread and a multi-finding PR loses the fix\u2194thread mapping"}, {"INSTEAD": "the test is the primary protection against re-introduction.", "dont": "skip writing the regression test 'because the fix is obvious'", "why": "every Codex finding here describes a class of regression that already shipped once"}, {"INSTEAD": "refile any DONE-related finding as a new TODO instead of editing in place.", "dont": "touch `_project/DONE/main/**` from any work unit", "why": "completed items are the source-of-truth for prior sweeps and audits"}, {"INSTEAD": "(not specified)", "dont": "widen the changes to also fix the items listed under `description:` as already-addressed (PR #99/#112/#134/#154)", "why": "they are intentionally out of scope; the relevant fixes already exist or are tracked elsewhere."}], "approach": "Implementation strategy mirrors the prior sweep TODO\n(`codex-pr-review-followups-week-2026-05-01`):\n\n- **Each work unit is independently shippable.** Don't batch unless\n units share a file. Already merged in this revision: w6 (PR #132,\n one template file) and w10 (PR #147, one validator function). Soft\n pairings expressed via `needs:` so they bundle in the same PR:\n w16 \u2192 w3 (`platform_options.py`), w21 \u2192 w11 (platform readiness flow).\n- **Read the cited file before editing.** The \"Current evidence on\n `origin/develop`\" section names the line range so you can confirm\n the issue still exists at fix time; the codebase is moving fast and\n a finding may already be irrelevant.\n- **Land tests with each fix.** Every work unit has a per-unit\n `Verification:` block with a specific pytest/vitest command \u2014 run\n it locally before pushing. Most regression tests are new fixtures;\n a few are mutations of existing test files.\n- **Security findings (w3 + w16, w10) are P0-tier within the P1\n batch.** Land them first or in parallel \u2014 PR #113's redaction gap\n leaks credentials, PR #147's cwd path enables code execution.\n- **Workflow change (w22) needs CI verification.** It modifies a\n GitHub Actions workflow that runs only on `develop`; review the\n next sync-to-published run after merge to confirm the\n deterministic SHA pin holds.\n- **w8 has an open question** about the precise local-envelope\n predicate (scheduler_address only, or also cluster_mode). Resolve\n against the original OOM repro before editing \u2014 see `open_questions`\n below.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-04T00:00:00Z", "completed_pr": null, "created_at": "2026-05-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 123, "reason": "The TODOs (`read-primitives-approximate-aggregate-queries.yaml`,\n`write-primitives-sketch-persistence-category.yaml`,\n`read-primitives-approximate-aggregates-dataframe-coverage.yaml`,\n`results-explorer-uat-multi-scale-corpus-sweep.yaml`) are now in\n`_project/DONE/` and their verification commands are no longer\nexecuted. The findings (wildcard `--queries`, `find -newer` arity,\nDuckDB filename glob, `$(date)` recomputation across midnight,\nDataFusion default SQL mode) suggest the historical \"verified\"\nchecks may have false-positively reported success during\ncompletion, but no current code change is required. Capture the\ncommon patterns in a future revision of\n`_project/audits/codex-weekly-sweep-template.md` so the next\nsweep flags wildcard `--queries`, `find -newer` multi-arg, and\n`$(date)` reads inside verification blocks before TODOs land.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: The sweep process was redesigned into the automated pr-review-followups routine with its own living template (_project/audits/pr-review-sweep-template.md); original capture note superseded.", "summary": "Audit the four DONE-state TODOs whose verification commands Codex flagged as buggy (PRs #110, #117, #121, #122)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 124, "reason": "`results-explorer-uat-methodology-blind-spot-remediation` is in\nDONE; its acceptance criteria required all linked blind-spots to\nend as `merged-to-todo` or `dismissed`, but\n`2026-05-03-081920-uat-cross-scale-deliverable-not-guarded` ended\nas `actioned`. This is a convention question, not a runtime\ndefect: either widen the criteria to allow `actioned` for\nconvention-only remediations, or amend the historical record.\nTrack the convention decision separately so future TODO\nacceptance criteria align with the broader blind-spot lifecycle.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: blind-spots/README.md lifecycle already lists 'actioned' as a legitimate terminal state ('a fix landed'), so acceptance criteria accommodate it; the historical mismatch is a one-off record question, not a standing tracked decision.", "summary": "Decide convention for marking blind-spot remediation TODOs Completed when linked findings end in `actioned` instead of `merged-to-todo`/`dismissed` (PR #163)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 125, "reason": "`results-explorer-uat-methodology-blind-spot-remediation` is in\nDONE; the flagged YAML-level multi-line `command:` would have\nreported \"all triaged\" even when status was unknown. As a\nforward-looking control, extend\n`_project/scripts/validate_todo.py` (or the schema) to reject\n`verification.command` values that contain unescaped newlines,\nso the same shape can't ship in a future TODO.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: validate_todo.py already rejects verification.command values spanning multiple physical lines unless 'command: |' is used; control exists (line ~152-159).", "summary": "Audit the multi-line shell `command:` parsing risk in TODO `verification:` blocks (PR #161)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 126, "reason": "The fix in w18 needs a live MotherDuck account to confirm runs\nland on the configured non-default database. Tracked as part of\nthe existing cloud verification TODOs; not blocking the local\nfix.\n", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Cloud verification of MotherDuck database propagation (w18)"}], "deps": [], "description": "Audit and fix the Codex web-agent review findings that were left on merged\nPRs during the 2026-04-26 through 2026-05-03 commit window \u2014 the second\nweekly Codex sweep, building on the prior sweep\n(`codex-pr-review-followups-week-2026-05-01`, merged via PRs #99/#104) and\nthe rescan audit at `_project/audits/codex-thread-rescan-week-2026-05-01.md`\nwhich closed out PRs #19\u2013#94.\n\nSource scan:\n - PR range: #95 through #170 merged to `develop` since 2026-04-26.\n - API source: `gh api repos/joeharris76/BenchBox/pulls//comments`.\n - Author filter: `chatgpt-codex-connector[bot]`.\n - Inline Codex findings found: 36 comments across 29 PRs.\n\nNot every unresolved GitHub thread still requires a code change. The\nfollowing groups of comments were filtered out at scan time and are\nintentionally NOT in the work breakdown below:\n\n - **Already addressed by a follow-up fix-PR:**\n - PR #99 (Makefile `test-unlock` `uv run` resolution) \u2014 fixed: the\n current target uses pure shell `$HOME` expansion, no `uv run`.\n - PR #112 (`expected_value_min/max` enforcement in `_check_validation_query`)\n \u2014 fixed in PR #114 (`_check_validation_query` now branches on\n `expected_value_min/max`).\n - PR #154 (`benchbox results --paths` reparses files) \u2014 fixed in\n PR #156, which restructured `results()` to use `if/elif/else` so\n `--paths` no longer also calls `show_results_summary()`.\n\n - **Already tracked in another open TODO:**\n - PR #134 (Redshift theta-merge validation queries) \u2014 the underlying\n per-platform validation-override gap is filed as blind-spot\n `2026-05-02-155448-validation-query-no-per-platform-override`\n (`merged-to-todo` \u2192 `write-primitives-architecture-fixes`). The\n Redshift override is unsafe to ship until that architectural fix\n lands; do not duplicate it here.\n\n - **Comments on TODO YAML files now in DONE \u2014 verification commands\n flagged as buggy but the work is closed:**\n - PR #110 (`approx_*` / `sketch_*` wildcard `--queries` in\n `read-primitives-approximate-aggregate-queries.yaml` and\n `write-primitives-sketch-persistence-category.yaml`).\n - PR #117 (DuckDB filename glob and `find -newer` arity in\n `results-explorer-uat-multi-scale-corpus-sweep.yaml`).\n - PR #121 (DataFusion smoke check uses SQL mode in\n `read-primitives-approximate-aggregates-dataframe-coverage.yaml`).\n - PR #122 (matrix-shaping checkpoint placement and `$(date)`\n recomputation in\n `results-explorer-uat-multi-scale-corpus-sweep.yaml`).\n Each finding may indicate the historical verification step\n false-positively reported success, but the items are closed and\n the verification commands are no longer used. Treat as a\n retrospective protocol-hygiene note for the weekly sweep template,\n not as code work to schedule.\n\n - **Comments on DONE items where the action is process / convention,\n not code:**\n - PR #161 (`results-explorer-uat-methodology-blind-spot-remediation`\n triage `command:` was split across YAML lines, breaking shell\n parsing). The TODO is now in DONE and the verification block is\n not re-run.\n - PR #163 (same TODO marked `Completed` while one linked blind-spot\n ended in `actioned`, not the criteria-required `merged-to-todo`\n or `dismissed`). Either widen the convention to allow `actioned`\n for convention-only remediations or amend the historical record\n \u2014 neither is a runtime defect.\n Both are noted under `deferred` below.\n\nPolicy carry-over from the 2026-05-01 sweep: historical DONE-item\nverification commands stay executable. If a Codex finding identifies a\nportability or correctness defect in a DONE-item verification block, fix\nit in place. The current sweep has no DONE-block code-fix items because\nthe four DONE-state TODO comments above are limited to verification\ncommands that are no longer used \u2014 recording them in `description` is\nsufficient, no work unit is required.\n\nTwo consolidations from the original 36-comment scan:\n\n - **w6 covers both PR #132 findings** (Axis 2 listing guidance + Axis 3\n end-of-day bound) \u2014 same template file, same review thread.\n - **w10 covers both PR #147 findings** (dry-run-when-validator-absent\n P1 + cwd code-execution P1 security) \u2014 same `_load_submission_validator_module`\n function; the security fix and the UX fix are the same refactor.\n\nEach work unit below cites:\n - The source PR(s) and Codex review URL(s).\n - A short restatement of the finding.\n - Current evidence on `origin/develop` (file:line) so the fix can be\n verified before editing.\n - The recommended fix and a runnable verification command, with\n enough specificity to make the change without re-reading the\n original Codex comment.\n - Per-unit `must_preserve:` invariants where the fix sits next to\n load-bearing existing behavior.", "id": "codex-pr-review-followups-week-2026-05-03", "preserves": ["Existing CI gates (`make pr-preflight`, lint, ty, fast tests) must pass on every work-unit PR.", "Existing TODO/blind-spot tooling (`make blind-spots-list`, `make worktree-pool-check`, `_project/scripts/validate_todo.py`) keeps its current invocation shape \u2014 w19 changes worktree-pool-check semantics, not its CLI.", "Public CLI surfaces (`benchbox submit`, `benchbox results`, `benchbox platforms check`, `benchbox run`) keep their exit-code contracts for unrelated paths \u2014 the only changed exit codes are the ones each work unit explicitly names.", "Public results-explorer URL/facet contracts (`bm`, `sf`, `instance_or_warehouse`, etc.) keep their existing serde shape \u2014 only the read/write merge logic changes."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/DONE/main/**"}, {"kind": "only_modify", "path_glob": ".github/workflows/sync-results-data-to-published.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/audits/codex-weekly-sweep-template.md"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/platform.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/platform_readiness.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/cost/calculator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/cost/integration.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/platform_options.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/motherduck.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/motherduck.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ChartPanel.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/RunReceipt.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartMath.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/facetModel.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "tests/**"}, {"kind": "only_modify", "path_glob": "tests/integration/test_hosted_submit_validator_contract.py"}], "state": "done", "title": "Address unresolved Codex PR review follow-ups from the week ending 2026-05-03", "verifications": [{"command": "for pr in 99 102 103 105 107 110 111 112 113 114 115 117 119 121 122 127 129 131 132 134 141 144 145 147 151 154 161 163 167; do gh api repos/joeharris76/BenchBox/pulls/$pr/comments --jq '[.[] | select(.user.login == \"chatgpt-codex-connector[bot]\")] | length' | { read n; [ \"$n\" -gt 0 ] && echo \"PR #$pr: $n comments\"; }; done", "description": "All 22 work units land on develop, individually or via `needs:`-linked pairs, with a corresponding follow-up commit referenced in this TODO's `commits:` field.", "expected": "Same set of PRs returns matching counts; each work unit's PR is referenced from its corresponding commit in `commits:`.", "item_id": "codex-pr-review-followups-week-2026-05-03", "last_result": null, "last_run": null, "seq": 1}, {"command": "make pr-preflight", "description": "Lint and fast-test gate is green after each work unit's fix", "expected": "Preflight reports lint, type, and Python fast-test gates passing.", "item_id": "codex-pr-review-followups-week-2026-05-03", "last_result": null, "last_run": null, "seq": 2}, {"command": "# Re-run the same scan on PRs #95-#170 with the codex-thread-rescan template; expect every comment from this sweep to be marked Resolved or Already-Fixed.", "description": "Next weekly Codex sweep (week ending 2026-05-10) reports zero unaddressed comments from this batch.", "expected": "All 36 originally-scanned comments classify as Resolved (by w1-w22 commits) or Already-Fixed (per the description filtered set).", "item_id": "codex-pr-review-followups-week-2026-05-03", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #103, P1, `results-explorer/src/components/RunReceipt.tsx`\n - https://github.com/joeharris76/BenchBox/pull/103#discussion_r3174480692\n\nFinding: `planDownloadUrl()` returns a `.plans.json` URL\nwhenever `detail.has_plans` is truthy and `bundle_download_url` ends\nin `.json`, but `has_plans` only reflects source-side detection. The\nexplorer pipeline excludes `*.plans.json` from bundle discovery and\nonly copies the primary `result_id.json`, so the generated link 404s\nfor every published bundle.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/components/RunReceipt.tsx:148-152` \u2014\n `planDownloadUrl()` only checks `has_plans` and the suffix; no\n publication-status gate.\n - `benchbox/core/explorer_pipeline/pipeline.py:439` \u2014 bundle scan\n excludes `(\".plans.json\", \".tuning.json\", SUBMISSION_MANIFEST_SUFFIX)`,\n so plan sidecars are never copied to `output_dir/bundles/`.\n\nRecommended fix: gate the link on a publication signal rather than\n`has_plans`. Either (a) extend `DetailResult` (and the pipeline's\ndetail emitter) with a `plans_published: bool` field set when the\npipeline copies a `*.plans.json`, then check it here; or (b) keep\nthe non-link \"Plans available\" fallback (line 140) and only render\na download link when a publication-status field is true.\n\nMust preserve:\n - `has_plans=false` continues to render the \"Plans not published\"\n string (do not collapse to empty).\n - `renderPlansStatus()` still returns the non-link \"Plans\n available\" fallback when a sidecar exists but isn't published \u2014\n tested by `RunReceipt.test.tsx` if present; add coverage if not.\n\nVerification:\n - description: \"Vitest regression: `has_plans=true`,\n `plans_published=false` \u2192 no `` link rendered.\"\n - command: \"cd results-explorer && pnpm test -- RunReceipt\"\n - expected_output: \"All RunReceipt tests pass; new case asserts\n absence of plan download anchor.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Stop linking unpublished `.plans.json` sidecars from the run receipt", "wid": "w1"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comments (both PR #147, same `_load_submission_validator_module`):\n - P1, dry-run unusable when validator absent:\n https://github.com/joeharris76/BenchBox/pull/147#discussion_r3177754515\n - P1 (security), cwd-relative validator code execution:\n https://github.com/joeharris76/BenchBox/pull/147#discussion_r3177754516\n\nFinding 1 (UX): dry-run unconditionally calls\n`_validate_submission_bundle_for_dry_run()`, which exits non-zero\nwhen `scripts/validate_submission.py` cannot be found. The wheel\nonly packages `benchbox*` modules, so non-source installs cannot\nuse `--dry-run` at all.\n\nFinding 2 (security): `_load_submission_validator_module()` falls\nback to `Path.cwd() / \"scripts\" / \"validate_submission.py\"` and\n`spec.loader.exec_module(module)`. In the common case where a\npackaged install has no repo `scripts/` directory, any unrelated\nor malicious `scripts/validate_submission.py` in the user's\ncurrent directory is imported and run when they invoke\n`benchbox submit --dry-run`.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/cli/commands/submit.py:155-173` \u2014 `candidate_paths`\n includes the cwd fallback; `_load_submission_validator_module`\n raises `FileNotFoundError` when neither path resolves.\n - `benchbox/cli/commands/submit.py:176-189` \u2014\n `_validate_submission_bundle_for_dry_run` catches that and\n `ctx.exit(1)`.\n - `benchbox/cli/commands/submit.py:627-628` \u2014 `if dry_run:` is\n unconditional.\n\nRecommended fix (single refactor, both findings):\n 1. Drop the `Path.cwd() / \"scripts\" / \"validate_submission.py\"`\n candidate. Only the source-checkout-relative path\n (`Path(__file__).resolve().parents[3] / \"scripts\" /\n \"validate_submission.py\"`) is acceptable.\n 2. When the source path doesn't resolve, downgrade\n `_validate_submission_bundle_for_dry_run` to a soft warning\n (\"dry-run preview without schema validation; run from a\n source checkout for full validation\") and return cleanly.\n Reserve `ctx.exit(1)` for the case where the validator is\n found but errors during execution.\n\nMust preserve:\n - Source-checkout dry-run continues to validate the bundle\n (the original P1 from PR #147 \u2014 non-validated submissions\n must still be impossible from a checkout).\n - Service-mode dispatch (`_dispatch_service_mode`) still\n receives the post-validation guarantees on the source path.\n - All existing exit codes for fully-validated dry-run failure\n modes (compliance-class refused, non-clean-pass refused) at\n lines 600-619 are untouched.\n\nVerification:\n - description: \"Pytest: source checkout \u2192 dry-run validates and\n exits 0/1 based on bundle. Wheel install (mocked\n `Path(__file__).parents[3]/scripts` missing) \u2192 dry-run\n warns and emits preview, exits 0. Sentinel\n `./scripts/validate_submission.py` raising in cwd \u2192 NOT\n executed.\"\n - command: \"uv run -- python -m pytest tests/unit/cli/commands/test_submit.py tests/integration/test_submit_dry_run.py -v -k 'dry_run or validator or cwd'\"\n - expected_output: \"All three scenarios pass; the cwd-sentinel\n test asserts the marker file was never opened/imported.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make `benchbox submit --dry-run` safe + drop cwd-validator path (security)", "wid": "w10"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #151, P1, `benchbox/cli/platform.py`\n - https://github.com/joeharris76/BenchBox/pull/151#discussion_r3177842037\n\nFinding: `benchbox platforms check` returns a failure for any\n*available but disabled* platform when its readiness probe fails,\nbecause the `info.available and readiness_failed` branch runs\nbefore the disabled-platform branch. This is inconsistent with\nthe prior behavior where disabled platforms are informational\n(\"Available but disabled\") and do not fail the command.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/cli/platform.py:881-891` \u2014 branches in order:\n (1) `info.available and readiness_failed` \u2192 \u26a0\ufe0f, all_good=False;\n (2) `info.enabled and info.available` \u2192 \u2705;\n (3) `info.available` (i.e. available-but-disabled) \u2192 \u25cb;\n (4) else \u2192 \u274c, all_good=False.\n\nRecommended fix: reorder the branches so the\n\"available-but-disabled\" check runs before the readiness-failed\ncheck. Disabled platforms are always informational; readiness\nfailures only fail the command when the platform is also enabled.\n\nMust preserve:\n - \"Missing dependencies\" (branch 4) still fails the command.\n - \"Ready\" (branch 2) stays at \u2705 exit 0.\n - The \u26a0\ufe0f \"Environment not ready\" status still fires for\n *enabled* available platforms whose readiness probe fails \u2014\n do not silence that path.\n - `_print_readiness_details(readiness_results)` continues to\n print after every branch.\n\nVerification:\n - description: \"Pytest: enabled+available+readiness-failed \u2192\n all_good=False, exit 1. Disabled+available+readiness-failed\n \u2192 all_good=True, exit 0 (with informational \u25cb message).\n Disabled+available+readiness-passed \u2192 unchanged (still \u25cb).\"\n - command: \"uv run -- python -m pytest tests/unit/cli/test_platform_check.py -v -k 'check or readiness or disabled'\"\n - expected_output: \"All three matrix cells pass; the\n disabled+failed-readiness case now exits 0 (was: exit 1).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Stop failing `benchbox platforms check` for available-but-disabled platforms", "wid": "w11"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #102, P2, `results-explorer/src/pages/Home.tsx`\n - https://github.com/joeharris76/BenchBox/pull/102#discussion_r3174367121\n\nFinding: the home leaderboard filter `onSelect` overwrites the\nfilter array with a single-element array (`[value]`), regressing\ntrue multi-select to single-select even though URL state still\nmodels `bm`/`sf` as arrays. Users opening\n`?bm=tpch,clickbench`-style links can no longer add or remove\nindividual selections from the UI; any change collapses the\nfilter to one value, so cross-benchmark / cross-scale\ncomparisons are no longer possible from the page controls.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/pages/Home.tsx:330-340` \u2014 both the\n benchmark and scale dropdowns use\n `onSelect={(value) => setFacet(\"\", value === \"all\" ? [] :\n [value])}`.\n\nRecommended fix: switch the dropdown component to a multi-select\n(or wire a second control for adding values) that calls a toggle\nhandler \u2014 `setFacet(\"benchmark\", toggleInList(current, value))` \u2014\ninstead of replacing. Preserve the \"all\" sentinel semantics\n(still clears to `[]`).\n\nMust preserve:\n - Phase, tuning_mode, trust_tier, and date_window dropdowns at\n lines 346-375 keep their existing single-select semantics\n (they are intentionally single-valued).\n - URL `arraySerde` encoding for `bm`/`sf` is unchanged.\n - The \"All benchmarks\" / \"All scales\" sentinel entry continues\n to clear the filter to `[]`.\n\nVerification:\n - description: \"Vitest: navigate to `?bm=tpch,clickbench` then\n toggle off `tpch` \u2192 URL contains only `bm=clickbench`.\"\n - command: \"cd results-explorer && pnpm test -- Home\"\n - expected_output: \"All Home page tests pass; new toggle-off\n case asserts the remaining selection persists.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Preserve multi-select semantics for benchmark/scale cohort filters in Home", "wid": "w12"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #105, P2, `benchbox/core/cost/calculator.py`\n - https://github.com/joeharris76/BenchBox/pull/105#discussion_r3174793239\n\nFinding: the local-platform allowlist omits established local\nDataFrame platform IDs such as `datafusion-df` and `pyspark-df`.\nThose runs are treated as cloud/unknown, producing\n`cost_status=\"unavailable\"` instead of `not_applicable_local`,\nmisclassifying local results and polluting cost facets and\nempty-state logic that depend on `cost_status`.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/cost/calculator.py:230-239` \u2014 local set\n contains `datafusion`, `polars`, `polars-df`, `pandas-df`,\n `cudf-df`, `modin-df`, `dask-df`, `pyspark`, `spark` but not\n `datafusion-df` or `pyspark-df`.\n\nRecommended fix: add `datafusion-df` and `pyspark-df` to the set.\nAudit the registered platform IDs in\n`benchbox/platforms/__init__.py` and add any other DataFrame\nvariants whose primary execution is on the developer's machine.\n\nMust preserve:\n - Cloud platforms (snowflake, bigquery, redshift, databricks,\n motherduck, etc.) continue to return False from\n `is_local_platform()`.\n - The case-insensitive lookup behavior is unchanged.\n\nVerification:\n - description: \"Pytest: every DataFrame-mode platform ID\n registered with `PlatformManager` is local; representative\n cloud platforms remain non-local.\"\n - command: \"uv run -- python -m pytest tests/unit/core/cost/test_calculator.py -v -k 'local_platform'\"\n - expected_output: \"All registered DataFrame platforms classify\n as local; cloud-platform negative cases still pass.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add datafusion-df, pyspark-df (and any peer DataFrame engines) to the local-platform allowlist", "wid": "w13"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #107, P2, `results-explorer/src/lib/chartMath.ts`\n - https://github.com/joeharris76/BenchBox/pull/107#discussion_r3176628095\n\nFinding: `latencyScaleTicks` returns only predefined powers of\nten when at least two match, omitting the actual `scale.max`\n(and sometimes `scale.min`) for common ranges like 0.2\u20132 ms or\n100k\u201350M ms. The right edge of the performance bar chart then\nhas no terminal tick label, so bar lengths are plotted against\nan unlabeled endpoint and the axis can be misread.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/lib/chartMath.ts:92-101` \u2014 the log-mode\n branch filters `LATENCY_LOG_TICKS_MS` to in-range values and\n only falls back to `[scale.min, scale.max]` when fewer than\n two match.\n\nRecommended fix: always include `scale.domainMin` and\n`scale.domainMax` as endpoint ticks, then dedupe against the\nfiltered powers-of-ten list. The full set should be\n`[domainMin, ...powersOfTenInRange, domainMax]` with duplicates\nand out-of-order values removed.\n\nMust preserve:\n - Linear scale branch (lines 93-95) keeps its\n `[0, 0.25, 0.5, 0.75, 1] * domainMax` ticks unchanged.\n - The fallback `[scale.min, scale.max]` path (when no powers\n of ten match) continues to work.\n - Tick values stay sorted ascending (axis assumes ordering).\n\nVerification:\n - description: \"Vitest: ranges 0.2\u20132 ms, 100k\u201350M ms, 1\u20131000 ms\n each include both endpoints in the returned tick list.\"\n - command: \"cd results-explorer && pnpm test -- chartMath\"\n - expected_output: \"All chartMath tests pass; new endpoint-\n inclusion cases assert presence of domainMin and domainMax.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Include log-scale endpoint ticks in the latency axis", "wid": "w14"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #111, P2, `results-explorer/src/lib/facetModel.ts`\n - https://github.com/joeharris76/BenchBox/pull/111#discussion_r3176716507\n\nFinding: the alias migration for `instance_or_warehouse` is lossy\nwhen a URL contains both legacy query params (`instance_type` and\n`warehouse_size`), which the Query page still supports as\nseparate filters. `readFacetParam()` only keeps the first\nmatching alias, and `writeFacetParam()` then deletes all aliases,\nso mounting shared facet state silently drops one of the active\nfilters and changes subsequent query results after navigation.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/lib/facetModel.ts:185` \u2014\n `findFirstParam(params, [FACET_URL_KEYS[key],\n ...(FACET_URL_ALIASES[key] ?? [])])` keeps only the first hit.\n - `results-explorer/src/lib/facetModel.ts:380-382` \u2014\n `writeFacetParam` deletes every alias before writing the\n canonical key.\n\nRecommended fix: when the canonical URL key is absent and\nmultiple aliases are present, merge values across aliases\n(concatenate the arrays then dedupe) into the canonical\n`instance_or_warehouse` list.\n\nMust preserve:\n - When the canonical key IS present, it still wins (no merge\n with stale aliases).\n - Single-alias URLs continue to map cleanly with no extra\n values introduced.\n - `writeFacetParam` continues to delete all aliases when\n writing the canonical key (avoids URL bloat).\n\nVerification:\n - description: \"Vitest: `?instance_type=foo&warehouse_size=bar`\n \u2192 facet contains both; `?instance_or_warehouse=foo&instance_type=bar`\n \u2192 facet is `[foo]` (canonical wins).\"\n - command: \"cd results-explorer && pnpm test -- facetModel\"\n - expected_output: \"All facetModel tests pass; new dual-alias\n and canonical-wins cases assert the right precedence.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Preserve both legacy shape aliases when canonicalizing the URL facet model", "wid": "w15"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": ["w3"], "notes": "Source review comment:\n - PR #113, P2, `benchbox/core/results/platform_options.py`\n - https://github.com/joeharris76/BenchBox/pull/113#discussion_r3176769305\n\nFinding: when a saved config value and a registered default are\nequal, the merge path rewrites the source to `registered_default`\neven though the value already came from `database_options`. This\ncontradicts the function contract that defaults should not\noverride config-derived values and produces incorrect provenance\nin `platform_option_sources`, misleading debugging and audit\nflows that rely on origin tracking.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/results/platform_options.py:88-100` \u2014\n `build_platform_options_capture` overrides `sources[key]` to\n `\"registered_default\"` whenever the requested source is\n `registered_default` *and* the values are equal, even though\n `key` is already in `values` from `database_options`.\n\nRecommended fix: when the requested source is\n`registered_default` and `key` is already populated from\n`database_options`, leave the source as `saved_config` and\n`continue` without modifying it. Only set\n`sources[key] = \"registered_default\"` when the key was not\npreviously populated.\n\nPairs with w3: both touch `platform_options.py`; land in the same\nPR with shared test refresh.\n\nMust preserve:\n - CLI-provided options (any `requested_source` other than\n `registered_default`) continue to override saved config \u2014\n contract documented in `build_platform_options_capture`\n docstring.\n - Sanitized output of `build_platform_options_capture` returns\n only public keys (`_iter_public_options` filter intact).\n\nVerification:\n - description: \"Pytest: database_options={'foo':'bar'} +\n requested registered_default 'bar' \u2192 sources['foo'] ==\n 'saved_config'. CLI-provided 'baz' overrides saved 'bar' \u2192\n sources['foo'] == 'requested'.\"\n - command: \"uv run -- python -m pytest tests/unit/core/results/test_platform_options.py -v -k 'provenance or registered_default or saved_config'\"\n - expected_output: \"All provenance precedence tests pass; the\n new equal-default-vs-saved test asserts saved_config wins.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Preserve `saved_config` provenance when the saved value happens to equal the registered default", "wid": "w16"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #115, P2, `results-explorer/src/pages/Compare.tsx`\n - https://github.com/joeharris76/BenchBox/pull/115#discussion_r3176818720\n\nFinding: when a benchmark/scale mismatch is detected, the new\ndecision summary suppresses winner language, but the page still\nrenders `ChartPanel` in normal compare mode. `SummaryBoxPanel`\n(rendered inside ChartPanel) can still present a computed\n\"Best power / Best geomean\" platform, reintroducing the same\nwinner claim the guardrail is meant to suppress for incomparable\ncohorts.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/pages/Compare.tsx:175-178` \u2014\n `decisionSummary` gets `suppressWinnerClaims`/`suppressionReason`.\n - `results-explorer/src/pages/Compare.tsx:366-370` \u2014\n `` is rendered without suppression props.\n - `results-explorer/src/components/ChartPanel.tsx:34-71` \u2014\n `ChartPanelProps` has no suppression field; line 353 renders\n `SummaryBoxPanel` unconditionally.\n\nRecommended fix: extend `ChartPanelProps` with optional\n`suppressWinnerClaims?: boolean` and `suppressionReason?: string`,\nthread them into `SummaryBoxPanel`, and replace winner-language\n(\"Best power\", \"Best geomean\") with neutral phrasing\n(\"Highest power score in cohort (cohort mismatch \u2014 not\ncomparable)\") when suppression is on.\n\nMust preserve:\n - Compare-page rendering with NO suppression (cohort matches)\n shows the same \"Best power\" / \"Best geomean\" text as today.\n - `decisionSummary` text already produced at the top of Compare\n stays unchanged.\n - ChartPanel callers other than Compare (detail pages) keep\n working unchanged.\n\nVerification:\n - description: \"Vitest: ChartPanel mounted with\n `suppressWinnerClaims={true}` \u2192 `SummaryBoxPanel` text does\n not contain 'Best'. Without suppression \u2192 'Best power' /\n 'Best geomean' still appears.\"\n - command: \"cd results-explorer && pnpm test -- ChartPanel Compare\"\n - expected_output: \"All ChartPanel and Compare tests pass; new\n suppression test asserts neutral wording.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Propagate winner-claim suppression into the chart-level summary on the Compare page", "wid": "w17"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #119, P2, `benchbox/platforms/credentials/motherduck.py`\n - https://github.com/joeharris76/BenchBox/pull/119#discussion_r3176973035\n\nFinding: `setup_motherduck_credentials` persists the user-selected\n`database`, but no one consumes it at run time, so runs still\ndefault to `benchbox` regardless of what the wizard captured.\nMotherDuck has no platform-specific config builder registration,\nso `PlatformHookRegistry._default_builder` is used and does not\nmerge `CredentialManager` data; `MotherDuckAdapter` then falls\nback to its internal default database. The new wizard prompt is\neffectively ineffective and benchmarks may run against the wrong\ndatabase.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/platforms/credentials/motherduck.py:70-74` \u2014 saves\n `{\"database\": database, \"token_env_var\": MOTHERDUCK_TOKEN_ENV}`.\n - `benchbox/platforms/__init__.py` \u2014 has no\n `PlatformHookRegistry.register_config_builder(\"motherduck\",\n ...)` call (the surrounding modules do for databricks,\n bigquery, trino, firebolt, presto, postgresql, etc.).\n\nRecommended fix: add a `_build_motherduck_config` builder in\n`benchbox/platforms/motherduck.py` that pulls the saved\n`database` from `CredentialManager` and merges it into the\nruntime `DatabaseConfig`. Register it from\n`benchbox/platforms/__init__.py` in the same pattern used for\nother adapters.\n\nMust preserve:\n - `MOTHERDUCK_TOKEN` env var precedence stays exactly as today\n (token resolution is unaffected).\n - The existing default `database=\"benchbox\"` is the fallback\n when no saved value exists.\n - Other platforms' config builders (databricks, bigquery,\n trino, firebolt, presto, postgresql, timescaledb,\n pg-duckdb) are untouched.\n\nVerification:\n - description: \"Pytest: CredentialManager with\n `motherduck.database='alt'` \u2192 `_build_motherduck_config`\n returns DatabaseConfig with `database='alt'`. No saved\n value \u2192 falls back to 'benchbox'.\"\n - command: \"uv run -- python -m pytest tests/unit/platforms/test_motherduck.py -v -k 'config_builder or wizard or database'\"\n - expected_output: \"Both saved and fallback paths pass; live\n cloud verification deferred (see deferred[]).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Propagate the configured MotherDuck database into runtime config", "wid": "w18"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #129, P2, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/129#discussion_r3177117127\n\nFinding: `worktree-pool-check` treats any\n`.benchbox/claim_in_progress` file as an aborted slot, but\n`worktree-claim-attempt` creates that marker at the start of a\nnormal claim and removes it only at the end. Because the check\nruns without the pool lock, invoking it during an active claim\ncan fail with a false positive even when the pool is healthy \u2014\nespecially noisy for the documented periodic / cron use case.\n\nCurrent evidence on `origin/develop`:\n - `Makefile:1219-1222` \u2014 `elif [ -f\n \"$$wt/.benchbox/claim_in_progress\" ]; then` increments\n `aborted_count` with no liveness check.\n - `Makefile:1207` \u2014 `worktree-pool-check` does not acquire the\n pool lock.\n\nRecommended fix: treat a marker as aborted only when it is\n\"stale\" \u2014 older than a configurable threshold (default 10\nminutes) computed from `mtime`. A fresh marker indicates an\nin-progress claim and should be reported as `claimed` (or\nsimply skipped) instead of `aborted`.\n\nMust preserve:\n - Truly aborted markers (older than the threshold) still\n surface as failures with the existing exit-1 behavior and\n recovery hints.\n - The \"missing worktree\" check at line 1216-1217 is unchanged.\n - `extras` detection for slots beyond `POOL_SIZE` still fires.\n - The non-locking design stays \u2014 keep the check fast and\n cron-safe.\n\nVerification:\n - description: \"Manual: run `make worktree-pool-check` while\n `make worktree-claim BRANCH=...` is mid-flight. With the\n fix, no aborted false positive.\"\n - command: \"(make worktree-claim BRANCH=chore/test-pool-race &) && sleep 1 && make worktree-pool-check && wait\"\n - expected_output: \"Pool invariant check OK with no aborted\n counter increment; the concurrent claim completes normally.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Exclude in-flight claims from `worktree-pool-check` aborted-marker false positives", "wid": "w19"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #105, P1, `benchbox/core/cost/integration.py`\n - https://github.com/joeharris76/BenchBox/pull/105#discussion_r3174793237\n\nFinding: `_extract_platform_config_from_info()` has explicit branches\nonly for `snowflake`, `bigquery`, `redshift`, `databricks`. Adapters\nthat emit `platform_type: \"athena\"` (and `\"azure_synapse\"`, future\n`\"clickhouse-cloud\"`, etc.) fall through with no `region`/`cloud`\nfields, so `calculate_normalized_benchmark_cost()` warns\n`pricing region metadata missing` and forces\n`cost_status=\"unavailable\"`, blocking public cost totals in\nsubmission validation even when the adapter populated `platform_info`.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/cost/integration.py:285-368` \u2014 only the four\n listed `platform_type` branches set region/cloud. After the\n last `elif` there is no generic fallback that reads\n `platform_info[\"region\"]` / `platform_info[\"cloud_provider\"]`.\n\nRecommended fix: add a generic post-branch fallback that tries the\nsame `_value_or_default(platform_info, \"cloud_provider\", default,\ndefaulted_fields, alias=\"cloud\")` and `... \"region\", default, ...`\nlookups for any `platform_type` not covered above. Defaults should\nmirror what each cloud adapter already emits (Athena \u2192 `aws`,\nAzure Synapse \u2192 `azure`).\n\nMust preserve:\n - The four explicit branches (snowflake/bigquery/redshift/\n databricks) keep their current defaults and field names exactly\n \u2014 submission validation depends on `cluster_size_dbu_per_hour`,\n `warehouse_size`, `node_type`, etc. being populated by the\n existing branches.\n - `_defaulted_fields` annotation still reflects only fields that\n actually defaulted (no spurious additions from the fallback).\n\nVerification:\n - description: \"Pytest: athena and azure_synapse fixtures populate\n cloud/region without warnings.\"\n - command: \"uv run -- python -m pytest tests/unit/core/cost/ -v -k 'athena or synapse or extract_platform_config'\"\n - expected_output: \"All cost-extraction tests pass; new athena\n and azure_synapse fixtures assert non-default cloud/region.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extract cloud/region metadata for non-DB cloud adapters (Athena, Synapse, \u2026)", "wid": "w2"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #131, P2, `tests/integration/test_hosted_submit_validator_contract.py`\n - https://github.com/joeharris76/BenchBox/pull/131#discussion_r3177127856\n\nFinding: the parity check does not exercise the hosted or PR\nsubmit code paths it claims to protect. Both manifests are built\nby calling `_build_submission_manifest` directly with hard-coded\narguments. If `_dispatch_service_mode` or the PR packaging flow\nlater changes its call-site arguments \u2014 or mutates the manifest\nafter construction \u2014 the test will still pass and miss the\ncross-path drift its docstring describes.\n\nCurrent evidence on `origin/develop`:\n - `tests/integration/test_hosted_submit_validator_contract.py:198-211` \u2014\n two direct `sub._build_submission_manifest(...)` calls with\n identical inputs except `submission_path`.\n\nRecommended fix: drive each manifest through its corresponding\npath. For PR-flow, drive the same code that\n`benchbox submit --output ...` calls (or a path-level helper);\nfor hosted, drive the same code `_dispatch_service_mode` calls.\nAcceptable shortcut: extract a tiny\n`_build_submission_manifest_for_pr_flow()` /\n`_build_submission_manifest_for_hosted_flow()` helper that the\ntest calls without faking the rest of the dispatch.\n\nMust preserve:\n - The other tests in this file (`_run_validator` happy path,\n missing-required-keys negative case) continue to pass.\n - The docstring's contract \u2014 \"if either path mutates the\n manifest after construction, this test fails\" \u2014 is now\n actually true.\n\nVerification:\n - description: \"Pytest: introduce a deliberate manifest mutation\n in one path locally, confirm this test fails. Revert,\n confirm it passes.\"\n - command: \"uv run -- python -m pytest tests/integration/test_hosted_submit_validator_contract.py -v\"\n - expected_output: \"All hosted-submit contract tests pass;\n local mutation experiment confirms the test catches drift\n (then revert).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make hosted-vs-PR submit manifest parity test exercise the actual call sites", "wid": "w20"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": ["w11"], "notes": "Source review comment:\n - PR #151, P2, `benchbox/cli/platform_readiness.py`\n - https://github.com/joeharris76/BenchBox/pull/151#discussion_r3177842040\n\nFinding: this readiness path always reports `environment_skip`\nwhen the endpoint is unreachable, even when `pysail` is\nimportable. For local SQL LakeSail, the adapter can auto-start\na local Sail server (`LakeSailAdapter._ensure_server_ready`),\nso `benchbox platforms check lakesail` is a false negative and\nexits non-zero even though runs would proceed.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/cli/platform_readiness.py:262-282` \u2014 when the\n endpoint is unreachable, `pysail_available` only changes the\n `detail` string; `status` is hardcoded to\n `environment_skip` regardless.\n\nRecommended fix: when the endpoint is unreachable but\n`pysail_available is True` and the platform is the SQL adapter\n(the one that supports auto-start), set `status=\"ready\"` (with\na detail noting auto-start) and surface it as `ready` in the\n`platforms check` summary. Keep DataFrame-mode LakeSail as\n`environment_skip` because it requires an already-running\nendpoint.\n\nPairs with w11: both touch the platform-readiness reporting\nflow; land together so the \"ready\" status emitted here is\nactually exit-0 in `benchbox platforms check` after w11 reorder.\n\nMust preserve:\n - DataFrame-mode LakeSail (no auto-start) still reports\n `environment_skip` when the endpoint is unreachable.\n - `pysail_available is False` keeps the current\n `environment_skip` status with the existing remediation\n string.\n - `endpoint=` field in the result still reports the configured\n endpoint, not a placeholder.\n\nVerification:\n - description: \"Pytest: SQL adapter, endpoint unreachable,\n pysail importable \u2192 status='ready'. DataFrame adapter, same\n conditions \u2192 status='environment_skip'. SQL adapter, pysail\n missing \u2192 status='environment_skip'.\"\n - command: \"uv run -- python -m pytest tests/unit/cli/test_platform_readiness.py -v -k 'lakesail'\"\n - expected_output: \"All three matrix cells pass; the SQL+pysail\n case now returns ready (was: environment_skip).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Mark LakeSail readiness as ready when local auto-start is feasible (`pysail` importable)", "wid": "w21"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #167, P2, `.github/workflows/sync-results-data-to-published.yml`\n - https://github.com/joeharris76/BenchBox/pull/167#discussion_r3178675036\n\nFinding: using `ref: develop` makes the job read whatever\n`develop` points to when the runner starts, not the commit that\ntriggered the run. With `concurrency.cancel-in-progress: false`,\nolder queued runs can execute after newer pushes and then\nmirror newer corpus content while still naming the branch / PR\nwith the older `GITHUB_SHA`, creating misattributed or\nduplicate mirror PRs.\n\nCurrent evidence on `origin/develop`:\n - `.github/workflows/sync-results-data-to-published.yml:34-47` \u2014\n `concurrency.cancel-in-progress: false` and\n `actions/checkout@v4` with `ref: develop`.\n\nRecommended fix: pin the checkout to `${{ github.sha }}` (or\nhard-reset to it before diffing) so each run mirrors the exact\ntriggering push deterministically.\n\nMust preserve:\n - `fetch-depth: 0` stays so subsequent steps can switch\n branches and diff.\n - The \"Configure git identity\" step (lines 49-52) is\n unchanged.\n - `concurrency.cancel-in-progress: false` is intentional and\n stays \u2014 older runs must not be cancelled, just made\n deterministic.\n\nVerification:\n - description: \"Static (local): the diff replaces `ref: develop`\n with `ref: ${{ github.sha }}` and adds no other ref-changing\n logic. Behavioral verification requires CI: after merge,\n replay sequence push A then push B on a test branch in\n repo settings \u2192 workflow runs and confirm each run mirrors\n its triggering tree (no cross-attribution).\"\n - command: \"git diff origin/develop -- .github/workflows/sync-results-data-to-published.yml | grep -E '^\\\\+.*ref:'\"\n - expected_output: \"`+ ref: ${{ github.sha }}` appears; no\n other ref-line additions. Behavioral check is manual on\n the next merge to develop (cannot be reproduced locally\n because the workflow only runs in GitHub Actions).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Pin the sync-results-data-to-published workflow checkout to the triggering SHA", "wid": "w22"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #113, P1 (security), `benchbox/core/results/platform_options.py`\n - https://github.com/joeharris76/BenchBox/pull/113#discussion_r3176769303\n\nFinding (security): `is_secret_option_key()` only `.lower()`s the key,\nthen substring-matches against `_SECRET_KEY_PARTS` entries that all\nuse underscores (e.g. `session_token`, `access_key`,\n`connection_string`). camelCase / PascalCase keys like\n`sessionToken`, `accessKeyId`, `connectionString` never match, so\n`sanitize_platform_options()` writes the raw credential into\n`config.platform_options` of the result artifact.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/results/platform_options.py:10-19` \u2014 secret part\n list uses `_`-separated tokens.\n - `benchbox/core/results/platform_options.py:51-53` \u2014\n `is_secret_option_key` does `key.lower()` only.\n\nRecommended fix: strip non-alphanumeric characters before matching\n(e.g. `re.sub(r\"[^a-z0-9]\", \"\", key.lower())`) and update\n`_SECRET_KEY_PARTS` to the corresponding alphanumeric-only forms\n(`sessiontoken`, `accesskey`, `connectionstring`, \u2026) so substring\nmatching still works after normalization.\n\nMust preserve:\n - Underscore-cased keys (`session_token`, `access_key`,\n `private_key`, `connection_string`) continue to match \u2014 current\n coverage must not regress.\n - Internal-option keys starting with `_` (`_explicit_platform_options`,\n `_defaulted_fields`) are filtered out by `_iter_public_options`,\n not redacted; this fix must not change that filter ordering.\n - Non-secret legitimate keys with embedded uppercase like\n `verboseLevel` or `tuningConfig` MUST NOT trigger redaction.\n\nVerification:\n - description: \"Pytest: every entry in `_SECRET_KEY_PARTS` matches\n its camelCase, PascalCase, kebab-case, and underscore form;\n non-secret keys with mixed case do not match.\"\n - command: \"uv run -- python -m pytest tests/unit/core/results/test_platform_options.py -v\"\n - expected_output: \"All variant-case secret keys redact;\n verboseLevel/tuningConfig fixtures pass through unredacted.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Normalize secret key names before redaction so camelCase keys are caught", "wid": "w3"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #114, P1, `benchbox/core/write_primitives/benchmark.py`\n - https://github.com/joeharris76/BenchBox/pull/114#discussion_r3176765596\n\nFinding: `_check_validation_query()` only inspects `val_result[0][0]`,\nso any additional rows returned by the validation query are ignored.\nSeveral scalar-bound sketch validations query partition tables and\ncan return multiple rows; out-of-range values in non-first rows\nstill pass and produce false-positive benchmark validations.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/write_primitives/benchmark.py:126-136` \u2014 the\n `expected_value_min/max` branch reads only `val_result[0][0]`.\n\nRecommended fix: iterate every row, coerce each `row[0]` to float,\nand fail on the first out-of-range row. (Iterating-all is preferred\nover single-row-strict because partition-aggregation validations\nlegitimately return multiple rows.) Empty result \u2192 fail (matches\ncurrent behavior at line 130-131).\n\nMust preserve:\n - `expected_rows` and `expected_rows_min/max` branches keep their\n current row-count semantics; this change is scoped to the\n `expected_value_min/max` branch.\n - Empty-result validation continues to fail (no silent pass).\n - Non-numeric coercion still fails closed (`TypeError`/`ValueError`\n \u2192 `False`).\n\nVerification:\n - description: \"Pytest: cover single in-range, single out-of-range,\n multi-row all-in, multi-row with bad row at index 0, multi-row\n with bad row at index N>0, empty result.\"\n - command: \"uv run -- python -m pytest tests/unit/core/write_primitives/test_benchmark.py -v -k 'check_validation_query or expected_value'\"\n - expected_output: \"All six test cases pass; bad row at N>0 now\n fails (previous behavior: passed).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Validate every row, not just `[0][0]`, when checking `expected_value_min/max`", "wid": "w4"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #127, P1, `results-explorer/src/db.ts`\n - https://github.com/joeharris76/BenchBox/pull/127#discussion_r3177095782\n\nFinding: `waitForSnapshotRows` treats every entry in\n`SNAPSHOT_READY_SCANS` as required-non-empty, so a valid snapshot\nwith empty optional tables (`bench.query_executions`,\n`bench.query_display_timings`, `bench.short_ids`) fails `getDb()`\nand blocks the entire explorer. Existing detail/query/short-id\npaths already handle missing data gracefully; readiness should\nverify attachment/queryability, not non-emptiness of every\nauxiliary table.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/db.ts:44-77` \u2014 `SNAPSHOT_READY_SCANS`\n lists 8 tables with no required/optional distinction.\n - `results-explorer/src/db.ts:88-107` \u2014 `waitForSnapshotRows`\n counts rows for each scan and rejects if any returns 0.\n\nRecommended fix: tag each scan with `required: true | false`.\nRequired: `results`, `platform_index_rows`, `benchmark_rankings`,\n`benchmark_matrix_cells`, `result_detail_metrics`. Optional:\n`query_executions`, `query_display_timings`, `short_ids`. For\noptional scans, only fail readiness if the *query itself* throws\n(table missing entirely / catalog corruption), not on empty rows.\n\nMust preserve:\n - Empty results in any of the five required tables still fail\n readiness with the current error message \u2014 the regression\n surface is \"missing core data hides as 'transient init failure'.\"\n - Transient DuckDB errors (`offset is out of bounds`,\n `fieldsLength`) still trigger the retry loop.\n - The `online` event listener resets `initFailures` and\n `initError` exactly as today.\n\nVerification:\n - description: \"Vitest: required-table empty \u2192 throws; optional-\n table empty \u2192 resolves; optional-table missing entirely \u2192 throws.\"\n - command: \"cd results-explorer && pnpm test -- db\"\n - expected_output: \"All db.ts tests pass; new optional-empty\n case resolves where it previously rejected.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Relax snapshot readiness so optional empty tables don't block the explorer", "wid": "w5"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comments (both PR #132, same file):\n - P1, Axis 3 EOD bound:\n https://github.com/joeharris76/BenchBox/pull/132#discussion_r3177129340\n - P2, Axis 2 listing guidance:\n https://github.com/joeharris76/BenchBox/pull/132#discussion_r3177129342\n\nFinding 1 (P1, Axis 3): the command uses\n`git log --since=\"$START_DATE\" --until=\"$END_DATE\"`, which makes\nthe upper bound midnight at the *start* of `END_DATE`. Test changes\nmerged later on the closing day are excluded \u2014 exactly the axis\nmeant to catch missed coverage downgrades silently misses them.\n\nFinding 2 (P2, Axis 2): the template says `make blind-spots-list`\nis a more *precise* way to get findings in the review window, but\nthat target maps to `sweep_blind_spots.py list`, which only filters\nby status/kind and does not support date-window filtering. In\npractice it can pull unrelated historical open findings and omit\nin-window non-open findings, so weekly sweep scope becomes\ninconsistent.\n\nCurrent evidence on `origin/develop`:\n - `_project/audits/codex-weekly-sweep-template.md:55-62` \u2014 Axis 2\n presents `make blind-spots-list` as the more-precise option.\n - `_project/audits/codex-weekly-sweep-template.md:88-92` \u2014 Axis 3\n uses bare `--since`/`--until` with no inclusive end-of-day.\n\nRecommended fix:\n - Axis 3: align with Axis 1's inclusive end pattern (e.g.\n `--until=\"$END_DATE 23:59:59\"` or\n `--until=\"$(date -d \"$END_DATE + 1 day\" +%Y-%m-%d)\"` \u2014 match\n whichever idiom Axis 1 already uses).\n - Axis 2: replace the \"more precise\" guidance with an actual\n date-bounded query. Option A: keep the `ls` glob with explicit\n start/end dates. Option B: extend `sweep_blind_spots.py list`\n with `--since`/`--until` and use that. Option A is lower-risk\n and ship-now; Option B can be filed as a follow-up if the\n team wants a CLI affordance.\n\nMust preserve:\n - Axis 1 (the existing reference for the inclusive bound) keeps\n its current command exactly.\n - The cross-link to the cold-load-rowcount blind-spot example at\n line 74-77 stays intact.\n\nVerification:\n - description: \"Run the rewritten Axis 3 command against a known\n commit merged at 23:30 on the closing day; confirm it appears.\"\n - command: \"git log --since=2026-05-02 --until='2026-05-03 23:59:59' -p --diff-filter=M --pretty='format:%H %s' origin/develop -- '*.spec.ts' '*.spec.tsx' '*test*.py' 'tests/' | head -20\"\n - expected_output: \"At least one commit from 2026-05-03 appears\n in the output (previously cut off at midnight 2026-05-03 00:00).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Fix Axis 2 (blind-spot listing) and Axis 3 (EOD bound) drift in the weekly sweep template", "wid": "w6"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #141, P1, `benchbox/core/results/schema.py`\n - https://github.com/joeharris76/BenchBox/pull/141#discussion_r3177580644\n\nFinding: re-exporting previously saved results silently drops\n`cost.total_usd`. `reconstruct_benchmark_results()` calls\n`_extract_cost_summary()` which only rebuilds `total_cost` and\n`cost_model` and does not restore `normalized_cost`, so this\nguard rejects emitting totals for any legacy bundle. Comparison\nand reporting flows that depend on `cost.total_usd` break for\nanything written before normalized_cost existed.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/results/schema.py:578-589` \u2014 `_add_cost_section`\n emits `cost.total_usd` only when\n `_normalized_cost_allows_direct_total(normalized_cost)` is true.\n - `benchbox/core/results/schema.py:592-598` \u2014 that helper returns\n `False` whenever `normalized_cost` is not a dict.\n - `benchbox/core/results/loader.py:394-401` \u2014\n `_extract_cost_summary` returns only\n `{\"total_cost\": ..., \"cost_model\": ...}`. No `normalized_cost`\n key is restored, so re-exports always hit the False branch.\n\nRecommended fix: restore `normalized_cost` during reconstruction so\nround-tripped bundles preserve the contract. Update\n`_extract_cost_summary` to also pull\n`cost_section.get(\"normalized_cost\")` (preserve the dict shape) and\nput it in `cost_summary[\"normalized_cost\"]`. As a defensive backstop,\ntreat \"no normalized block at all\" (legacy) as \"allow direct total\"\nin `_normalized_cost_allows_direct_total` so a missing-vs-rejected\nblock has different semantics.\n\nMust preserve:\n - Bundles that have `normalized_cost` with `cost_status` outside\n `{\"normalized\", \"not_applicable_local\"}` continue to suppress\n `cost.total_usd` (this is the original P1 fix from PR #141).\n - `cost.model` field continues to round-trip from `cost_model`.\n - Empty `cost_summary` (no cost data at all) emits no `cost`\n section \u2014 current behavior must hold.\n\nVerification:\n - description: \"Pytest: legacy bundle (no `normalized_cost`) \u2192\n re-export \u2192 `cost.total_usd` survives. Modern bundle with\n `cost_status='unavailable'` \u2192 re-export \u2192 no `cost.total_usd`.\"\n - command: \"uv run -- python -m pytest tests/unit/core/results/ -v -k 'cost or round_trip or legacy'\"\n - expected_output: \"Both the legacy round-trip and the modern\n suppression cases pass; legacy bundle keeps its total_usd.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Round-trip `cost.total_usd` for legacy bundles that have no normalized_cost block", "wid": "w7"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #144, P1, `benchbox/platforms/dataframe/dask_df.py`\n - https://github.com/joeharris76/BenchBox/pull/144#discussion_r3177676880\n\nFinding: the new skip logic unconditionally excludes Q10 for any\nbenchmark detected as TPC-H, regardless of whether the run is\nusing the constrained local envelope. Users who provide a remote\n`scheduler_address` or provision enough resources still lose Q10\ncoverage, producing incomplete TPC-H results and preventing valid\nfull-suite runs.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/platforms/dataframe/dask_df.py:447-455` \u2014\n `_collect_skip_query_ids` only checks `_is_tpch_benchmark`\n before adding `_RESOURCE_ENVELOPE_SKIP_QUERY_IDS`.\n - `benchbox/platforms/dataframe/dask_df.py:178,239,298` \u2014\n `self.scheduler_address` and `self.use_distributed` flags are\n available; an external scheduler is treated as\n \"not local cluster\".\n\nRecommended fix: gate the Q10 skip on a local-envelope predicate.\nInitial proposal: skip when `not self.scheduler_address and not\nself.use_distributed`. When the user provides an external\nscheduler, run Q10 normally. Update the diagnostic message to\nexplain *why* it skipped and how to opt out (provide a remote\nscheduler).\n\nMust preserve:\n - The OOM exit-137 protection still applies for actual local-\n cluster TPC-H runs (the original incident).\n - `_build_skipped_results` continues to attach\n `dask_resource_envelope_guarded=True` and the diagnostic\n string for the queries it actually skipped.\n - No change to skip lists for non-TPC-H benchmarks.\n\nOpen question (resolve before edit):\n - Is the OOM gate `not scheduler_address` (covers user-supplied\n remote schedulers) or also `cluster_mode != \"local\"` (covers\n managed-Dask/Coiled distributed clusters)? Verify against\n how the original OOM was reproduced; default to the broader\n predicate if both are needed.\n\nVerification:\n - description: \"Pytest: adapter constructed with\n `scheduler_address='tcp://...'` and TPC-H benchmark \u2192 Q10 NOT\n in skip list. Local-only TPC-H \u2192 Q10 IS in skip list.\"\n - command: \"uv run -- python -m pytest tests/unit/platforms/dataframe/test_dask_df.py -v -k 'q10 or skip or envelope'\"\n - expected_output: \"New remote-scheduler test passes; existing\n local-envelope skip test continues to pass.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Restrict Q10 skip guard in dask-df to local-envelope runs", "wid": "w8"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #145, P1, `benchbox/platforms/datafusion.py`\n - https://github.com/joeharris76/BenchBox/pull/145#discussion_r3177722807\n\nFinding: when `file_paths` contain multiple shard names that do\nnot share a leading substring (UUID- or hash-named parts),\n`os.path.commonprefix(...)` returns `\"\"`, so `location` becomes\n`\"/*\"`. DataFusion will then register every file in the\ndirectory, pulling in other tables' data and silently corrupting\nrow counts.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/platforms/datafusion.py:909-921` \u2014 multi-file branch\n builds `location = str(parent_dir / f\"{common_prefix}*\")` with\n no check that `common_prefix != \"\"`.\n\nRecommended fix: when `common_prefix == \"\"`, fall back to one of\n(a) registering each file by exact path and unioning, (b) using\nDataFusion's `LISTING_TABLE` with an explicit file list, or\n(c) the existing single-file fallback applied per shard. Pick the\nleast-disruptive option that preserves row counts (likely (a) \u2014 a\nunion view over explicit paths).\n\nMust preserve:\n - Single-file branch (`len(file_paths) == 1`) keeps its current\n behavior unchanged.\n - Multi-directory fallback (line 916-921) keeps its current\n \"first file with warning\" behavior.\n - Schema-clause inference (lines 899-906) is independent of\n this fix; do not touch.\n\nVerification:\n - description: \"Pytest: multi-file with uuid-style names (no\n common prefix) \u2192 registered location does not glob the\n parent directory. Multi-file with shared prefix \u2192 glob still\n works.\"\n - command: \"uv run -- python -m pytest tests/unit/platforms/test_datafusion.py -v -k 'glob or common_prefix or external_table'\"\n - expected_output: \"New no-common-prefix test passes; existing\n shared-prefix glob test continues to pass.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Guard empty common prefix in DataFusion CSV multi-file glob", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a blanket 'no multi-line commands' rule in the schema if intentional multi-line blocks (|) are used elsewhere", "why": "scope the check to detect unescaped inline newlines only"}, {"INSTEAD": "(not specified)", "dont": "amend the four DONE files' broken verification commands", "why": "they are in DONE and should not be reopened; the template fix prevents recurrence"}], "approach": null, "blocked_reason": null, "category": "Developer Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Three process gaps were deferred from `codex-pr-review-followups-week-2026-05-03`\n(DONE, 2026-05-04). All three relate to the quality of TODO `verification:`\nblocks and the blind-spot remediation lifecycle:\n\n(a) BUGGY VERIFICATION COMMANDS IN DONE FILES \u2014 Codex flagged verification\n commands in four DONE-state TODOs as likely broken: wildcard `--queries`,\n `find -newer` multi-arg arity, DuckDB filename glob, and `$(date)`\n recomputed across midnight (PRs #110, #117, #121, #122). These TODOs are\n in DONE and their commands are no longer run, but the patterns expose a\n class of silent false-positive \"verified\" completions. The fix belongs in\n `_project/audits/codex-weekly-sweep-template.md` \u2014 add negative checks for\n these antipatterns so the next sweep catches them before TODOs land.\n\n(b) MULTI-LINE COMMAND: SCHEMA VALIDATION \u2014 A `verification.command` value\n containing unescaped newlines would silently misparse as multi-line YAML\n (surfaced by PR #161 review of\n `results-explorer-uat-methodology-blind-spot-remediation`). Extend\n `_project/scripts/validate_todo.py` or the JSON schema to reject\n `verification.command` values that contain newlines, so this class can't\n ship in a future TODO.\n\n(c) ACTIONED-COMPLETION CONVENTION \u2014 When a blind-spot remediation TODO has\n acceptance criteria requiring all linked findings to end as\n `merged-to-todo` or `dismissed`, but one ends as `actioned` instead, the\n TODO fails its own completion gate even though the finding was addressed\n (surfaced by PR #163: `results-explorer-uat-methodology-blind-spot-remediation`).\n Decide: widen the criteria to allow `actioned` for convention-only\n remediations, OR amend the historical DONE entry. Document the chosen\n convention so future blind-spot remediation TODOs use it consistently.", "id": "codex-sweep-verification-command-antipatterns", "preserves": ["The codex sweep template remains runnable (do not add checks that require interactive approval)", "validate_todo.py --all still passes all existing TODO/DONE files after the newline validation is added"], "prior_art": [{"concept": "_project/TODO/README.md \u2014 the natural home for the blind-spot remediation convention documentation", "decision": "reuse", "path": "_project/TODO/README.md"}, {"concept": "_project/TODO_SCHEMA.yaml \u2014 consider adding a pattern or format constraint to verification.command", "decision": "reuse", "path": "_project/TODO_SCHEMA.yaml"}, {"concept": "_project/audits/codex-weekly-sweep-template.md \u2014 extend with antipattern checks rather than creating a parallel checklist", "decision": "extend", "path": "_project/audits/codex-weekly-sweep-template.md"}, {"concept": "_project/scripts/validate_todo.py \u2014 extend existing validation rather than writing a separate linter", "decision": "extend", "path": "_project/scripts/validate_todo.py"}], "priority": "medium", "scope": [], "state": "done", "title": "Harden TODO verification commands: audit buggy DONE examples, schema-enforce single-line commands, decide actioned-completion convention", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py /tmp/test-newline-cmd.yaml", "description": "validate_todo.py rejects a synthetic TODO with a newline in verification.command", "expected": "error mentioning verification.command or newline", "item_id": "codex-sweep-verification-command-antipatterns", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py --all", "description": "validate_todo.py --all still passes all existing files", "expected": "0 errors", "item_id": "codex-sweep-verification-command-antipatterns", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -c 'find -newer\\|--queries\\|\\$(date\\|filename.*glob' _project/audits/codex-weekly-sweep-template.md", "description": "codex-weekly-sweep-template.md contains the four antipattern patterns", "expected": "4 or more", "item_id": "codex-sweep-verification-command-antipatterns", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "codex-sweep-verification-command-antipatterns", "needs": [], "notes": "Add negative checks to _project/audits/codex-weekly-sweep-template.md for:\n- Wildcard --queries (should be explicit query names)\n- `find -newer` with more than two positional args (arity bug)\n- DuckDB filename glob in a command string (e.g., *.duckdb patterns)\n- $(date) or $(date +...) inside a verification command (recomputed across midnight)\nThe four DONE files with the known bugs are:\n read-primitives-approximate-aggregate-queries.yaml\n write-primitives-sketch-persistence-category.yaml\n read-primitives-approximate-aggregates-dataframe-coverage.yaml\n results-explorer-uat-multi-scale-corpus-sweep.yaml\nRecord which patterns they embodied so the template is concrete.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add verification-command antipattern checks to codex-weekly-sweep-template.md", "wid": "w1"}, {"evidence": null, "item_id": "codex-sweep-verification-command-antipatterns", "needs": [], "notes": "In _project/scripts/validate_todo.py or _project/TODO_SCHEMA.yaml, add\nvalidation that rejects verification.command strings containing '\\n'.\nA multi-line command in YAML must use a block scalar (|) which already\nsignals intent \u2014 the validation should catch the accidental inline newline\ncase, not intentional multi-line blocks. Test with a synthetic TODO that\nhas a command with an embedded newline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend validate_todo.py (or schema) to reject verification.command values containing newlines", "wid": "w2"}, {"evidence": null, "item_id": "codex-sweep-verification-command-antipatterns", "needs": [], "notes": "Read the existing acceptance criteria pattern in blind-spot remediation\nTODOs. Decide: should 'actioned' be an accepted terminal state alongside\n'merged-to-todo' and 'dismissed' for convention-only findings? Document\nthe decision in _project/TODO/README.md under a \"Blind-spot remediation\nlifecycle\" heading (or equivalent section). Optionally amend the historical\nDONE entry for results-explorer-uat-methodology-blind-spot-remediation to\nrecord the finding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the actioned-completion convention for blind-spot remediation TODOs", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "bundle the skill file edit into the BenchBox PR diff", "instead": "the BenchBox PR ships only the TODO and triage updates.", "why": "the skill file lives outside the repo"}, {"dont": "generalise the proposals/spec checks into a sixth axis", "instead": "keep them in their own subsection.", "why": "the five axes are stable and broadly applicable to code, while these checks apply specifically to proposals/spec docs"}, {"dont": "split into three separate TODOs", "instead": "bundling avoids three PRs reviewing nearly-identical changes.", "why": "each finding produces a one-bullet edit to the same skill file"}], "approach": "Single bundled PR for the planning artifacts (TODO file + three\nblind-spot triage updates). The skill file edit lives in the\nuser-global `~/.claude/` directory; do that edit alongside the\nBenchBox commits but expect it not to land via the PR.\n\nPromotion via `sweep_blind_spots.py triage --action promote\n--todo-id ` only updates frontmatter `status` / `todo_id` and\nappends a `## Triage log` entry \u2014 it does not author the TODO.\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-03T00:00:00Z", "completed_pr": null, "created_at": "2026-05-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "code-review-checklist-additions-for-spec-docs", "id": 121, "reason": "Premature generalisation. The five axes are stable; these checks apply only to proposals/spec docs. Promote to a sixth axis only if the subsection grows past three or four items and starts to show structural patterns of its own.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly premature: promote to a sixth rubric axis only if the proposals/spec subsection grows past 3-4 items; no evidence that threshold was reached.", "summary": "Generalise the proposals/spec checks into a sixth axis of the rubric"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "code-review-checklist-additions-for-spec-docs", "id": 122, "reason": "Convention + reviewer attention is the proposed enforcement model, mirroring the parent UAT methodology spec's approach. Tooling is a lever to pull only if a future review shows the convention drifting.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Tooling enforcement explicitly deferred pending evidence of convention drift; no such evidence cited, no open item tracks it.", "summary": "Tooling enforcement (e.g., a /code linter that flags spec docs missing an approval protocol)"}], "deps": [], "description": "Promotes three meta blind-spot findings produced *during* the\n2026-05-03 review of the UAT methodology remediation work \u2014 findings\nabout how proposals/spec docs were reviewed, not about the source\nUAT \u2014 into a single update to the `/code` review skill.\n\nSource findings (all currently `status: open` in\n`_project/blind-spots/`):\n\n1. `2026-05-03-083859-proposals-doc-decidability-gap` \u2014 proposals\n reviewed via the five-axis rubric can pass on Correctness /\n Readability while still failing the user-facing function: each\n user-facing choice should ship with a default, a consequence, and\n a recommendation, not just a question.\n\n2. `2026-05-03-084354-stress-test-self-bias` \u2014 when the proposer also\n designs the stress-test, \"all proposals survive\" reads as\n validation but is closer to a sanity check. Reviews of stress-test\n handoffs should require a \"strongest argument against each\n proposal\" subsection (or split designer/reviewer roles).\n\n3. `2026-05-03-084923-spec-approval-ergonomics` \u2014 specs with a\n user-approval gate should declare the response shape (yes/no per\n question, defaults, partial-acceptance cascades). Reviewers should\n flag specs that ask questions without an approval protocol.\n\nImplementation surface: a single new \"Reviewing proposals/spec docs\"\nsection appended to `~/.claude/skills/references/five-axis-review.md`\nwith three checklist items. The skill file lives in the user-global\n`~/.claude/` directory and so the actual content edit ships outside\nany BenchBox PR; the BenchBox PR carries only the TODO record and\nthe three blind-spot frontmatter triage updates.", "id": "code-review-checklist-additions-for-spec-docs", "preserves": ["Existing blind-spot bodies \u2014 only `status` and `todo_id` (and the `## Triage log` entry) may change during promotion.", "The Severity Classification, Change Sizing, and Rules sections at the bottom of `five-axis-review.md`.", "The five existing axes (Correctness, Readability, Architecture, Security, Performance) and their bullets \u2014 the new section is additive."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "_project/handoffs/"}, {"kind": "do_not_modify", "path_glob": "_project/specs/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/"}, {"kind": "do_not_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/code-review-checklist-additions-for-spec-docs.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/code-review-checklist-additions-for-spec-docs.yaml"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-03-083859-proposals-doc-decidability-gap.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-03-084354-stress-test-self-bias.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-05-03-084923-spec-approval-ergonomics.md"}], "state": "done", "title": "Add proposal/spec-doc checks to the /code review checklist", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py list --status merged-to-todo | grep -c code-review-checklist-additions-for-spec-docs", "description": "All three blind-spots reach merged-to-todo status with todo_id set", "expected": ">= 3", "item_id": "code-review-checklist-additions-for-spec-docs", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -c 'Reviewing proposals/spec docs' ~/.claude/skills/references/five-axis-review.md", "description": "Skill file gained a 'Reviewing proposals/spec docs' subsection", "expected": ">= 1", "item_id": "code-review-checklist-additions-for-spec-docs", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "code-review-checklist-additions-for-spec-docs", "needs": [], "notes": "Append to `~/.claude/skills/references/five-axis-review.md` (under a\nnew \"Reviewing proposals/spec docs\" subsection) a bullet that says:\n\"for each user-facing choice, the doc should state a default, a\nconsequence, and a recommendation \u2014 conditional phrasing without a\ndefault pushes the decision back onto the user.\"\n\nThen promote the source blind-spot:\n uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py \\\n triage 2026-05-03-083859-proposals-doc-decidability-gap \\\n --action promote --todo-id code-review-checklist-additions-for-spec-docs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a 'decidability check' item for proposals/spec docs", "wid": "w1"}, {"evidence": null, "item_id": "code-review-checklist-additions-for-spec-docs", "needs": ["w1"], "notes": "Add a sibling bullet under the same \"Reviewing proposals/spec docs\"\nsubsection: \"when a proposals doc includes a stress-test that\nconcludes 'all proposals survive', flag the self-grading risk and\nrequire either (a) a 'strongest argument against each proposal'\nsubsection in the same doc, or (b) an independent reviewer for the\nstress-test step.\"\n\nThen promote:\n uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py \\\n triage 2026-05-03-084354-stress-test-self-bias \\\n --action promote --todo-id code-review-checklist-additions-for-spec-docs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a 'self-bias guard' item for stress-test sections", "wid": "w2"}, {"evidence": null, "item_id": "code-review-checklist-additions-for-spec-docs", "needs": ["w1"], "notes": "Add the third bullet: \"specs with a user-approval gate must declare\nthe response shape \u2014 yes/no per question, what 'accept defaults'\nselects, how partial acceptance cascades into downstream\nparameterised work \u2014 before reaching the open-questions section.\"\n\nThen promote:\n uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py \\\n triage 2026-05-03-084923-spec-approval-ergonomics \\\n --action promote --todo-id code-review-checklist-additions-for-spec-docs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an 'approval ergonomics' item for specs with user-approval gates", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fix only the Home.tsx empty-state issue from PR #86 without also restoring strict row-count assertions in `platform-index-cold-load.spec.ts`", "instead": "(not specified)", "why": "the loosened `count > 0` assertion will not detect partial-row cold-load regressions, which is exactly the failure mode the original test was guarding."}, {"dont": "remove the hosted-submit env-token guard globally", "instead": "add a login-specific prompt path instead.", "why": "CI and non-interactive submit flows depend on env-token precedence"}, {"dont": "replace the results explorer retry with a generic empty state", "instead": "keep one retry and handle only persistent mismatch differently.", "why": "transient DuckDB snapshot ordering can recover on retry"}, {"dont": "resolve or dismiss a GitHub review thread solely because it is outdated", "instead": "verify current behavior first.", "why": "later commits may or may not have fixed it"}, {"dont": "switch the whole Makefile to bash to fix one trap", "instead": "use POSIX shell patterns or justify a scoped shell change.", "why": "these recipes currently run under `/bin/sh` on common Linux hosts"}], "approach": "Treat each work unit as a small targeted fix. Before editing, reopen the\ncurrent `origin/develop` file and the linked GitHub review comment, then\nconfirm whether a newer commit already handled the issue. Prefer regression\ntests around each behavior rather than broad rewrites.\n\nFor Makefile work, keep shell syntax POSIX-compatible unless the target\nexplicitly sets `SHELL` and all callers/tests are updated. For DONE-item\nverification text, follow the policy in `description`: edit in place to\nkeep historical commands portable and runnable.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Audit and fix the Codex web-agent review findings that were left on merged\nPRs during the 2026-04-24 through 2026-05-01 commit window.\n\nSource scan:\n - Git range: `origin/develop` commits since `2026-04-24 00:00:00 -0400`.\n - PR-numbered squash merges scanned: 68 merged PRs, #19 through #94 where\n present in the commit subjects.\n - Codex author matched: `chatgpt-codex-connector`.\n - Inline Codex findings found: 26 comments across 17 merged PRs.\n\nNot every unresolved GitHub thread still requires a code change. Several\ncomments were fixed in later PRs but remain unresolved/outdated on GitHub.\nThis TODO keeps the active work units focused on findings that still looked\nunresolved against `origin/develop` at scan time, with enough source context\nto re-check before editing.\n\nPolicy decision (resolves prior `open_questions` ambiguity): historical\nDONE-item verification commands are kept executable. If a Codex finding\nidentifies a portability or correctness defect in a DONE-item verification\nblock, fix it in place. Rationale: those commands serve as runnable\ndocumentation of how the policy was confirmed, so anyone re-verifying the\nhistorical decision (forks, recreated rulesets, future audits) needs them\nto work. This policy applies to w11, w12, and w13 below; future sweeps\ninherit the same rule.", "id": "codex-pr-review-followups-week-2026-05-01", "preserves": ["DONE-item verification commands edited under this TODO remain executable on a clean fork (no hard-coded ruleset IDs, no repo-specific account paths).", "Environment tokens remain the normal precedence for submit/auth resolution outside the explicit `auth login` prompt path.", "Results explorer Home keeps the transient empty-results retry, but persistent snapshot mismatch must not render an endless spinner.", "TODO and DONE YAML edited by this work remains schema-valid and graph-valid.", "The path classifier continues to include deleted files via `ACDMRT` and content-only deletion PRs keep the intended cheap guard path.", "`worktree-claim` remains compatible with `/bin/sh` on Ubuntu/dash; do not reintroduce bash-only `ERR` traps.", "`worktree-release` and `worktree-pool-reset` must not discard dirty pool worktrees without an explicit force path."], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Address unresolved Codex PR review follow-ups from the week ending 2026-05-01", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/DONE/main/active/codex-pr-review-followups-week-2026-05-01.yaml", "description": "This TODO item validates cleanly", "expected": "Validation passes with no schema or DAG errors", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO dependency graph remains valid", "expected": "Graph validation passes", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/scripts/test_blind_spot_tools.py -q", "description": "w1 - blind-spot frontmatter stamping regression test", "expected": "All blind-spot tests pass, including the new block/sequence-shaped status field regression", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 3}, {"command": "tmp_home=$(mktemp -d); trap 'rm -rf \"$tmp_home\"' EXIT; mkdir -p \"$tmp_home/tmp/benchbox-lock-probe\"; touch \"$tmp_home/tmp/benchbox-lock-probe/test.lock\"; HOME=\"$tmp_home\" PATH='/usr/bin:/bin' BENCHBOX_TEST_LOCK_DIR='~/tmp/benchbox-lock-probe' make test-unlock; test ! -e \"$tmp_home/tmp/benchbox-lock-probe/test.lock\"", "description": "w2 - test-unlock honors BENCHBOX_TEST_LOCK_DIR with ~", "expected": "Resolved lock path expands ~ to $HOME and deletes the stale lock without requiring uv on PATH", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/integration/worktree/test_worktree_release_dirty_guard.py -q || echo 'test must be created by w3 implementation'", "description": "w3 - worktree-release refuses dirty trees without FORCE=1", "expected": "New test exercises a pool worktree with staged changes and confirms release exits nonzero without FORCE=1", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/integration/worktree/test_worktree_claim_benchbox_scratch.py -q || echo 'test must be created by w4 implementation'", "description": "w4 - .benchbox scratch files do not block claim", "expected": "New test seeds a free pool slot with .benchbox/foo and confirms claim succeeds", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest tests/integration/worktree/test_worktree_claim_signal_exit.py -q || echo 'test must be created by w5 implementation'", "description": "w5 - INT/TERM trap exits nonzero", "expected": "New test sends INT mid-claim under /bin/sh and confirms recipe exits nonzero with cleanup performed", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 7}, {"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml", "description": "w6 / w7 - Step 5 TODO remains schema-valid after edits", "expected": "Validation passes; last_updated is full ISO date-time; due_date is max(step_3a_merged_at, step_4_merged_at) + 30 days", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 8}, {"command": "uv run -- python -m pytest tests/unit/workflows/test_develop_post_merge_metrics.py -q || echo 'fixture test does not exist yet - w8 must create it'", "description": "w8 - develop-post-merge metrics jq fixture for two failed jobs", "expected": "Two failed jobs at different completed_at produce develop_red_detected_at equal to the earlier timestamp", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 9}, {"command": "cd results-explorer && npm run test -- src/pages/__tests__/Home.test.tsx", "description": "w9 - Home.tsx persistent-mismatch unit test", "expected": "Home tests pass; new test asserts persistent empty-results renders an actionable error/empty state, not a spinner", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 10}, {"command": "grep -E 'toHaveCount\\(' results-explorer/e2e/failures/platform-index-cold-load.spec.ts", "description": "w9 (related) - cold-load e2e restores strict row-count assertions", "expected": "Test contains explicit toHaveCount(N) assertions, not just count > 0", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 11}, {"command": "uv run -- python -m pytest tests/unit/cli/commands/test_auth.py -q", "description": "w10 - benchbox auth login prompt with env token set", "expected": "All auth tests pass; new test sets BENCHBOX_SUBMIT_TOKEN, runs auth login without --token, and confirms keyring stores prompted value", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 12}, {"command": "! grep -nE 'rulesets/[0-9]+' _project/DONE/main/active/dev-loop-step-3a-path-based-ci-skip.yaml", "description": "w11 - Step 3a ruleset verification is repo-instance-agnostic", "expected": "No matches (hard-coded numeric ruleset ID has been replaced by dynamic lookup); grep exit 1 inverted to 0", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 13}, {"command": "grep -nE 'baseRefName' _project/DONE/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml", "description": "w12 - Step 3 verification asserts develop-targeted PR base", "expected": "baseRefName guard present before gh run list ... --event pull_request", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 14}, {"command": "grep -nE 'gh pr list[^|\\\\n]*--state[^|\\\\n]*--base develop' _project/DONE/main/planning/dev-loop-step-0-verify-assumptions.yaml", "description": "w13 - Step 0 secondary probe is scoped", "expected": "Secondary probe includes --state and --base develop filters", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 15}, {"command": "test -f _project/audits/codex-thread-rescan-week-2026-05-01.md && [ \"$(grep -cE '^## ' _project/audits/codex-thread-rescan-week-2026-05-01.md)\" -ge 3 ]", "description": "w14 - rescan audit artifact exists with required sections", "expected": "File exists with at least three ## section headers (Resolved / Already-fixed / Still actionable)", "item_id": "codex-pr-review-followups-week-2026-05-01", "last_result": null, "last_run": null, "seq": 16}], "work": [{"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #59, P2, `_project/scripts/sweep_blind_spots.py`\n - https://github.com/joeharris76/BenchBox/pull/59#discussion_r3164438197\n\nFinding: `replace_frontmatter_field()` replaces only the first line that\nstarts with `status:` or `todo_id:`. If a malformed finding has a block\nor sequence-shaped value, triage can leave orphaned continuation lines\nbehind and make the markdown file unparsable.\n\nCurrent evidence on `origin/develop`: lines 223-228 still use a\nregex-line substitution over raw frontmatter. That preserves formatting\nfor normal scalar fields, but it does not replace the whole YAML node.\n\nRecommended fix: parse the frontmatter into a YAML node tree or use a\nsafer text rewrite that can remove the full field block before inserting\nthe normalized scalar. Add a regression test that starts with\n`status:\\n - open`, stamps `dismissed`, and verifies the result parses\nand contains exactly scalar `status: dismissed`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make blind-spot frontmatter stamping replace complete YAML nodes", "wid": "w1"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #93, P2, `benchbox/cli/commands/auth.py`\n - https://github.com/joeharris76/BenchBox/pull/93#discussion_r3172049089\n\nFinding: `auth.login()` calls `refresh_submission_token()` when\n`--token` is omitted. That helper intentionally raises if\n`BENCHBOX_SUBMIT_TOKEN` or `BENCHBOX_SERVICE_TOKEN` is active, so a user\ncannot use the login prompt to bootstrap/update keyring credentials from\na shell that also has an environment token.\n\nCurrent evidence on `origin/develop`: `auth.login()` calls\n`refresh_submission_token()` for the prompt path; the helper in\n`submit_auth.py` (`refresh_submission_token` and its env-token check)\nstill rejects active env tokens before prompting.\n\nRecommended fix: keep env-token precedence and refresh safety for\nnon-login flows, but give `auth login` an explicit prompt/store path\nthat bypasses the env-token refresh guard. Add CLI coverage that sets\n`BENCHBOX_SUBMIT_TOKEN`, runs `benchbox auth login` without `--token`,\nand confirms the prompt value is saved.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Allow `benchbox auth login` to prompt even when an env token is set", "wid": "w10"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #80, P2, `_project/DONE/main/active/dev-loop-step-3a-path-based-ci-skip.yaml`\n - https://github.com/joeharris76/BenchBox/pull/80#discussion_r3170714532\n\nFinding: a DONE-item verification command calls\n`gh api repos/joeharris76/BenchBox/rulesets/15611785`, which is\nrepository-instance-specific. Forks or recreated rulesets can make this\nhistorical checklist fail even if the policy is configured correctly.\n\nCurrent evidence on `origin/develop`: line 448 still hard-codes\n`rulesets/15611785`.\n\nPer the policy decision in `description` above, edit the DONE file\nin place rather than closing as no-op.\n\nRecommended fix: replace the command with a dynamic ruleset lookup that\nfilters for develop-targeting required status checks before asserting\n`ci-required-result`. Pattern:\n`gh api repos/{owner}/{repo}/rulesets --jq '.[] | select(.target == \"branch\" and (.conditions.ref_name.include // []) | any(. == \"refs/heads/develop\")) | .id'`\nthen pipe the resolved id back into the existing `--jq` extraction.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove hard-coded repository ruleset ID from Step 3a verification", "wid": "w11"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #67, P2, `_project/TODO/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml`\n - https://github.com/joeharris76/BenchBox/pull/67#discussion_r3168098551\n\nFinding: the verification command for routine develop PR job count can\ninspect the latest `pull_request` `test.yml` run for the current branch\nwithout proving the run came from a PR targeting `develop`. A\nmain-targeted PR run could produce a false failure.\n\nCurrent evidence on `origin/develop`: the item has moved to\n`_project/DONE/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml`;\nline 239 uses `gh run list --workflow test.yml --branch \"$branch\"\n--event pull_request --limit 1` before `gh run view`.\n\nPer the policy decision in `description` above, edit the DONE file\nin place rather than closing as no-op.\n\nRecommended fix: resolve the current branch's PR and assert\n`baseRefName == develop` before selecting the run. Pattern:\n`pr_base=$(gh pr view \"$branch\" --json baseRefName --jq .baseRefName); [ \"$pr_base\" = \"develop\" ] || { echo \"PR for $branch targets $pr_base, not develop\"; exit 1; }`\nthen proceed with the existing `gh run list ... --event pull_request`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Step 3 job-count verification select develop-targeted PR runs", "wid": "w12"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment (newly tracked \u2014 was acknowledged in inventory\nbut not assigned to a work unit in the prior revision):\n - PR #67, P2, `_project/DONE/main/planning/dev-loop-step-0-verify-assumptions.yaml`\n\nFinding: the Step 0 secondary-signal probe runs\n`gh pr list --json comments` without `--state`, `--base`, or `--limit`.\nThat scans the full PR history (open + closed + merged across all base\nbranches), which is slow on busy repos, returns a different population\nthan the primary probe (which is develop-merged-recent), and can\nrate-limit the gh client mid-audit.\n\nCurrent evidence on `origin/develop`: line 83 of\n`_project/DONE/main/planning/dev-loop-step-0-verify-assumptions.yaml`\nstill has the unscoped form.\n\nPer the policy decision in `description` above, edit the DONE file\nin place rather than closing as no-op.\n\nRecommended fix: scope the secondary probe to match the primary probe's\npopulation. Pattern:\n`gh pr list --state merged --base develop --limit 50 --json number,comments`\nthen grep the comments payload for the `pr-refresh` / `make pr-refresh`\n/ `merged develop into branch` markers as today.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Scope Step 0 secondary-probe `gh pr list --json comments` query", "wid": "w13"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": ["w1", "w10", "w11", "w12", "w13", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "Re-run the PR thread scan for `chatgpt-codex-connector` on the same\nmerge window and record which comments are now fixed, stale, or still\nactionable. Do not mark a thread resolved only because it is outdated;\nverify the current code or TODO text first.\n\nRequired output artifact (resolves prior ambiguity):\n - Write findings to a new file under `_project/audits/` named\n `codex-thread-rescan-week-2026-05-01.md`.\n - File must contain three sections: (1) Resolved by w1-w13 (one row\n per thread, link to commit/PR that fixed it), (2) Already-fixed by\n earlier merges (one row per stale-but-ok thread, link to the merge\n that fixed it), (3) Still actionable (one row per thread that\n neither this TODO nor any earlier merge addressed \u2014 promote each\n to a new TODO or blind-spot).\n - Cross-check: diff `_project/blind-spots/` for entries dated within\n 2026-04-24 .. 2026-05-01 and either fold them into the audit or\n mark out-of-scope with a one-line rationale. This addresses the\n framework-gap finding in\n `_project/blind-spots/2026-05-01-130000-codex-followups-todo-misses-coverage-downgrades.md`.\n\nFindings that looked already fixed or stale during the 2026-05-01 scan:\n - PR #56 date sorting: current code uses `fmt_date(...)` before\n sorting/listing blind-spot dates.\n - PR #64 gitignore semantics: current workflow uses\n `git check-ignore -z -v --no-index --stdin`, not pathspec matching.\n - PR #64 PR-base reuse: current `pr-open` reuses PRs only with\n `gh pr list --base develop --head \"$CURRENT\"`.\n - PR #66 Step 0 merge-queue signal: current TODO explicitly says not\n to infer merge-queue availability from `.has_issues`.\n - PR #66 Step 1 lock-path command: current verification uses valid\n Python syntax.\n - PR #66 Step 3 unsupported `gh run list --json jobs`: current\n verification lists run IDs and uses `gh run view` for jobs.\n - PR #66 Step 6 machine-specific Makefile path: current TODO no longer\n references `/Users/joe/Developer/BenchBox/Makefile`.\n - PR #70 content-guard aggregator exit: current `.github/workflows/pr.yml`\n exits immediately on content-guard failure before content-only\n success messages.\n - PR #75 and PR #76 non-POSIX `ERR` trap: current claim path uses a\n POSIX `EXIT INT TERM` trap, though w5 tracks the remaining signal\n exit problem.\n - PR #75 and PR #76 PR-list cap: current pool status/sweep use\n `--limit 1000` plus per-branch fallback lookups.\n - PR #79 deleted-file classifier: current diff filter is `ACDMRT`,\n including deletions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-audit stale Codex threads after fixes and record which ones are closed", "wid": "w14"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #71, P2, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/71#discussion_r3169224322\n\nFinding: `tests/conftest.py` resolves `BENCHBOX_TEST_LOCK_DIR` with\n`Path(...).expanduser()`, but `make test-unlock` removes\n`$LOCK_DIR/test.lock` directly. If the environment variable contains a\nliteral `~/tmp`, pytest and the Make target disagree on the lock path.\n\nCurrent evidence on `origin/develop`: `test-unlock` lines 58-61 still\nset `LOCK_DIR=\"$${BENCHBOX_TEST_LOCK_DIR:-$$HOME/.benchbox}\"` and run\n`rm -f \"$$LOCK_DIR/test.lock\"` without expanduser-equivalent handling.\n\nRecommended fix: resolve the lock directory through a tiny `uv run --`\nPython one-liner or an existing helper so the Make target matches\n`tests.conftest._get_test_lock_path()`. Add or update coverage for the\n`BENCHBOX_TEST_LOCK_DIR=~/...` case.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand BENCHBOX_TEST_LOCK_DIR consistently in `make test-unlock`", "wid": "w2"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #73, P1, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/73#discussion_r3169372557\n\nFinding: `worktree-release-locked` detaches and hard-resets a pool\nworktree after checking the PR is merged, but it does not first refuse\nstaged, unstaged, or non-ignored untracked changes. A user can lose local\nedits when releasing a slot.\n\nCurrent evidence on `origin/develop`: lines 1057-1072 perform\n`git checkout --detach origin/develop` and `git reset --hard\norigin/develop` without the dirty guard that `worktree-pool-reset-locked`\nhas at lines 1177-1183.\n\nRecommended fix: before detach/reset, compute dirty state with the same\n`.benchbox` exclusion used by pool status/reset. Refuse by default and\nallow discard only with `FORCE=1`, preserving the existing PR-merged\nsafety check.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refuse `worktree-release` on dirty pool worktrees unless forced", "wid": "w3"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #73, P2, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/73#discussion_r3169372563\n\nFinding: `worktree-claim-attempt` treats any `git status --porcelain`\noutput as dirty, but pool status/reset already ignore expected\nuntracked `.benchbox/` scratch artifacts. A released slot with harmless\n`.benchbox/*` files can become unclaimable.\n\nCurrent evidence on `origin/develop`: line 1032 uses raw\n`git status --porcelain`; line 1033 requires it to be empty. Other pool\ncommands filter `^\\?\\? \\.benchbox(/|$)`.\n\nRecommended fix: reuse the same filtered dirty check in\n`worktree-claim-attempt`, while still treating\n`.benchbox/claim_in_progress` as an aborted claim marker that must not\nbe ignored.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ignore expected `.benchbox` scratch files when claiming free pool slots", "wid": "w4"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #78, P1, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/78#discussion_r3170491244\n\nFinding: the claim cleanup handler is registered for `EXIT INT TERM`,\nbut the handler does not exit when invoked by INT/TERM. In `/bin/sh`\nimplementations such as dash, the trap can swallow cancellation and let\nthe recipe continue or return success.\n\nCurrent evidence on `origin/develop`: lines 1014-1022 define\n`cleanup()` and install `trap cleanup EXIT INT TERM`; the function has no\nsignal-aware exit path.\n\nRecommended fix: split cleanup from signal handling, or have the\nINT/TERM trap run cleanup and then exit with a nonzero status. Keep the\nrecipe POSIX-sh compatible; do not reintroduce a bash-only `ERR` trap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make INT/TERM claim-trap handling exit nonzero after cleanup", "wid": "w5"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #70, P2, `_project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml`\n - https://github.com/joeharris76/BenchBox/pull/70#discussion_r3168890227\n\nFinding: Step 5 says the measurement window runs after Steps 1-4 plus\nStep 3a, but w1 still says \"post-Step-4-merge\" and the notes start the\ntimer from Step 4 alone.\n\nCurrent evidence on `origin/develop`: lines 33-42 still describe a\nStep-4-only timer. In the actual merge order Step 3a landed before\nStep 4, so the current `due_date: 2026-05-30` may still be valid;\nverify before changing dates.\n\nRecommended fix: update w1's summary and notes to define the start\ntimestamp as the later of Step 3a and Step 4 merge times. Confirm\nwhether the existing `2026-05-30` due date remains correct (compute\n`max(step_3a_merged_at, step_4_merged_at) + 30 days`); only adjust\n`due_date` if the math diverges.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct Step 5 measurement-window start description in TODO", "wid": "w6"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #83, P2, `_project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml`\n - https://github.com/joeharris76/BenchBox/pull/83#discussion_r3171491441\n\nFinding: `metadata.last_updated` is `2026-04-30`, but the TODO schema\nexpects a date-time value.\n\nCurrent evidence on `origin/develop`: line 234 has\n`last_updated: \"2026-04-30\"`. The repo convention elsewhere is full\nISO 8601 with `Z` (e.g. this TODO's own `2026-05-01T13:30:00Z`).\n\nRecommended fix: change `last_updated` to a full ISO date-time stamped\nat the time of edit. Independent of w6 \u2014 they happen to touch the same\nfile but have unrelated acceptance criteria.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Convert Step 5 `last_updated` to ISO date-time as schema expects", "wid": "w7"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #81, P2, `.github/workflows/develop-post-merge.yml`\n - https://github.com/joeharris76/BenchBox/pull/81#discussion_r3170954637\n\nFinding: the metrics job computes `develop_red_detected_at` using\n`max` over failed lint/fast-test completion timestamps. If both fail,\nthis records the last failure and understates time to revert/recover.\n\nCurrent evidence on `origin/develop`: lines 326-330 select failed\n`lint`/`fast-test` jobs and use `.completed_at] | max // null`.\n\nRecommended fix: use the earliest failed completion timestamp, e.g.\n`min`, and add a local jq regression fixture with two failed jobs at\ndifferent completion times.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record first failed post-merge job completion as red-detection time", "wid": "w8"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-01", "needs": [], "notes": "Source review comment:\n - PR #86, P1, `results-explorer/src/pages/Home.tsx`\n - https://github.com/joeharris76/BenchBox/pull/86#discussion_r3171854306\n\nFinding: when `listResults()` returns `[]` while meta-leaderboard data\nis present, the Home page retries once. If the retry also returns empty,\n`hasInconsistentEmptySnapshot` remains true and the page renders\n`Loading results...` forever instead of surfacing a recoverable error or\nempty-state message.\n\nCurrent evidence on `origin/develop`: lines 60-76 implement a one-shot\nretry, while lines 159-162 still render the loading spinner whenever\n`hasInconsistentEmptySnapshot` is true.\n\nRecommended fix: after the one retry has completed, render an\nactionable error/empty state for persistent mismatch. Preserve the\ntransient retry path. Add a Home unit test in\n`results-explorer/src/pages/__tests__/Home.test.tsx`.\n\nRelated blind-spot (separate regression on the same PR):\n - `_project/blind-spots/2026-05-01-103002-explorer-cold-load-test-coverage-weakened.md`\n - PR #86 also weakened `platform-index-cold-load.spec.ts` from strict\n row counts (`toHaveCount(4)` / `toHaveCount(2)`) to `count > 0`.\n Whoever picks up w9 should restore strict assertions in the same\n PR or open a follow-up \u2014 the cold-load partial-row regression is a\n different failure mode than the Home empty-state and the loosened\n assertion will not catch it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle persistent results-explorer snapshot mismatch without an infinite spinner", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add new helpers, abstractions, or refactors beyond what the work unit names", "instead": "keep the diff minimal and add the regression test the work unit calls for.", "why": "the prior sweep ran into review churn from incidental refactors"}, {"dont": "batch unrelated work units into one PR", "instead": "open one PR per work unit (or per `needs:`-linked pair).", "why": "each Codex finding lands a focused review thread and a multi-finding PR loses the fix\u2194thread mapping"}, {"dont": "skip writing the regression test 'because the fix is obvious'", "instead": "the test is the primary protection against re-introduction.", "why": "every Codex finding here describes a class of regression that already shipped once"}, {"dont": "touch `_project/DONE/main/**` from any work unit", "instead": "refile any DONE-related finding as a new TODO instead of editing in place.", "why": "completed items are the source-of-truth for prior sweeps and audits"}, {"dont": "widen the changes to also fix the items listed under `description:` as already-addressed (PR #99/#112/#134/#154)", "instead": "(not specified)", "why": "they are intentionally out of scope; the relevant fixes already exist or are tracked elsewhere."}], "approach": "Implementation strategy mirrors the prior sweep TODO\n(`codex-pr-review-followups-week-2026-05-01`):\n\n- **Each work unit is independently shippable.** Don't batch unless\n units share a file. Already merged in this revision: w6 (PR #132,\n one template file) and w10 (PR #147, one validator function). Soft\n pairings expressed via `needs:` so they bundle in the same PR:\n w16 \u2192 w3 (`platform_options.py`), w21 \u2192 w11 (platform readiness flow).\n- **Read the cited file before editing.** The \"Current evidence on\n `origin/develop`\" section names the line range so you can confirm\n the issue still exists at fix time; the codebase is moving fast and\n a finding may already be irrelevant.\n- **Land tests with each fix.** Every work unit has a per-unit\n `Verification:` block with a specific pytest/vitest command \u2014 run\n it locally before pushing. Most regression tests are new fixtures;\n a few are mutations of existing test files.\n- **Security findings (w3 + w16, w10) are P0-tier within the P1\n batch.** Land them first or in parallel \u2014 PR #113's redaction gap\n leaks credentials, PR #147's cwd path enables code execution.\n- **Workflow change (w22) needs CI verification.** It modifies a\n GitHub Actions workflow that runs only on `develop`; review the\n next sync-to-published run after merge to confirm the\n deterministic SHA pin holds.\n- **w8 has an open question** about the precise local-envelope\n predicate (scheduler_address only, or also cluster_mode). Resolve\n against the original OOM repro before editing \u2014 see `open_questions`\n below.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-04T00:00:00Z", "completed_pr": null, "created_at": "2026-05-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 123, "reason": "The TODOs (`read-primitives-approximate-aggregate-queries.yaml`,\n`write-primitives-sketch-persistence-category.yaml`,\n`read-primitives-approximate-aggregates-dataframe-coverage.yaml`,\n`results-explorer-uat-multi-scale-corpus-sweep.yaml`) are now in\n`_project/DONE/` and their verification commands are no longer\nexecuted. The findings (wildcard `--queries`, `find -newer` arity,\nDuckDB filename glob, `$(date)` recomputation across midnight,\nDataFusion default SQL mode) suggest the historical \"verified\"\nchecks may have false-positively reported success during\ncompletion, but no current code change is required. Capture the\ncommon patterns in a future revision of\n`_project/audits/codex-weekly-sweep-template.md` so the next\nsweep flags wildcard `--queries`, `find -newer` multi-arg, and\n`$(date)` reads inside verification blocks before TODOs land.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: The sweep process was redesigned into the automated pr-review-followups routine with its own living template (_project/audits/pr-review-sweep-template.md); original capture note superseded.", "summary": "Audit the four DONE-state TODOs whose verification commands Codex flagged as buggy (PRs #110, #117, #121, #122)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 124, "reason": "`results-explorer-uat-methodology-blind-spot-remediation` is in\nDONE; its acceptance criteria required all linked blind-spots to\nend as `merged-to-todo` or `dismissed`, but\n`2026-05-03-081920-uat-cross-scale-deliverable-not-guarded` ended\nas `actioned`. This is a convention question, not a runtime\ndefect: either widen the criteria to allow `actioned` for\nconvention-only remediations, or amend the historical record.\nTrack the convention decision separately so future TODO\nacceptance criteria align with the broader blind-spot lifecycle.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: blind-spots/README.md lifecycle already lists 'actioned' as a legitimate terminal state ('a fix landed'), so acceptance criteria accommodate it; the historical mismatch is a one-off record question, not a standing tracked decision.", "summary": "Decide convention for marking blind-spot remediation TODOs Completed when linked findings end in `actioned` instead of `merged-to-todo`/`dismissed` (PR #163)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 125, "reason": "`results-explorer-uat-methodology-blind-spot-remediation` is in\nDONE; the flagged YAML-level multi-line `command:` would have\nreported \"all triaged\" even when status was unknown. As a\nforward-looking control, extend\n`_project/scripts/validate_todo.py` (or the schema) to reject\n`verification.command` values that contain unescaped newlines,\nso the same shape can't ship in a future TODO.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: validate_todo.py already rejects verification.command values spanning multiple physical lines unless 'command: |' is used; control exists (line ~152-159).", "summary": "Audit the multi-line shell `command:` parsing risk in TODO `verification:` blocks (PR #161)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "codex-pr-review-followups-week-2026-05-03", "id": 126, "reason": "The fix in w18 needs a live MotherDuck account to confirm runs\nland on the configured non-default database. Tracked as part of\nthe existing cloud verification TODOs; not blocking the local\nfix.\n", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Cloud verification of MotherDuck database propagation (w18)"}], "deps": [], "description": "Audit and fix the Codex web-agent review findings that were left on merged\nPRs during the 2026-04-26 through 2026-05-03 commit window \u2014 the second\nweekly Codex sweep, building on the prior sweep\n(`codex-pr-review-followups-week-2026-05-01`, merged via PRs #99/#104) and\nthe rescan audit at `_project/audits/codex-thread-rescan-week-2026-05-01.md`\nwhich closed out PRs #19\u2013#94.\n\nSource scan:\n - PR range: #95 through #170 merged to `develop` since 2026-04-26.\n - API source: `gh api repos/joeharris76/BenchBox/pulls//comments`.\n - Author filter: `chatgpt-codex-connector[bot]`.\n - Inline Codex findings found: 36 comments across 29 PRs.\n\nNot every unresolved GitHub thread still requires a code change. The\nfollowing groups of comments were filtered out at scan time and are\nintentionally NOT in the work breakdown below:\n\n - **Already addressed by a follow-up fix-PR:**\n - PR #99 (Makefile `test-unlock` `uv run` resolution) \u2014 fixed: the\n current target uses pure shell `$HOME` expansion, no `uv run`.\n - PR #112 (`expected_value_min/max` enforcement in `_check_validation_query`)\n \u2014 fixed in PR #114 (`_check_validation_query` now branches on\n `expected_value_min/max`).\n - PR #154 (`benchbox results --paths` reparses files) \u2014 fixed in\n PR #156, which restructured `results()` to use `if/elif/else` so\n `--paths` no longer also calls `show_results_summary()`.\n\n - **Already tracked in another open TODO:**\n - PR #134 (Redshift theta-merge validation queries) \u2014 the underlying\n per-platform validation-override gap is filed as blind-spot\n `2026-05-02-155448-validation-query-no-per-platform-override`\n (`merged-to-todo` \u2192 `write-primitives-architecture-fixes`). The\n Redshift override is unsafe to ship until that architectural fix\n lands; do not duplicate it here.\n\n - **Comments on TODO YAML files now in DONE \u2014 verification commands\n flagged as buggy but the work is closed:**\n - PR #110 (`approx_*` / `sketch_*` wildcard `--queries` in\n `read-primitives-approximate-aggregate-queries.yaml` and\n `write-primitives-sketch-persistence-category.yaml`).\n - PR #117 (DuckDB filename glob and `find -newer` arity in\n `results-explorer-uat-multi-scale-corpus-sweep.yaml`).\n - PR #121 (DataFusion smoke check uses SQL mode in\n `read-primitives-approximate-aggregates-dataframe-coverage.yaml`).\n - PR #122 (matrix-shaping checkpoint placement and `$(date)`\n recomputation in\n `results-explorer-uat-multi-scale-corpus-sweep.yaml`).\n Each finding may indicate the historical verification step\n false-positively reported success, but the items are closed and\n the verification commands are no longer used. Treat as a\n retrospective protocol-hygiene note for the weekly sweep template,\n not as code work to schedule.\n\n - **Comments on DONE items where the action is process / convention,\n not code:**\n - PR #161 (`results-explorer-uat-methodology-blind-spot-remediation`\n triage `command:` was split across YAML lines, breaking shell\n parsing). The TODO is now in DONE and the verification block is\n not re-run.\n - PR #163 (same TODO marked `Completed` while one linked blind-spot\n ended in `actioned`, not the criteria-required `merged-to-todo`\n or `dismissed`). Either widen the convention to allow `actioned`\n for convention-only remediations or amend the historical record\n \u2014 neither is a runtime defect.\n Both are noted under `deferred` below.\n\nPolicy carry-over from the 2026-05-01 sweep: historical DONE-item\nverification commands stay executable. If a Codex finding identifies a\nportability or correctness defect in a DONE-item verification block, fix\nit in place. The current sweep has no DONE-block code-fix items because\nthe four DONE-state TODO comments above are limited to verification\ncommands that are no longer used \u2014 recording them in `description` is\nsufficient, no work unit is required.\n\nTwo consolidations from the original 36-comment scan:\n\n - **w6 covers both PR #132 findings** (Axis 2 listing guidance + Axis 3\n end-of-day bound) \u2014 same template file, same review thread.\n - **w10 covers both PR #147 findings** (dry-run-when-validator-absent\n P1 + cwd code-execution P1 security) \u2014 same `_load_submission_validator_module`\n function; the security fix and the UX fix are the same refactor.\n\nEach work unit below cites:\n - The source PR(s) and Codex review URL(s).\n - A short restatement of the finding.\n - Current evidence on `origin/develop` (file:line) so the fix can be\n verified before editing.\n - The recommended fix and a runnable verification command, with\n enough specificity to make the change without re-reading the\n original Codex comment.\n - Per-unit `must_preserve:` invariants where the fix sits next to\n load-bearing existing behavior.", "id": "codex-pr-review-followups-week-2026-05-03", "preserves": ["Existing CI gates (`make pr-preflight`, lint, ty, fast tests) must pass on every work-unit PR.", "Existing TODO/blind-spot tooling (`make blind-spots-list`, `make worktree-pool-check`, `_project/scripts/validate_todo.py`) keeps its current invocation shape \u2014 w19 changes worktree-pool-check semantics, not its CLI.", "Public CLI surfaces (`benchbox submit`, `benchbox results`, `benchbox platforms check`, `benchbox run`) keep their exit-code contracts for unrelated paths \u2014 the only changed exit codes are the ones each work unit explicitly names.", "Public results-explorer URL/facet contracts (`bm`, `sf`, `instance_or_warehouse`, etc.) keep their existing serde shape \u2014 only the read/write merge logic changes."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/DONE/main/**"}, {"kind": "only_modify", "path_glob": ".github/workflows/sync-results-data-to-published.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/audits/codex-weekly-sweep-template.md"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/platform.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/platform_readiness.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/cost/calculator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/cost/integration.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/platform_options.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/motherduck.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/motherduck.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ChartPanel.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/RunReceipt.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartMath.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/facetModel.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "tests/**"}, {"kind": "only_modify", "path_glob": "tests/integration/test_hosted_submit_validator_contract.py"}], "state": "done", "title": "Address unresolved Codex PR review follow-ups from the week ending 2026-05-03", "verifications": [{"command": "for pr in 99 102 103 105 107 110 111 112 113 114 115 117 119 121 122 127 129 131 132 134 141 144 145 147 151 154 161 163 167; do gh api repos/joeharris76/BenchBox/pulls/$pr/comments --jq '[.[] | select(.user.login == \"chatgpt-codex-connector[bot]\")] | length' | { read n; [ \"$n\" -gt 0 ] && echo \"PR #$pr: $n comments\"; }; done", "description": "All 22 work units land on develop, individually or via `needs:`-linked pairs, with a corresponding follow-up commit referenced in this TODO's `commits:` field.", "expected": "Same set of PRs returns matching counts; each work unit's PR is referenced from its corresponding commit in `commits:`.", "item_id": "codex-pr-review-followups-week-2026-05-03", "last_result": null, "last_run": null, "seq": 1}, {"command": "make pr-preflight", "description": "Lint and fast-test gate is green after each work unit's fix", "expected": "Preflight reports lint, type, and Python fast-test gates passing.", "item_id": "codex-pr-review-followups-week-2026-05-03", "last_result": null, "last_run": null, "seq": 2}, {"command": "# Re-run the same scan on PRs #95-#170 with the codex-thread-rescan template; expect every comment from this sweep to be marked Resolved or Already-Fixed.", "description": "Next weekly Codex sweep (week ending 2026-05-10) reports zero unaddressed comments from this batch.", "expected": "All 36 originally-scanned comments classify as Resolved (by w1-w22 commits) or Already-Fixed (per the description filtered set).", "item_id": "codex-pr-review-followups-week-2026-05-03", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #103, P1, `results-explorer/src/components/RunReceipt.tsx`\n - https://github.com/joeharris76/BenchBox/pull/103#discussion_r3174480692\n\nFinding: `planDownloadUrl()` returns a `.plans.json` URL\nwhenever `detail.has_plans` is truthy and `bundle_download_url` ends\nin `.json`, but `has_plans` only reflects source-side detection. The\nexplorer pipeline excludes `*.plans.json` from bundle discovery and\nonly copies the primary `result_id.json`, so the generated link 404s\nfor every published bundle.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/components/RunReceipt.tsx:148-152` \u2014\n `planDownloadUrl()` only checks `has_plans` and the suffix; no\n publication-status gate.\n - `benchbox/core/explorer_pipeline/pipeline.py:439` \u2014 bundle scan\n excludes `(\".plans.json\", \".tuning.json\", SUBMISSION_MANIFEST_SUFFIX)`,\n so plan sidecars are never copied to `output_dir/bundles/`.\n\nRecommended fix: gate the link on a publication signal rather than\n`has_plans`. Either (a) extend `DetailResult` (and the pipeline's\ndetail emitter) with a `plans_published: bool` field set when the\npipeline copies a `*.plans.json`, then check it here; or (b) keep\nthe non-link \"Plans available\" fallback (line 140) and only render\na download link when a publication-status field is true.\n\nMust preserve:\n - `has_plans=false` continues to render the \"Plans not published\"\n string (do not collapse to empty).\n - `renderPlansStatus()` still returns the non-link \"Plans\n available\" fallback when a sidecar exists but isn't published \u2014\n tested by `RunReceipt.test.tsx` if present; add coverage if not.\n\nVerification:\n - description: \"Vitest regression: `has_plans=true`,\n `plans_published=false` \u2192 no `` link rendered.\"\n - command: \"cd results-explorer && pnpm test -- RunReceipt\"\n - expected_output: \"All RunReceipt tests pass; new case asserts\n absence of plan download anchor.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Stop linking unpublished `.plans.json` sidecars from the run receipt", "wid": "w1"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comments (both PR #147, same `_load_submission_validator_module`):\n - P1, dry-run unusable when validator absent:\n https://github.com/joeharris76/BenchBox/pull/147#discussion_r3177754515\n - P1 (security), cwd-relative validator code execution:\n https://github.com/joeharris76/BenchBox/pull/147#discussion_r3177754516\n\nFinding 1 (UX): dry-run unconditionally calls\n`_validate_submission_bundle_for_dry_run()`, which exits non-zero\nwhen `scripts/validate_submission.py` cannot be found. The wheel\nonly packages `benchbox*` modules, so non-source installs cannot\nuse `--dry-run` at all.\n\nFinding 2 (security): `_load_submission_validator_module()` falls\nback to `Path.cwd() / \"scripts\" / \"validate_submission.py\"` and\n`spec.loader.exec_module(module)`. In the common case where a\npackaged install has no repo `scripts/` directory, any unrelated\nor malicious `scripts/validate_submission.py` in the user's\ncurrent directory is imported and run when they invoke\n`benchbox submit --dry-run`.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/cli/commands/submit.py:155-173` \u2014 `candidate_paths`\n includes the cwd fallback; `_load_submission_validator_module`\n raises `FileNotFoundError` when neither path resolves.\n - `benchbox/cli/commands/submit.py:176-189` \u2014\n `_validate_submission_bundle_for_dry_run` catches that and\n `ctx.exit(1)`.\n - `benchbox/cli/commands/submit.py:627-628` \u2014 `if dry_run:` is\n unconditional.\n\nRecommended fix (single refactor, both findings):\n 1. Drop the `Path.cwd() / \"scripts\" / \"validate_submission.py\"`\n candidate. Only the source-checkout-relative path\n (`Path(__file__).resolve().parents[3] / \"scripts\" /\n \"validate_submission.py\"`) is acceptable.\n 2. When the source path doesn't resolve, downgrade\n `_validate_submission_bundle_for_dry_run` to a soft warning\n (\"dry-run preview without schema validation; run from a\n source checkout for full validation\") and return cleanly.\n Reserve `ctx.exit(1)` for the case where the validator is\n found but errors during execution.\n\nMust preserve:\n - Source-checkout dry-run continues to validate the bundle\n (the original P1 from PR #147 \u2014 non-validated submissions\n must still be impossible from a checkout).\n - Service-mode dispatch (`_dispatch_service_mode`) still\n receives the post-validation guarantees on the source path.\n - All existing exit codes for fully-validated dry-run failure\n modes (compliance-class refused, non-clean-pass refused) at\n lines 600-619 are untouched.\n\nVerification:\n - description: \"Pytest: source checkout \u2192 dry-run validates and\n exits 0/1 based on bundle. Wheel install (mocked\n `Path(__file__).parents[3]/scripts` missing) \u2192 dry-run\n warns and emits preview, exits 0. Sentinel\n `./scripts/validate_submission.py` raising in cwd \u2192 NOT\n executed.\"\n - command: \"uv run -- python -m pytest tests/unit/cli/commands/test_submit.py tests/integration/test_submit_dry_run.py -v -k 'dry_run or validator or cwd'\"\n - expected_output: \"All three scenarios pass; the cwd-sentinel\n test asserts the marker file was never opened/imported.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make `benchbox submit --dry-run` safe + drop cwd-validator path (security)", "wid": "w10"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #151, P1, `benchbox/cli/platform.py`\n - https://github.com/joeharris76/BenchBox/pull/151#discussion_r3177842037\n\nFinding: `benchbox platforms check` returns a failure for any\n*available but disabled* platform when its readiness probe fails,\nbecause the `info.available and readiness_failed` branch runs\nbefore the disabled-platform branch. This is inconsistent with\nthe prior behavior where disabled platforms are informational\n(\"Available but disabled\") and do not fail the command.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/cli/platform.py:881-891` \u2014 branches in order:\n (1) `info.available and readiness_failed` \u2192 \u26a0\ufe0f, all_good=False;\n (2) `info.enabled and info.available` \u2192 \u2705;\n (3) `info.available` (i.e. available-but-disabled) \u2192 \u25cb;\n (4) else \u2192 \u274c, all_good=False.\n\nRecommended fix: reorder the branches so the\n\"available-but-disabled\" check runs before the readiness-failed\ncheck. Disabled platforms are always informational; readiness\nfailures only fail the command when the platform is also enabled.\n\nMust preserve:\n - \"Missing dependencies\" (branch 4) still fails the command.\n - \"Ready\" (branch 2) stays at \u2705 exit 0.\n - The \u26a0\ufe0f \"Environment not ready\" status still fires for\n *enabled* available platforms whose readiness probe fails \u2014\n do not silence that path.\n - `_print_readiness_details(readiness_results)` continues to\n print after every branch.\n\nVerification:\n - description: \"Pytest: enabled+available+readiness-failed \u2192\n all_good=False, exit 1. Disabled+available+readiness-failed\n \u2192 all_good=True, exit 0 (with informational \u25cb message).\n Disabled+available+readiness-passed \u2192 unchanged (still \u25cb).\"\n - command: \"uv run -- python -m pytest tests/unit/cli/test_platform_check.py -v -k 'check or readiness or disabled'\"\n - expected_output: \"All three matrix cells pass; the\n disabled+failed-readiness case now exits 0 (was: exit 1).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Stop failing `benchbox platforms check` for available-but-disabled platforms", "wid": "w11"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #102, P2, `results-explorer/src/pages/Home.tsx`\n - https://github.com/joeharris76/BenchBox/pull/102#discussion_r3174367121\n\nFinding: the home leaderboard filter `onSelect` overwrites the\nfilter array with a single-element array (`[value]`), regressing\ntrue multi-select to single-select even though URL state still\nmodels `bm`/`sf` as arrays. Users opening\n`?bm=tpch,clickbench`-style links can no longer add or remove\nindividual selections from the UI; any change collapses the\nfilter to one value, so cross-benchmark / cross-scale\ncomparisons are no longer possible from the page controls.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/pages/Home.tsx:330-340` \u2014 both the\n benchmark and scale dropdowns use\n `onSelect={(value) => setFacet(\"\", value === \"all\" ? [] :\n [value])}`.\n\nRecommended fix: switch the dropdown component to a multi-select\n(or wire a second control for adding values) that calls a toggle\nhandler \u2014 `setFacet(\"benchmark\", toggleInList(current, value))` \u2014\ninstead of replacing. Preserve the \"all\" sentinel semantics\n(still clears to `[]`).\n\nMust preserve:\n - Phase, tuning_mode, trust_tier, and date_window dropdowns at\n lines 346-375 keep their existing single-select semantics\n (they are intentionally single-valued).\n - URL `arraySerde` encoding for `bm`/`sf` is unchanged.\n - The \"All benchmarks\" / \"All scales\" sentinel entry continues\n to clear the filter to `[]`.\n\nVerification:\n - description: \"Vitest: navigate to `?bm=tpch,clickbench` then\n toggle off `tpch` \u2192 URL contains only `bm=clickbench`.\"\n - command: \"cd results-explorer && pnpm test -- Home\"\n - expected_output: \"All Home page tests pass; new toggle-off\n case asserts the remaining selection persists.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Preserve multi-select semantics for benchmark/scale cohort filters in Home", "wid": "w12"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #105, P2, `benchbox/core/cost/calculator.py`\n - https://github.com/joeharris76/BenchBox/pull/105#discussion_r3174793239\n\nFinding: the local-platform allowlist omits established local\nDataFrame platform IDs such as `datafusion-df` and `pyspark-df`.\nThose runs are treated as cloud/unknown, producing\n`cost_status=\"unavailable\"` instead of `not_applicable_local`,\nmisclassifying local results and polluting cost facets and\nempty-state logic that depend on `cost_status`.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/cost/calculator.py:230-239` \u2014 local set\n contains `datafusion`, `polars`, `polars-df`, `pandas-df`,\n `cudf-df`, `modin-df`, `dask-df`, `pyspark`, `spark` but not\n `datafusion-df` or `pyspark-df`.\n\nRecommended fix: add `datafusion-df` and `pyspark-df` to the set.\nAudit the registered platform IDs in\n`benchbox/platforms/__init__.py` and add any other DataFrame\nvariants whose primary execution is on the developer's machine.\n\nMust preserve:\n - Cloud platforms (snowflake, bigquery, redshift, databricks,\n motherduck, etc.) continue to return False from\n `is_local_platform()`.\n - The case-insensitive lookup behavior is unchanged.\n\nVerification:\n - description: \"Pytest: every DataFrame-mode platform ID\n registered with `PlatformManager` is local; representative\n cloud platforms remain non-local.\"\n - command: \"uv run -- python -m pytest tests/unit/core/cost/test_calculator.py -v -k 'local_platform'\"\n - expected_output: \"All registered DataFrame platforms classify\n as local; cloud-platform negative cases still pass.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add datafusion-df, pyspark-df (and any peer DataFrame engines) to the local-platform allowlist", "wid": "w13"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #107, P2, `results-explorer/src/lib/chartMath.ts`\n - https://github.com/joeharris76/BenchBox/pull/107#discussion_r3176628095\n\nFinding: `latencyScaleTicks` returns only predefined powers of\nten when at least two match, omitting the actual `scale.max`\n(and sometimes `scale.min`) for common ranges like 0.2\u20132 ms or\n100k\u201350M ms. The right edge of the performance bar chart then\nhas no terminal tick label, so bar lengths are plotted against\nan unlabeled endpoint and the axis can be misread.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/lib/chartMath.ts:92-101` \u2014 the log-mode\n branch filters `LATENCY_LOG_TICKS_MS` to in-range values and\n only falls back to `[scale.min, scale.max]` when fewer than\n two match.\n\nRecommended fix: always include `scale.domainMin` and\n`scale.domainMax` as endpoint ticks, then dedupe against the\nfiltered powers-of-ten list. The full set should be\n`[domainMin, ...powersOfTenInRange, domainMax]` with duplicates\nand out-of-order values removed.\n\nMust preserve:\n - Linear scale branch (lines 93-95) keeps its\n `[0, 0.25, 0.5, 0.75, 1] * domainMax` ticks unchanged.\n - The fallback `[scale.min, scale.max]` path (when no powers\n of ten match) continues to work.\n - Tick values stay sorted ascending (axis assumes ordering).\n\nVerification:\n - description: \"Vitest: ranges 0.2\u20132 ms, 100k\u201350M ms, 1\u20131000 ms\n each include both endpoints in the returned tick list.\"\n - command: \"cd results-explorer && pnpm test -- chartMath\"\n - expected_output: \"All chartMath tests pass; new endpoint-\n inclusion cases assert presence of domainMin and domainMax.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Include log-scale endpoint ticks in the latency axis", "wid": "w14"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #111, P2, `results-explorer/src/lib/facetModel.ts`\n - https://github.com/joeharris76/BenchBox/pull/111#discussion_r3176716507\n\nFinding: the alias migration for `instance_or_warehouse` is lossy\nwhen a URL contains both legacy query params (`instance_type` and\n`warehouse_size`), which the Query page still supports as\nseparate filters. `readFacetParam()` only keeps the first\nmatching alias, and `writeFacetParam()` then deletes all aliases,\nso mounting shared facet state silently drops one of the active\nfilters and changes subsequent query results after navigation.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/lib/facetModel.ts:185` \u2014\n `findFirstParam(params, [FACET_URL_KEYS[key],\n ...(FACET_URL_ALIASES[key] ?? [])])` keeps only the first hit.\n - `results-explorer/src/lib/facetModel.ts:380-382` \u2014\n `writeFacetParam` deletes every alias before writing the\n canonical key.\n\nRecommended fix: when the canonical URL key is absent and\nmultiple aliases are present, merge values across aliases\n(concatenate the arrays then dedupe) into the canonical\n`instance_or_warehouse` list.\n\nMust preserve:\n - When the canonical key IS present, it still wins (no merge\n with stale aliases).\n - Single-alias URLs continue to map cleanly with no extra\n values introduced.\n - `writeFacetParam` continues to delete all aliases when\n writing the canonical key (avoids URL bloat).\n\nVerification:\n - description: \"Vitest: `?instance_type=foo&warehouse_size=bar`\n \u2192 facet contains both; `?instance_or_warehouse=foo&instance_type=bar`\n \u2192 facet is `[foo]` (canonical wins).\"\n - command: \"cd results-explorer && pnpm test -- facetModel\"\n - expected_output: \"All facetModel tests pass; new dual-alias\n and canonical-wins cases assert the right precedence.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Preserve both legacy shape aliases when canonicalizing the URL facet model", "wid": "w15"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": ["w3"], "notes": "Source review comment:\n - PR #113, P2, `benchbox/core/results/platform_options.py`\n - https://github.com/joeharris76/BenchBox/pull/113#discussion_r3176769305\n\nFinding: when a saved config value and a registered default are\nequal, the merge path rewrites the source to `registered_default`\neven though the value already came from `database_options`. This\ncontradicts the function contract that defaults should not\noverride config-derived values and produces incorrect provenance\nin `platform_option_sources`, misleading debugging and audit\nflows that rely on origin tracking.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/results/platform_options.py:88-100` \u2014\n `build_platform_options_capture` overrides `sources[key]` to\n `\"registered_default\"` whenever the requested source is\n `registered_default` *and* the values are equal, even though\n `key` is already in `values` from `database_options`.\n\nRecommended fix: when the requested source is\n`registered_default` and `key` is already populated from\n`database_options`, leave the source as `saved_config` and\n`continue` without modifying it. Only set\n`sources[key] = \"registered_default\"` when the key was not\npreviously populated.\n\nPairs with w3: both touch `platform_options.py`; land in the same\nPR with shared test refresh.\n\nMust preserve:\n - CLI-provided options (any `requested_source` other than\n `registered_default`) continue to override saved config \u2014\n contract documented in `build_platform_options_capture`\n docstring.\n - Sanitized output of `build_platform_options_capture` returns\n only public keys (`_iter_public_options` filter intact).\n\nVerification:\n - description: \"Pytest: database_options={'foo':'bar'} +\n requested registered_default 'bar' \u2192 sources['foo'] ==\n 'saved_config'. CLI-provided 'baz' overrides saved 'bar' \u2192\n sources['foo'] == 'requested'.\"\n - command: \"uv run -- python -m pytest tests/unit/core/results/test_platform_options.py -v -k 'provenance or registered_default or saved_config'\"\n - expected_output: \"All provenance precedence tests pass; the\n new equal-default-vs-saved test asserts saved_config wins.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Preserve `saved_config` provenance when the saved value happens to equal the registered default", "wid": "w16"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #115, P2, `results-explorer/src/pages/Compare.tsx`\n - https://github.com/joeharris76/BenchBox/pull/115#discussion_r3176818720\n\nFinding: when a benchmark/scale mismatch is detected, the new\ndecision summary suppresses winner language, but the page still\nrenders `ChartPanel` in normal compare mode. `SummaryBoxPanel`\n(rendered inside ChartPanel) can still present a computed\n\"Best power / Best geomean\" platform, reintroducing the same\nwinner claim the guardrail is meant to suppress for incomparable\ncohorts.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/pages/Compare.tsx:175-178` \u2014\n `decisionSummary` gets `suppressWinnerClaims`/`suppressionReason`.\n - `results-explorer/src/pages/Compare.tsx:366-370` \u2014\n `` is rendered without suppression props.\n - `results-explorer/src/components/ChartPanel.tsx:34-71` \u2014\n `ChartPanelProps` has no suppression field; line 353 renders\n `SummaryBoxPanel` unconditionally.\n\nRecommended fix: extend `ChartPanelProps` with optional\n`suppressWinnerClaims?: boolean` and `suppressionReason?: string`,\nthread them into `SummaryBoxPanel`, and replace winner-language\n(\"Best power\", \"Best geomean\") with neutral phrasing\n(\"Highest power score in cohort (cohort mismatch \u2014 not\ncomparable)\") when suppression is on.\n\nMust preserve:\n - Compare-page rendering with NO suppression (cohort matches)\n shows the same \"Best power\" / \"Best geomean\" text as today.\n - `decisionSummary` text already produced at the top of Compare\n stays unchanged.\n - ChartPanel callers other than Compare (detail pages) keep\n working unchanged.\n\nVerification:\n - description: \"Vitest: ChartPanel mounted with\n `suppressWinnerClaims={true}` \u2192 `SummaryBoxPanel` text does\n not contain 'Best'. Without suppression \u2192 'Best power' /\n 'Best geomean' still appears.\"\n - command: \"cd results-explorer && pnpm test -- ChartPanel Compare\"\n - expected_output: \"All ChartPanel and Compare tests pass; new\n suppression test asserts neutral wording.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Propagate winner-claim suppression into the chart-level summary on the Compare page", "wid": "w17"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #119, P2, `benchbox/platforms/credentials/motherduck.py`\n - https://github.com/joeharris76/BenchBox/pull/119#discussion_r3176973035\n\nFinding: `setup_motherduck_credentials` persists the user-selected\n`database`, but no one consumes it at run time, so runs still\ndefault to `benchbox` regardless of what the wizard captured.\nMotherDuck has no platform-specific config builder registration,\nso `PlatformHookRegistry._default_builder` is used and does not\nmerge `CredentialManager` data; `MotherDuckAdapter` then falls\nback to its internal default database. The new wizard prompt is\neffectively ineffective and benchmarks may run against the wrong\ndatabase.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/platforms/credentials/motherduck.py:70-74` \u2014 saves\n `{\"database\": database, \"token_env_var\": MOTHERDUCK_TOKEN_ENV}`.\n - `benchbox/platforms/__init__.py` \u2014 has no\n `PlatformHookRegistry.register_config_builder(\"motherduck\",\n ...)` call (the surrounding modules do for databricks,\n bigquery, trino, firebolt, presto, postgresql, etc.).\n\nRecommended fix: add a `_build_motherduck_config` builder in\n`benchbox/platforms/motherduck.py` that pulls the saved\n`database` from `CredentialManager` and merges it into the\nruntime `DatabaseConfig`. Register it from\n`benchbox/platforms/__init__.py` in the same pattern used for\nother adapters.\n\nMust preserve:\n - `MOTHERDUCK_TOKEN` env var precedence stays exactly as today\n (token resolution is unaffected).\n - The existing default `database=\"benchbox\"` is the fallback\n when no saved value exists.\n - Other platforms' config builders (databricks, bigquery,\n trino, firebolt, presto, postgresql, timescaledb,\n pg-duckdb) are untouched.\n\nVerification:\n - description: \"Pytest: CredentialManager with\n `motherduck.database='alt'` \u2192 `_build_motherduck_config`\n returns DatabaseConfig with `database='alt'`. No saved\n value \u2192 falls back to 'benchbox'.\"\n - command: \"uv run -- python -m pytest tests/unit/platforms/test_motherduck.py -v -k 'config_builder or wizard or database'\"\n - expected_output: \"Both saved and fallback paths pass; live\n cloud verification deferred (see deferred[]).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Propagate the configured MotherDuck database into runtime config", "wid": "w18"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #129, P2, `Makefile`\n - https://github.com/joeharris76/BenchBox/pull/129#discussion_r3177117127\n\nFinding: `worktree-pool-check` treats any\n`.benchbox/claim_in_progress` file as an aborted slot, but\n`worktree-claim-attempt` creates that marker at the start of a\nnormal claim and removes it only at the end. Because the check\nruns without the pool lock, invoking it during an active claim\ncan fail with a false positive even when the pool is healthy \u2014\nespecially noisy for the documented periodic / cron use case.\n\nCurrent evidence on `origin/develop`:\n - `Makefile:1219-1222` \u2014 `elif [ -f\n \"$$wt/.benchbox/claim_in_progress\" ]; then` increments\n `aborted_count` with no liveness check.\n - `Makefile:1207` \u2014 `worktree-pool-check` does not acquire the\n pool lock.\n\nRecommended fix: treat a marker as aborted only when it is\n\"stale\" \u2014 older than a configurable threshold (default 10\nminutes) computed from `mtime`. A fresh marker indicates an\nin-progress claim and should be reported as `claimed` (or\nsimply skipped) instead of `aborted`.\n\nMust preserve:\n - Truly aborted markers (older than the threshold) still\n surface as failures with the existing exit-1 behavior and\n recovery hints.\n - The \"missing worktree\" check at line 1216-1217 is unchanged.\n - `extras` detection for slots beyond `POOL_SIZE` still fires.\n - The non-locking design stays \u2014 keep the check fast and\n cron-safe.\n\nVerification:\n - description: \"Manual: run `make worktree-pool-check` while\n `make worktree-claim BRANCH=...` is mid-flight. With the\n fix, no aborted false positive.\"\n - command: \"(make worktree-claim BRANCH=chore/test-pool-race &) && sleep 1 && make worktree-pool-check && wait\"\n - expected_output: \"Pool invariant check OK with no aborted\n counter increment; the concurrent claim completes normally.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Exclude in-flight claims from `worktree-pool-check` aborted-marker false positives", "wid": "w19"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #105, P1, `benchbox/core/cost/integration.py`\n - https://github.com/joeharris76/BenchBox/pull/105#discussion_r3174793237\n\nFinding: `_extract_platform_config_from_info()` has explicit branches\nonly for `snowflake`, `bigquery`, `redshift`, `databricks`. Adapters\nthat emit `platform_type: \"athena\"` (and `\"azure_synapse\"`, future\n`\"clickhouse-cloud\"`, etc.) fall through with no `region`/`cloud`\nfields, so `calculate_normalized_benchmark_cost()` warns\n`pricing region metadata missing` and forces\n`cost_status=\"unavailable\"`, blocking public cost totals in\nsubmission validation even when the adapter populated `platform_info`.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/cost/integration.py:285-368` \u2014 only the four\n listed `platform_type` branches set region/cloud. After the\n last `elif` there is no generic fallback that reads\n `platform_info[\"region\"]` / `platform_info[\"cloud_provider\"]`.\n\nRecommended fix: add a generic post-branch fallback that tries the\nsame `_value_or_default(platform_info, \"cloud_provider\", default,\ndefaulted_fields, alias=\"cloud\")` and `... \"region\", default, ...`\nlookups for any `platform_type` not covered above. Defaults should\nmirror what each cloud adapter already emits (Athena \u2192 `aws`,\nAzure Synapse \u2192 `azure`).\n\nMust preserve:\n - The four explicit branches (snowflake/bigquery/redshift/\n databricks) keep their current defaults and field names exactly\n \u2014 submission validation depends on `cluster_size_dbu_per_hour`,\n `warehouse_size`, `node_type`, etc. being populated by the\n existing branches.\n - `_defaulted_fields` annotation still reflects only fields that\n actually defaulted (no spurious additions from the fallback).\n\nVerification:\n - description: \"Pytest: athena and azure_synapse fixtures populate\n cloud/region without warnings.\"\n - command: \"uv run -- python -m pytest tests/unit/core/cost/ -v -k 'athena or synapse or extract_platform_config'\"\n - expected_output: \"All cost-extraction tests pass; new athena\n and azure_synapse fixtures assert non-default cloud/region.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extract cloud/region metadata for non-DB cloud adapters (Athena, Synapse, \u2026)", "wid": "w2"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #131, P2, `tests/integration/test_hosted_submit_validator_contract.py`\n - https://github.com/joeharris76/BenchBox/pull/131#discussion_r3177127856\n\nFinding: the parity check does not exercise the hosted or PR\nsubmit code paths it claims to protect. Both manifests are built\nby calling `_build_submission_manifest` directly with hard-coded\narguments. If `_dispatch_service_mode` or the PR packaging flow\nlater changes its call-site arguments \u2014 or mutates the manifest\nafter construction \u2014 the test will still pass and miss the\ncross-path drift its docstring describes.\n\nCurrent evidence on `origin/develop`:\n - `tests/integration/test_hosted_submit_validator_contract.py:198-211` \u2014\n two direct `sub._build_submission_manifest(...)` calls with\n identical inputs except `submission_path`.\n\nRecommended fix: drive each manifest through its corresponding\npath. For PR-flow, drive the same code that\n`benchbox submit --output ...` calls (or a path-level helper);\nfor hosted, drive the same code `_dispatch_service_mode` calls.\nAcceptable shortcut: extract a tiny\n`_build_submission_manifest_for_pr_flow()` /\n`_build_submission_manifest_for_hosted_flow()` helper that the\ntest calls without faking the rest of the dispatch.\n\nMust preserve:\n - The other tests in this file (`_run_validator` happy path,\n missing-required-keys negative case) continue to pass.\n - The docstring's contract \u2014 \"if either path mutates the\n manifest after construction, this test fails\" \u2014 is now\n actually true.\n\nVerification:\n - description: \"Pytest: introduce a deliberate manifest mutation\n in one path locally, confirm this test fails. Revert,\n confirm it passes.\"\n - command: \"uv run -- python -m pytest tests/integration/test_hosted_submit_validator_contract.py -v\"\n - expected_output: \"All hosted-submit contract tests pass;\n local mutation experiment confirms the test catches drift\n (then revert).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make hosted-vs-PR submit manifest parity test exercise the actual call sites", "wid": "w20"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": ["w11"], "notes": "Source review comment:\n - PR #151, P2, `benchbox/cli/platform_readiness.py`\n - https://github.com/joeharris76/BenchBox/pull/151#discussion_r3177842040\n\nFinding: this readiness path always reports `environment_skip`\nwhen the endpoint is unreachable, even when `pysail` is\nimportable. For local SQL LakeSail, the adapter can auto-start\na local Sail server (`LakeSailAdapter._ensure_server_ready`),\nso `benchbox platforms check lakesail` is a false negative and\nexits non-zero even though runs would proceed.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/cli/platform_readiness.py:262-282` \u2014 when the\n endpoint is unreachable, `pysail_available` only changes the\n `detail` string; `status` is hardcoded to\n `environment_skip` regardless.\n\nRecommended fix: when the endpoint is unreachable but\n`pysail_available is True` and the platform is the SQL adapter\n(the one that supports auto-start), set `status=\"ready\"` (with\na detail noting auto-start) and surface it as `ready` in the\n`platforms check` summary. Keep DataFrame-mode LakeSail as\n`environment_skip` because it requires an already-running\nendpoint.\n\nPairs with w11: both touch the platform-readiness reporting\nflow; land together so the \"ready\" status emitted here is\nactually exit-0 in `benchbox platforms check` after w11 reorder.\n\nMust preserve:\n - DataFrame-mode LakeSail (no auto-start) still reports\n `environment_skip` when the endpoint is unreachable.\n - `pysail_available is False` keeps the current\n `environment_skip` status with the existing remediation\n string.\n - `endpoint=` field in the result still reports the configured\n endpoint, not a placeholder.\n\nVerification:\n - description: \"Pytest: SQL adapter, endpoint unreachable,\n pysail importable \u2192 status='ready'. DataFrame adapter, same\n conditions \u2192 status='environment_skip'. SQL adapter, pysail\n missing \u2192 status='environment_skip'.\"\n - command: \"uv run -- python -m pytest tests/unit/cli/test_platform_readiness.py -v -k 'lakesail'\"\n - expected_output: \"All three matrix cells pass; the SQL+pysail\n case now returns ready (was: environment_skip).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Mark LakeSail readiness as ready when local auto-start is feasible (`pysail` importable)", "wid": "w21"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #167, P2, `.github/workflows/sync-results-data-to-published.yml`\n - https://github.com/joeharris76/BenchBox/pull/167#discussion_r3178675036\n\nFinding: using `ref: develop` makes the job read whatever\n`develop` points to when the runner starts, not the commit that\ntriggered the run. With `concurrency.cancel-in-progress: false`,\nolder queued runs can execute after newer pushes and then\nmirror newer corpus content while still naming the branch / PR\nwith the older `GITHUB_SHA`, creating misattributed or\nduplicate mirror PRs.\n\nCurrent evidence on `origin/develop`:\n - `.github/workflows/sync-results-data-to-published.yml:34-47` \u2014\n `concurrency.cancel-in-progress: false` and\n `actions/checkout@v4` with `ref: develop`.\n\nRecommended fix: pin the checkout to `${{ github.sha }}` (or\nhard-reset to it before diffing) so each run mirrors the exact\ntriggering push deterministically.\n\nMust preserve:\n - `fetch-depth: 0` stays so subsequent steps can switch\n branches and diff.\n - The \"Configure git identity\" step (lines 49-52) is\n unchanged.\n - `concurrency.cancel-in-progress: false` is intentional and\n stays \u2014 older runs must not be cancelled, just made\n deterministic.\n\nVerification:\n - description: \"Static (local): the diff replaces `ref: develop`\n with `ref: ${{ github.sha }}` and adds no other ref-changing\n logic. Behavioral verification requires CI: after merge,\n replay sequence push A then push B on a test branch in\n repo settings \u2192 workflow runs and confirm each run mirrors\n its triggering tree (no cross-attribution).\"\n - command: \"git diff origin/develop -- .github/workflows/sync-results-data-to-published.yml | grep -E '^\\\\+.*ref:'\"\n - expected_output: \"`+ ref: ${{ github.sha }}` appears; no\n other ref-line additions. Behavioral check is manual on\n the next merge to develop (cannot be reproduced locally\n because the workflow only runs in GitHub Actions).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Pin the sync-results-data-to-published workflow checkout to the triggering SHA", "wid": "w22"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #113, P1 (security), `benchbox/core/results/platform_options.py`\n - https://github.com/joeharris76/BenchBox/pull/113#discussion_r3176769303\n\nFinding (security): `is_secret_option_key()` only `.lower()`s the key,\nthen substring-matches against `_SECRET_KEY_PARTS` entries that all\nuse underscores (e.g. `session_token`, `access_key`,\n`connection_string`). camelCase / PascalCase keys like\n`sessionToken`, `accessKeyId`, `connectionString` never match, so\n`sanitize_platform_options()` writes the raw credential into\n`config.platform_options` of the result artifact.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/results/platform_options.py:10-19` \u2014 secret part\n list uses `_`-separated tokens.\n - `benchbox/core/results/platform_options.py:51-53` \u2014\n `is_secret_option_key` does `key.lower()` only.\n\nRecommended fix: strip non-alphanumeric characters before matching\n(e.g. `re.sub(r\"[^a-z0-9]\", \"\", key.lower())`) and update\n`_SECRET_KEY_PARTS` to the corresponding alphanumeric-only forms\n(`sessiontoken`, `accesskey`, `connectionstring`, \u2026) so substring\nmatching still works after normalization.\n\nMust preserve:\n - Underscore-cased keys (`session_token`, `access_key`,\n `private_key`, `connection_string`) continue to match \u2014 current\n coverage must not regress.\n - Internal-option keys starting with `_` (`_explicit_platform_options`,\n `_defaulted_fields`) are filtered out by `_iter_public_options`,\n not redacted; this fix must not change that filter ordering.\n - Non-secret legitimate keys with embedded uppercase like\n `verboseLevel` or `tuningConfig` MUST NOT trigger redaction.\n\nVerification:\n - description: \"Pytest: every entry in `_SECRET_KEY_PARTS` matches\n its camelCase, PascalCase, kebab-case, and underscore form;\n non-secret keys with mixed case do not match.\"\n - command: \"uv run -- python -m pytest tests/unit/core/results/test_platform_options.py -v\"\n - expected_output: \"All variant-case secret keys redact;\n verboseLevel/tuningConfig fixtures pass through unredacted.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Normalize secret key names before redaction so camelCase keys are caught", "wid": "w3"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #114, P1, `benchbox/core/write_primitives/benchmark.py`\n - https://github.com/joeharris76/BenchBox/pull/114#discussion_r3176765596\n\nFinding: `_check_validation_query()` only inspects `val_result[0][0]`,\nso any additional rows returned by the validation query are ignored.\nSeveral scalar-bound sketch validations query partition tables and\ncan return multiple rows; out-of-range values in non-first rows\nstill pass and produce false-positive benchmark validations.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/write_primitives/benchmark.py:126-136` \u2014 the\n `expected_value_min/max` branch reads only `val_result[0][0]`.\n\nRecommended fix: iterate every row, coerce each `row[0]` to float,\nand fail on the first out-of-range row. (Iterating-all is preferred\nover single-row-strict because partition-aggregation validations\nlegitimately return multiple rows.) Empty result \u2192 fail (matches\ncurrent behavior at line 130-131).\n\nMust preserve:\n - `expected_rows` and `expected_rows_min/max` branches keep their\n current row-count semantics; this change is scoped to the\n `expected_value_min/max` branch.\n - Empty-result validation continues to fail (no silent pass).\n - Non-numeric coercion still fails closed (`TypeError`/`ValueError`\n \u2192 `False`).\n\nVerification:\n - description: \"Pytest: cover single in-range, single out-of-range,\n multi-row all-in, multi-row with bad row at index 0, multi-row\n with bad row at index N>0, empty result.\"\n - command: \"uv run -- python -m pytest tests/unit/core/write_primitives/test_benchmark.py -v -k 'check_validation_query or expected_value'\"\n - expected_output: \"All six test cases pass; bad row at N>0 now\n fails (previous behavior: passed).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Validate every row, not just `[0][0]`, when checking `expected_value_min/max`", "wid": "w4"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #127, P1, `results-explorer/src/db.ts`\n - https://github.com/joeharris76/BenchBox/pull/127#discussion_r3177095782\n\nFinding: `waitForSnapshotRows` treats every entry in\n`SNAPSHOT_READY_SCANS` as required-non-empty, so a valid snapshot\nwith empty optional tables (`bench.query_executions`,\n`bench.query_display_timings`, `bench.short_ids`) fails `getDb()`\nand blocks the entire explorer. Existing detail/query/short-id\npaths already handle missing data gracefully; readiness should\nverify attachment/queryability, not non-emptiness of every\nauxiliary table.\n\nCurrent evidence on `origin/develop`:\n - `results-explorer/src/db.ts:44-77` \u2014 `SNAPSHOT_READY_SCANS`\n lists 8 tables with no required/optional distinction.\n - `results-explorer/src/db.ts:88-107` \u2014 `waitForSnapshotRows`\n counts rows for each scan and rejects if any returns 0.\n\nRecommended fix: tag each scan with `required: true | false`.\nRequired: `results`, `platform_index_rows`, `benchmark_rankings`,\n`benchmark_matrix_cells`, `result_detail_metrics`. Optional:\n`query_executions`, `query_display_timings`, `short_ids`. For\noptional scans, only fail readiness if the *query itself* throws\n(table missing entirely / catalog corruption), not on empty rows.\n\nMust preserve:\n - Empty results in any of the five required tables still fail\n readiness with the current error message \u2014 the regression\n surface is \"missing core data hides as 'transient init failure'.\"\n - Transient DuckDB errors (`offset is out of bounds`,\n `fieldsLength`) still trigger the retry loop.\n - The `online` event listener resets `initFailures` and\n `initError` exactly as today.\n\nVerification:\n - description: \"Vitest: required-table empty \u2192 throws; optional-\n table empty \u2192 resolves; optional-table missing entirely \u2192 throws.\"\n - command: \"cd results-explorer && pnpm test -- db\"\n - expected_output: \"All db.ts tests pass; new optional-empty\n case resolves where it previously rejected.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Relax snapshot readiness so optional empty tables don't block the explorer", "wid": "w5"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comments (both PR #132, same file):\n - P1, Axis 3 EOD bound:\n https://github.com/joeharris76/BenchBox/pull/132#discussion_r3177129340\n - P2, Axis 2 listing guidance:\n https://github.com/joeharris76/BenchBox/pull/132#discussion_r3177129342\n\nFinding 1 (P1, Axis 3): the command uses\n`git log --since=\"$START_DATE\" --until=\"$END_DATE\"`, which makes\nthe upper bound midnight at the *start* of `END_DATE`. Test changes\nmerged later on the closing day are excluded \u2014 exactly the axis\nmeant to catch missed coverage downgrades silently misses them.\n\nFinding 2 (P2, Axis 2): the template says `make blind-spots-list`\nis a more *precise* way to get findings in the review window, but\nthat target maps to `sweep_blind_spots.py list`, which only filters\nby status/kind and does not support date-window filtering. In\npractice it can pull unrelated historical open findings and omit\nin-window non-open findings, so weekly sweep scope becomes\ninconsistent.\n\nCurrent evidence on `origin/develop`:\n - `_project/audits/codex-weekly-sweep-template.md:55-62` \u2014 Axis 2\n presents `make blind-spots-list` as the more-precise option.\n - `_project/audits/codex-weekly-sweep-template.md:88-92` \u2014 Axis 3\n uses bare `--since`/`--until` with no inclusive end-of-day.\n\nRecommended fix:\n - Axis 3: align with Axis 1's inclusive end pattern (e.g.\n `--until=\"$END_DATE 23:59:59\"` or\n `--until=\"$(date -d \"$END_DATE + 1 day\" +%Y-%m-%d)\"` \u2014 match\n whichever idiom Axis 1 already uses).\n - Axis 2: replace the \"more precise\" guidance with an actual\n date-bounded query. Option A: keep the `ls` glob with explicit\n start/end dates. Option B: extend `sweep_blind_spots.py list`\n with `--since`/`--until` and use that. Option A is lower-risk\n and ship-now; Option B can be filed as a follow-up if the\n team wants a CLI affordance.\n\nMust preserve:\n - Axis 1 (the existing reference for the inclusive bound) keeps\n its current command exactly.\n - The cross-link to the cold-load-rowcount blind-spot example at\n line 74-77 stays intact.\n\nVerification:\n - description: \"Run the rewritten Axis 3 command against a known\n commit merged at 23:30 on the closing day; confirm it appears.\"\n - command: \"git log --since=2026-05-02 --until='2026-05-03 23:59:59' -p --diff-filter=M --pretty='format:%H %s' origin/develop -- '*.spec.ts' '*.spec.tsx' '*test*.py' 'tests/' | head -20\"\n - expected_output: \"At least one commit from 2026-05-03 appears\n in the output (previously cut off at midnight 2026-05-03 00:00).\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Fix Axis 2 (blind-spot listing) and Axis 3 (EOD bound) drift in the weekly sweep template", "wid": "w6"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #141, P1, `benchbox/core/results/schema.py`\n - https://github.com/joeharris76/BenchBox/pull/141#discussion_r3177580644\n\nFinding: re-exporting previously saved results silently drops\n`cost.total_usd`. `reconstruct_benchmark_results()` calls\n`_extract_cost_summary()` which only rebuilds `total_cost` and\n`cost_model` and does not restore `normalized_cost`, so this\nguard rejects emitting totals for any legacy bundle. Comparison\nand reporting flows that depend on `cost.total_usd` break for\nanything written before normalized_cost existed.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/core/results/schema.py:578-589` \u2014 `_add_cost_section`\n emits `cost.total_usd` only when\n `_normalized_cost_allows_direct_total(normalized_cost)` is true.\n - `benchbox/core/results/schema.py:592-598` \u2014 that helper returns\n `False` whenever `normalized_cost` is not a dict.\n - `benchbox/core/results/loader.py:394-401` \u2014\n `_extract_cost_summary` returns only\n `{\"total_cost\": ..., \"cost_model\": ...}`. No `normalized_cost`\n key is restored, so re-exports always hit the False branch.\n\nRecommended fix: restore `normalized_cost` during reconstruction so\nround-tripped bundles preserve the contract. Update\n`_extract_cost_summary` to also pull\n`cost_section.get(\"normalized_cost\")` (preserve the dict shape) and\nput it in `cost_summary[\"normalized_cost\"]`. As a defensive backstop,\ntreat \"no normalized block at all\" (legacy) as \"allow direct total\"\nin `_normalized_cost_allows_direct_total` so a missing-vs-rejected\nblock has different semantics.\n\nMust preserve:\n - Bundles that have `normalized_cost` with `cost_status` outside\n `{\"normalized\", \"not_applicable_local\"}` continue to suppress\n `cost.total_usd` (this is the original P1 fix from PR #141).\n - `cost.model` field continues to round-trip from `cost_model`.\n - Empty `cost_summary` (no cost data at all) emits no `cost`\n section \u2014 current behavior must hold.\n\nVerification:\n - description: \"Pytest: legacy bundle (no `normalized_cost`) \u2192\n re-export \u2192 `cost.total_usd` survives. Modern bundle with\n `cost_status='unavailable'` \u2192 re-export \u2192 no `cost.total_usd`.\"\n - command: \"uv run -- python -m pytest tests/unit/core/results/ -v -k 'cost or round_trip or legacy'\"\n - expected_output: \"Both the legacy round-trip and the modern\n suppression cases pass; legacy bundle keeps its total_usd.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Round-trip `cost.total_usd` for legacy bundles that have no normalized_cost block", "wid": "w7"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #144, P1, `benchbox/platforms/dataframe/dask_df.py`\n - https://github.com/joeharris76/BenchBox/pull/144#discussion_r3177676880\n\nFinding: the new skip logic unconditionally excludes Q10 for any\nbenchmark detected as TPC-H, regardless of whether the run is\nusing the constrained local envelope. Users who provide a remote\n`scheduler_address` or provision enough resources still lose Q10\ncoverage, producing incomplete TPC-H results and preventing valid\nfull-suite runs.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/platforms/dataframe/dask_df.py:447-455` \u2014\n `_collect_skip_query_ids` only checks `_is_tpch_benchmark`\n before adding `_RESOURCE_ENVELOPE_SKIP_QUERY_IDS`.\n - `benchbox/platforms/dataframe/dask_df.py:178,239,298` \u2014\n `self.scheduler_address` and `self.use_distributed` flags are\n available; an external scheduler is treated as\n \"not local cluster\".\n\nRecommended fix: gate the Q10 skip on a local-envelope predicate.\nInitial proposal: skip when `not self.scheduler_address and not\nself.use_distributed`. When the user provides an external\nscheduler, run Q10 normally. Update the diagnostic message to\nexplain *why* it skipped and how to opt out (provide a remote\nscheduler).\n\nMust preserve:\n - The OOM exit-137 protection still applies for actual local-\n cluster TPC-H runs (the original incident).\n - `_build_skipped_results` continues to attach\n `dask_resource_envelope_guarded=True` and the diagnostic\n string for the queries it actually skipped.\n - No change to skip lists for non-TPC-H benchmarks.\n\nOpen question (resolve before edit):\n - Is the OOM gate `not scheduler_address` (covers user-supplied\n remote schedulers) or also `cluster_mode != \"local\"` (covers\n managed-Dask/Coiled distributed clusters)? Verify against\n how the original OOM was reproduced; default to the broader\n predicate if both are needed.\n\nVerification:\n - description: \"Pytest: adapter constructed with\n `scheduler_address='tcp://...'` and TPC-H benchmark \u2192 Q10 NOT\n in skip list. Local-only TPC-H \u2192 Q10 IS in skip list.\"\n - command: \"uv run -- python -m pytest tests/unit/platforms/dataframe/test_dask_df.py -v -k 'q10 or skip or envelope'\"\n - expected_output: \"New remote-scheduler test passes; existing\n local-envelope skip test continues to pass.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Restrict Q10 skip guard in dask-df to local-envelope runs", "wid": "w8"}, {"evidence": null, "item_id": "codex-pr-review-followups-week-2026-05-03", "needs": [], "notes": "Source review comment:\n - PR #145, P1, `benchbox/platforms/datafusion.py`\n - https://github.com/joeharris76/BenchBox/pull/145#discussion_r3177722807\n\nFinding: when `file_paths` contain multiple shard names that do\nnot share a leading substring (UUID- or hash-named parts),\n`os.path.commonprefix(...)` returns `\"\"`, so `location` becomes\n`\"/*\"`. DataFusion will then register every file in the\ndirectory, pulling in other tables' data and silently corrupting\nrow counts.\n\nCurrent evidence on `origin/develop`:\n - `benchbox/platforms/datafusion.py:909-921` \u2014 multi-file branch\n builds `location = str(parent_dir / f\"{common_prefix}*\")` with\n no check that `common_prefix != \"\"`.\n\nRecommended fix: when `common_prefix == \"\"`, fall back to one of\n(a) registering each file by exact path and unioning, (b) using\nDataFusion's `LISTING_TABLE` with an explicit file list, or\n(c) the existing single-file fallback applied per shard. Pick the\nleast-disruptive option that preserves row counts (likely (a) \u2014 a\nunion view over explicit paths).\n\nMust preserve:\n - Single-file branch (`len(file_paths) == 1`) keeps its current\n behavior unchanged.\n - Multi-directory fallback (line 916-921) keeps its current\n \"first file with warning\" behavior.\n - Schema-clause inference (lines 899-906) is independent of\n this fix; do not touch.\n\nVerification:\n - description: \"Pytest: multi-file with uuid-style names (no\n common prefix) \u2192 registered location does not glob the\n parent directory. Multi-file with shared prefix \u2192 glob still\n works.\"\n - command: \"uv run -- python -m pytest tests/unit/platforms/test_datafusion.py -v -k 'glob or common_prefix or external_table'\"\n - expected_output: \"New no-common-prefix test passes; existing\n shared-prefix glob test continues to pass.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Guard empty common prefix in DataFusion CSV multi-file glob", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add a blanket 'no multi-line commands' rule in the schema if intentional multi-line blocks (|) are used elsewhere", "instead": "(not specified)", "why": "scope the check to detect unescaped inline newlines only"}, {"dont": "amend the four DONE files' broken verification commands", "instead": "(not specified)", "why": "they are in DONE and should not be reopened; the template fix prevents recurrence"}], "approach": null, "blocked_reason": null, "category": "Developer Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Three process gaps were deferred from `codex-pr-review-followups-week-2026-05-03`\n(DONE, 2026-05-04). All three relate to the quality of TODO `verification:`\nblocks and the blind-spot remediation lifecycle:\n\n(a) BUGGY VERIFICATION COMMANDS IN DONE FILES \u2014 Codex flagged verification\n commands in four DONE-state TODOs as likely broken: wildcard `--queries`,\n `find -newer` multi-arg arity, DuckDB filename glob, and `$(date)`\n recomputed across midnight (PRs #110, #117, #121, #122). These TODOs are\n in DONE and their commands are no longer run, but the patterns expose a\n class of silent false-positive \"verified\" completions. The fix belongs in\n `_project/audits/codex-weekly-sweep-template.md` \u2014 add negative checks for\n these antipatterns so the next sweep catches them before TODOs land.\n\n(b) MULTI-LINE COMMAND: SCHEMA VALIDATION \u2014 A `verification.command` value\n containing unescaped newlines would silently misparse as multi-line YAML\n (surfaced by PR #161 review of\n `results-explorer-uat-methodology-blind-spot-remediation`). Extend\n `_project/scripts/validate_todo.py` or the JSON schema to reject\n `verification.command` values that contain newlines, so this class can't\n ship in a future TODO.\n\n(c) ACTIONED-COMPLETION CONVENTION \u2014 When a blind-spot remediation TODO has\n acceptance criteria requiring all linked findings to end as\n `merged-to-todo` or `dismissed`, but one ends as `actioned` instead, the\n TODO fails its own completion gate even though the finding was addressed\n (surfaced by PR #163: `results-explorer-uat-methodology-blind-spot-remediation`).\n Decide: widen the criteria to allow `actioned` for convention-only\n remediations, OR amend the historical DONE entry. Document the chosen\n convention so future blind-spot remediation TODOs use it consistently.", "id": "codex-sweep-verification-command-antipatterns", "preserves": ["The codex sweep template remains runnable (do not add checks that require interactive approval)", "validate_todo.py --all still passes all existing TODO/DONE files after the newline validation is added"], "prior_art": [{"concept": "_project/TODO/README.md \u2014 the natural home for the blind-spot remediation convention documentation", "decision": "reuse", "path": "_project/TODO/README.md"}, {"concept": "_project/TODO_SCHEMA.yaml \u2014 consider adding a pattern or format constraint to verification.command", "decision": "reuse", "path": "_project/TODO_SCHEMA.yaml"}, {"concept": "_project/audits/codex-weekly-sweep-template.md \u2014 extend with antipattern checks rather than creating a parallel checklist", "decision": "extend", "path": "_project/audits/codex-weekly-sweep-template.md"}, {"concept": "_project/scripts/validate_todo.py \u2014 extend existing validation rather than writing a separate linter", "decision": "extend", "path": "_project/scripts/validate_todo.py"}], "priority": "medium", "scope": [], "state": "done", "title": "Harden TODO verification commands: audit buggy DONE examples, schema-enforce single-line commands, decide actioned-completion convention", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py /tmp/test-newline-cmd.yaml", "description": "validate_todo.py rejects a synthetic TODO with a newline in verification.command", "expected": "error mentioning verification.command or newline", "item_id": "codex-sweep-verification-command-antipatterns", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py --all", "description": "validate_todo.py --all still passes all existing files", "expected": "0 errors", "item_id": "codex-sweep-verification-command-antipatterns", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -c 'find -newer\\|--queries\\|\\$(date\\|filename.*glob' _project/audits/codex-weekly-sweep-template.md", "description": "codex-weekly-sweep-template.md contains the four antipattern patterns", "expected": "4 or more", "item_id": "codex-sweep-verification-command-antipatterns", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "codex-sweep-verification-command-antipatterns", "needs": [], "notes": "Add negative checks to _project/audits/codex-weekly-sweep-template.md for:\n- Wildcard --queries (should be explicit query names)\n- `find -newer` with more than two positional args (arity bug)\n- DuckDB filename glob in a command string (e.g., *.duckdb patterns)\n- $(date) or $(date +...) inside a verification command (recomputed across midnight)\nThe four DONE files with the known bugs are:\n read-primitives-approximate-aggregate-queries.yaml\n write-primitives-sketch-persistence-category.yaml\n read-primitives-approximate-aggregates-dataframe-coverage.yaml\n results-explorer-uat-multi-scale-corpus-sweep.yaml\nRecord which patterns they embodied so the template is concrete.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add verification-command antipattern checks to codex-weekly-sweep-template.md", "wid": "w1"}, {"evidence": null, "item_id": "codex-sweep-verification-command-antipatterns", "needs": [], "notes": "In _project/scripts/validate_todo.py or _project/TODO_SCHEMA.yaml, add\nvalidation that rejects verification.command strings containing '\\n'.\nA multi-line command in YAML must use a block scalar (|) which already\nsignals intent \u2014 the validation should catch the accidental inline newline\ncase, not intentional multi-line blocks. Test with a synthetic TODO that\nhas a command with an embedded newline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend validate_todo.py (or schema) to reject verification.command values containing newlines", "wid": "w2"}, {"evidence": null, "item_id": "codex-sweep-verification-command-antipatterns", "needs": [], "notes": "Read the existing acceptance criteria pattern in blind-spot remediation\nTODOs. Decide: should 'actioned' be an accepted terminal state alongside\n'merged-to-todo' and 'dismissed' for convention-only findings? Document\nthe decision in _project/TODO/README.md under a \"Blind-spot remediation\nlifecycle\" heading (or equivalent section). Optionally amend the historical\nDONE entry for results-explorer-uat-methodology-blind-spot-remediation to\nrecord the finding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the actioned-completion convention for blind-spot remediation TODOs", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "ux-improvements", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-14T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "The `benchbox compare` and `benchbox compare-plans` commands serve related\npurposes (performance comparison vs optimizer analysis) but are completely\nseparate. This fragments the UX and creates confusion about which command\nto use.\n\nThese should be unified so that all comparison functionality is accessible\nfrom `benchbox compare`, with plan comparison as an optional deep-dive.", "id": "compare-command-unification", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Unify compare and compare-plans commands", "verifications": [], "work": [], "worktree": "ux-improvements"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Usability", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Address two UX issues with the `benchbox compare` command:\n\n1. **Superfluous --run flag**: The `--run` flag is awkward and inconsistent with\n `benchbox run` which doesn't require a `--run` flag. Mode should be inferred\n from the arguments provided.\n\n2. **No interactive guidance**: Unlike `benchbox run`, `compare` has no interactive\n wizard when called without arguments. Users must already know the exact syntax.\n\nCurrent state:\n- benchbox compare --run -p duckdb -p sqlite # Awkward --run flag\n- benchbox compare result1.json result2.json # File comparison\n- benchbox compare # Error - no guidance\n\nTarget state:\n- benchbox compare -p duckdb -p sqlite # Run mode (inferred from -p)\n- benchbox compare result1.json result2.json # File mode (inferred from args)\n- benchbox compare # Interactive wizard", "id": "compare-command-ux-improvements", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Compare Command UX Improvements", "verifications": [], "work": [], "worktree": "ux-improvements"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-15T00:00:00Z", "completed_pr": null, "created_at": "2025-12-15T00:00:00Z", "deferrals": [], "deps": [], "description": "The `benchbox compare` command's interactive file comparison mode needs significant\nUX improvements. Currently it shows a flat list of JSON files with just paths,\nrequiring users to manually identify which results they want to compare.\n\nUsers should be able to either:\nA) Specify files directly by path (keep existing behavior)\nB) Interactively filter by benchmark/platform/scale, then select from the 10\n most recent matching results\n\nThis makes the compare workflow much more intuitive for users with many result\nfiles accumulated over time.", "id": "compare-interactive-file-selection", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Improve interactive file selection in compare command", "verifications": [], "work": [], "worktree": "ux-improvements"} @@ -203,20 +203,20 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 **COMPLETED**\"\n- \"**Description**: Implemented comprehensive advanced ETL processing capabilities including FinWire and Customer Management processing\"\n- \"**Context**: Enhanced basic ETL pipeline with sophisticated real-world ETL processing for TPC-DI compliance\"\n- \"**Timeline**: Completed\"\n- \"**Tasks**:\"\n- \"**Files Affected**:\"\n- \"**Implementation Results**:\"\n- \"**Impact**: Provides realistic ETL processing scenarios matching production financial data integration requirements, enabling comprehensive TPC-DI benchmark compliance testing\"", "id": "complete-tpc-di-implementation-phase-3-etl-pipeline-enhancement", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"Complete TPC-DI Implementation - Phase 3: ETL Pipeline Enhancement \u2705\"", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 **COMPLETED** (2025-08-28) - **FULLY VERIFIED AND FUNCTIONAL**\"\n- \"**Description**: Create comprehensive testing, validation, and documentation for publication-ready TPC-DI implementation\"\n- \"**Context**: Need thorough testing and validation to ensure TPC-DI specification compliance and production readiness.\"\n- \"**Timeline**: 1-2 weeks \u2705 **COMPLETED ON SCHEDULE WITH FIXES**\"\n- \"**Completion Summary**: Successfully implemented comprehensive Phase 4 validation and testing framework with 4,007 lines of new code across 6 files, with critical API compatibility issues resolved\"\n- \"**Critical Fixes Applied (2025-08-28)**:\"\n- \"**Tasks**:\"\n- \"**Files Completed**:\"\n- \"**Test Results**: \u2705 **ALL TESTS PASSING**\"\n- \"**Impact**: \u2705 **ACHIEVED** - Publication-ready quality with comprehensive testing coverage, TPC-DI specification compliance validation, complete validation framework with detailed reporting and actionable recommendations\"", "id": "complete-tpc-di-implementation-phase-4-validation-and-testing-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"Complete TPC-DI Implementation - Phase 4: Validation and Testing \u2705 COMPLETED\"", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (Should-Fix Before Release)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Implemented end-to-end version consistency checks, enriched CLI reporting, and documented the release policy so every touchpoint agrees on the published BenchBox version.", "id": "complete-version-management-rollout", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Complete Version Management Rollout", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change the public render() signature", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "create a base class or mixin for shared helpers", "why": "module-level functions are fine"}, {"INSTEAD": "(not specified)", "dont": "extract so many small helpers that reading render() requires jumping between 10 methods", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "optimize the rendering logic while refactoring", "why": "behavior first, optimization later"}], "approach": "Extract helper methods as private (prefixed _) on the same class or as module-level\nfunctions if they don't use self. Keep render() as the public entry point that\norchestrates helpers in order.\n\nReference existing decomposed charts: ascii/line_chart.py is well-structured -\nnote how it organizes helpers as private methods. Follow that pattern.\n\nTest approach: capture render() output before and after and compare with assertEqual.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "7 ASCII chart render() functions accumulate complexity by doing layout calculation,\ndata normalization, header rendering, bar/cell rendering, and legend rendering all in\none method. Each concern is independent and can be a private helper method. This\nreduces per-function complexity from 15-20 to well below the warn threshold without\nchanging behavior.\n\nTarget functions:\n benchbox/core/visualization/ascii/heatmap.py:render (20)\n benchbox/core/visualization/ascii/histogram.py:_render_simple_chunk (20)\n benchbox/core/visualization/ascii/histogram.py:_render_grouped_chunk (20)\n benchbox/core/visualization/ascii/diverging_bar.py:render (17)\n benchbox/core/visualization/ascii/summary_box.py:render (17)\n benchbox/core/visualization/ascii/sparkline_table.py:render (15)\n benchbox/cli/commands/visualize.py:_render_ascii_charts (16)\n\nNote: Several render() functions are already approved as required-complexity ignores\n(line_chart, box_plot, scatter_plot, bar_chart, rank_table, cdf_chart). The functions\nlisted above are NOT in that list - their complexity is reducible.", "id": "complexity-ascii-renderers-w14", "preserves": ["All chart output must be pixel-for-pixel identical - test with known data fixtures", "Color codes and ANSI escape sequences must be identical", "Column width calculations must produce the same values", "render() method signatures unchanged (callers pass same arguments)"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/visualize.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/diverging_bar.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/heatmap.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/histogram.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/sparkline_table.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/summary_box.py"}], "state": "done", "title": "Decompose monolithic ASCII visualization render() functions (complexity batch 3)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-ascii-renderers-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'ascii or visual or chart or render' -q", "description": "Visualization-specific tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-ascii-renderers-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(heatmap|histogram|diverging|summary_box|sparkline|visualize)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 7 target functions", "item_id": "complexity-ascii-renderers-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "benchbox/core/visualization/ascii/heatmap.py:render (line 91)\nExtract to private methods on the class:\n- _compute_column_widths(data, headers) \u2192 list[int]\n- _render_title_header(title, total_width) \u2192 list[str]\n- _render_cell(value, normalized, width, colorize) \u2192 str\n- _render_legend(lines) \u2192 list[str]\nMain render() orchestrates via calls to these helpers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose heatmap render(): extract column-width calc, header, cell rendering, legend", "wid": "w1"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "benchbox/core/visualization/ascii/histogram.py - both functions (lines 236, 369)\nBoth share bar-height calculation and y-axis label logic:\n- Extract _compute_bar_height(value, max_value, chart_height) \u2192 int\n- Extract _render_y_axis_label(row_idx, max_value, chart_height) \u2192 str\n- Extract _render_footer_legend(series_names, colors) \u2192 list[str]\n_render_grouped_chunk also: extract _get_series_lookup(data) \u2192 dict\nAfter extraction, _render_grouped_chunk should call _render_simple_chunk for\nshared rendering logic where possible (DRY).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose histogram _render_simple_chunk and _render_grouped_chunk", "wid": "w2"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "diverging_bar.py:render (line 64):\n- Extract _partition_series(data) \u2192 (improvements, regressions)\n- Extract _compute_layout(improvements, regressions, width) \u2192 LayoutConfig\nsummary_box.py:render (line 88):\n- Extract _build_metric_rows(metrics) \u2192 list[str]\n- Extract _pad_to_equal_length(left, right) \u2192 tuple[list, list]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose diverging_bar and summary_box render() functions", "wid": "w3"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "sparkline_table.py:render (line 63):\n- Extract _normalize_column(col_data) \u2192 list[float]\n- Extract _render_sparkline(values, width) \u2192 str\nvisualize.py:_render_ascii_charts (line 20):\n- Extract _resolve_chart_type(template, chart_type) \u2192 str\n- Replace if/elif chain for chart type validation with dict lookup\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose sparkline_table render() and _render_ascii_charts dispatch", "wid": "w4"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 7 functions below warn threshold", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "add only function-level excludes with rationale.", "dont": "add module-wide or directory-wide complexity excludes", "why": "this hides future regressions"}, {"INSTEAD": "share scaffolding only.", "dont": "collapse platform-specific error handling into generic paths", "why": "adapters have distinct semantics"}, {"INSTEAD": "reduce code complexity or justify narrow ignores.", "dont": "lower global complexity thresholds to make checks pass", "why": "this weakens enforcement"}, {"INSTEAD": "keep functional behavior stable and test-backed.", "dont": "mix behavior changes with complexity-only refactors", "why": "this obscures regressions"}], "approach": "Use an evidence-first workflow:\n1) Build a complete classification matrix for every flagged function.\n2) Apply ignore entries only for truly irreducible complexity with explicit rationale.\n3) Refactor accidental complexity in subsystem batches with focused tests.\n4) Re-run complexity and regression tests after each batch.\n5) Keep threshold policy (`max_complexity = 20`) intact unless a separate policy change is approved.\n\nPreferred refactor strategy is extraction into private helpers with clear contracts and\nreduced branch nesting, not broad architectural rewrites.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "complexity-check-triage-and-remediation", "id": 63, "reason": "Out of scope for this complexity remediation item; requires separate architecture proposal.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'deep redesign' with no scope; explicitly requires a separate architecture proposal that does not exist.", "summary": "Deep redesign of benchmark lifecycle architecture beyond targeted extraction"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "complexity-check-triage-and-remediation", "id": 64, "reason": "Requires credentials/environments and should be handled in dedicated live-test follow-up if unit/integration mocks are insufficient.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague trigger ('if mocks insufficient'); no concrete live-test gap identified, no open item covers general adapter-refactor live validation.", "summary": "Live cloud integration validation for every adapter refactor"}], "deps": [], "description": "`make complexity-check` currently fails with significant complexity debt that mixes\nintentional orchestration logic and accidental branching complexity.\n\nBaseline captured on 2026-02-20:\n- Total functions scanned: 4906\n- Warnings (score >= 15): 106\n- Failures (score > 20): 70\n- Highest offenders: `_extract_common_params` (76), `build_result_payload` (63),\n `render_ascii_chart_from_results` (51), multiple `load_data` methods in cloud\n adapters, and runner/dataframe orchestration methods.\n\nThis TODO delivers a complete decision record for every flagged function (simplify vs\nintentional ignore), implements refactors for accidental complexity, and applies narrow\nignore entries only where complexity is demonstrably irreducible and justified.", "id": "complexity-check-triage-and-remediation", "preserves": ["CLI output contracts for compare/export/visualization commands remain stable.", "Complexity check continues to detect real regressions (no broad suppressions).", "DataFrame benchmark execution parity between pandas-family and expression-family adapters.", "Platform `load_data` semantics, supported file types, and option handling remain backward compatible.", "`benchbox run` lifecycle behavior and phase ordering remain unchanged."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/ or CI/CD configuration"}, {"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "benchbox/**/public API signatures or CLI command interfaces"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml (dependency specifications)"}, {"kind": "do_not_modify", "path_glob": "tests/fixtures/ or test fixture data"}, {"kind": "only_modify", "path_glob": "_project/quality/complexity/"}, {"kind": "only_modify", "path_glob": "benchbox/**/*.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml (complexity config section only)"}, {"kind": "only_modify", "path_glob": "quality/complexity/classification-matrix.yaml"}, {"kind": "only_modify", "path_glob": "scripts/check_complexity.py"}], "state": "done", "title": "Complexity check follow-up: triage intended complexity and remediate accidental complexity", "verifications": [{"command": "uv run -- python scripts/check_complexity.py --no-fail --top 200", "description": "Complexity baseline and classification inventory are complete and consistent", "expected": "Report completes successfully and all functions above warning threshold are represented in the classification matrix.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 1}, {"command": "make complexity-check", "description": "Primary complexity gate passes after remediation and approved ignores", "expected": "PASS with zero functions above max complexity threshold.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q -n 0", "description": "Fast regression suite remains green", "expected": "All fast tests pass with 0 failures.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core tests/unit/platforms tests/unit/cli -q -n 0", "description": "Touched subsystem tests remain green", "expected": "Targeted unit suites pass for modified modules.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": [], "notes": "Persist current output from `scripts/check_complexity.py --no-fail --top 200` under\n`_project/quality/complexity/` and seed the canonical machine-readable inventory at\n`quality/complexity/classification-matrix.yaml` with all functions at score >= 15,\nincluding score, file, function, and module family.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline complexity artifacts and seed function inventory", "wid": "w1"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Normalize shared load flow into scoped helpers/mixins for adapters with parallel logic\n(Redshift/BigQuery/Databricks/Trino/Presto/Athena/Snowflake) without changing platform-specific semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch C: platform adapter load_data families", "wid": "w10"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Decompose large lifecycle/result-shaping functions into deterministic units with clear\ncontracts and easier test coverage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch D: core runner, result schema, and TPC-DS paths", "wid": "w11"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w10", "w11", "w8", "w9"], "notes": "Validated touched areas with focused suites:\n- `tests/unit/platforms/test_{bigquery,presto,trino,athena,snowflake}_adapter.py`\n- `tests/integration/test_tpch_maintenance_test.py`\n- `tests/unit/core/tpcds_obt/test_tpcds_obt_queries.py`\n- `tests/unit/cli/test_quiet_mode.py`\n- `tests/unit/platforms/test_platform_driver_runtime_contract.py`\nCombined targeted run result: 250 passed, 19 deselected.\nFull `-m fast` run currently reports one order-dependent failure in\n`test_dataframe_adapter_contract_rejects_isolated_runtime_without_binding_support`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run targeted tests for each refactor batch and fix regressions", "wid": "w12"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w12", "w7"], "notes": "Ensure intentional complexity is narrowly allowlisted and accidental complexity\nreductions remove all remaining failures above max threshold.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run complexity checks and confirm pass/fail posture with updated ignores", "wid": "w13"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w13"], "notes": "Before (2026-02-20 baseline): 4906 scanned, 106 warnings, 70 failures.\nAfter remediation: 5048 scanned, 109 warnings, 0 failures.\nNet change: failures reduced by 70; complexity gate now passes.\nUpdated complexity artifact: `_project/quality/complexity/latest-report.txt`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document before/after metrics, update TODO status, and regenerate indexes", "wid": "w14"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w1"], "notes": "Create explicit criteria for `simplify_now`, `ignore_with_rationale`, and\n`defer_with_owner`, including examples and disqualifiers for broad ignores.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define triage rubric for intended vs accidental complexity", "wid": "w2"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w2"], "notes": "Review each failing function in `benchbox/core/` (runner, result shaping, lifecycle)\nand record: decision category, concrete rationale, proposed refactor strategy or exact\nignore key (`file:function`), and required regression tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify failing functions in core runner, result schema, and TPC-DS paths (>20)", "wid": "w3"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w2"], "notes": "Review each failing function in `benchbox/platforms/` and dataframe-related modules.\nRecord same evidence as w3. Includes cloud adapter `load_data` families and\ndataframe runner/mixin hotspots.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify failing functions in platform adapters and dataframe orchestration (>20)", "wid": "w4"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w2"], "notes": "Review remaining failing functions in `benchbox/cli/`, chart rendering, comparison\noutput, and any other modules. Record same evidence as w3.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify failing functions in CLI, visualization, and remaining modules (>20)", "wid": "w5"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w3", "w4", "w5"], "notes": "Classify all warning-band functions and assign priority buckets:\nimmediate, near-term, monitor-only. May span multiple sessions if count exceeds ~40;\nbatch by module family if needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify warning-band functions (15-20) and prioritize remediation queue", "wid": "w6"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Update `[tool.benchbox.complexity].exclude_functions` in `pyproject.toml` only for\napproved `ignore_with_rationale` cases. No module-wide excludes. Every entry must map\nto the classification matrix and include a re-review note.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Finalize ignore candidates and add narrow exclusions with rationale linkage", "wid": "w7"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Split high-complexity render/format functions into smaller pure helpers while\npreserving output formatting and user-facing behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch A: visualization and CLI formatting paths", "wid": "w8"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Reduce branching in dataframe benchmark mixins and unified frame hotspots by extracting\nmode-specific helpers and consolidating repeated decision logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch B: dataframe orchestration hotspots", "wid": "w9"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add logging or validation to the new helpers beyond what's in the original", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "change function signatures visible to callers (CLI commands, test fixtures)", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "merge phases that have different error handling", "why": "keep their error paths distinct"}, {"INSTEAD": "(not specified)", "dont": "over-extract: aim for 3-5 helpers per function, not 10", "why": "(unstated)"}], "approach": "Pattern: identify the phases of each function using blank-line or comment boundaries.\nExtract each phase to a private function with a clear name describing its role.\nKeep all extracted functions in the same file unless they're generic utilities.\n\nFor wizard steps: prefer a single _wizard_step() helper over one helper per step,\nsince steps are structurally identical (same shape, different data).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "9 CLI and core orchestration functions mix multiple independent concerns (validation,\nloading, business logic, output) in a single large function. Each function has 3-6\ndistinct phases that can be extracted to named private helpers, making the top-level\nfunction a readable orchestration narrative.\n\nTarget functions:\n benchbox/base.py:create_enhanced_benchmark_result (20)\n benchbox/cli/commands/compare.py:_run_file_comparison (20)\n benchbox/cli/tuning.py:run_dataframe_write_wizard (20)\n benchbox/cli/tuning_resolver.py:resolve_tuning (18)\n benchbox/cli/commands/shell.py:shell (18)\n benchbox/core/cost/integration.py:add_cost_estimation_to_results (18)\n benchbox/core/dryrun.py:_extract_ddl_preview (18)\n benchbox/core/dryrun.py:save_dry_run_results (15)\n benchbox/cli/commands/aggregate.py:aggregate (16)", "id": "complexity-cli-orchestration-w14", "preserves": ["_run_file_comparison: comparison output (all formats) must be identical", "add_cost_estimation_to_results: cost values must be identical (floating point exact)", "create_enhanced_benchmark_result: output BenchmarkResult must be structurally identical", "resolve_tuning: tuning resolution logic must produce same config for all inputs", "run_dataframe_write_wizard: wizard UX flow and resulting config must be unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/aggregate.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/compare.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/shell.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/tuning_resolver.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/cost/integration.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}], "state": "done", "title": "Extract phases from monolithic CLI orchestration functions (complexity batch 4)", "verifications": [{"command": "uv run -- python -m pytest tests/integration/ -q --timeout=30", "description": "Integration tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-cli-orchestration-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-cli-orchestration-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(create_enhanced|run_file_comparison|write_wizard|resolve_tuning|shell.*329|add_cost_estimation|extract_ddl|save_dry_run|aggregate.*40)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 9 target functions", "item_id": "complexity-cli-orchestration-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/base.py:588 - Mix of result field extraction, fallback normalization,\nand dict manipulation. Extract:\n- _normalize_benchmark_id(raw) \u2192 str\n- _extract_query_results(raw_queries) \u2192 list[QueryResult]\n- _build_execution_meta(raw) \u2192 ExecutionMeta\nTop-level becomes: normalize \u2192 extract \u2192 build \u2192 return.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose create_enhanced_benchmark_result into factory + normalizer helpers", "wid": "w1"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/cli/commands/compare.py:1082 - four clear phases:\n- _validate_comparison_inputs(file1, file2, opts) \u2192 or raise\n- _load_comparison_files(file1, file2) \u2192 (result1, result2)\n- _compute_comparison(r1, r2, opts) \u2192 ComparisonResult (delegate to existing)\n- _output_comparison(comparison, opts) \u2192 None\nMain function becomes 4 calls + early returns on validation failure.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _run_file_comparison into validate/load/compare/output helpers", "wid": "w2"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/cli/tuning.py:694 - 4+ wizard steps follow identical pattern:\n capability_check \u2192 user_prompt \u2192 config_update\nExtract _wizard_step(capability, prompt, config_key) \u2192 value or None.\nEach step becomes a single function call. Steps are independent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose run_dataframe_write_wizard into per-step handlers", "wid": "w3"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/cli/tuning_resolver.py:162 - 4 sequential cases (notuning, auto, explicit path, tuned).\nEach case is self-contained. Extract:\n- _resolve_notuning() \u2192 TuningConfig\n- _resolve_auto(platform, benchmark) \u2192 TuningConfig\n- _resolve_explicit_path(path) \u2192 TuningConfig\n- _resolve_tuned(platform, benchmark) \u2192 TuningConfig\nMain function: mode \u2192 handler dispatch (dict or if/elif, each a 1-liner).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose resolve_tuning into per-mode handler functions", "wid": "w4"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "shell (cli/commands/shell.py:329): extract _resolve_platform_from_args,\n_discover_platform_interactive, _launch_shell_for_platform.\nadd_cost_estimation_to_results (core/cost/integration.py:70): extract\n_validate_cost_inputs, _compute_query_costs, _compute_phase_costs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose shell command and add_cost_estimation_to_results", "wid": "w5"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "_extract_ddl_preview (dryrun.py:963): extract _build_table_ddl(table, tunings) \u2192 str.\nsave_dry_run_results (dryrun.py:175): extract _save_queries_to_dir, _save_ddl_file.\naggregate (cli/commands/aggregate.py:40): extract _filter_results, _aggregate_metrics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _extract_ddl_preview, save_dry_run_results, and aggregate", "wid": "w6"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 9 functions below warn threshold", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "only extract internal helpers", "dont": "change public function signatures", "why": "callers depend on them"}, {"INSTEAD": "keep them in the same module", "dont": "move helpers to separate files", "why": "these are small extractions"}], "approach": "Follow the pattern established in complexity batches 3-7: extract private helper\nfunctions within the same module, preserve the public function signature exactly,\nand ensure the extracted helpers are independently testable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Four CLI functions at CC=14 have extractable sub-concerns that can be\ndecomposed into focused helpers without losing clarity.\n\nTarget functions:\n1. prompt_phases (benchmarks.py:891) - extract _parse_phase_input() for dual-mode parser\n2. prompt_platform_options (benchmarks.py:1387) - extract _prompt_single_option() type-dispatch\n3. prompt_cloud_output_location (cloud_storage.py:21) - extract _prompt_and_validate_cloud_path()\n4. check_dependencies (commands/checks.py:20) - extract _show_matrix(), _show_platform(), _show_overview()", "id": "complexity-cli-prompts-w15", "preserves": ["check_dependencies --matrix, --platform, and default output formats unchanged", "prompt_cloud_output_location() validation logic for cloud:// paths unchanged", "prompt_phases() return type and behavior remain identical (list of phase strings)", "prompt_platform_options() correctly prompts for all option types (choice, bool, int, string)"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/benchmarks.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/cloud_storage.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/checks.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}], "state": "done", "title": "Reduce complexity in CLI prompt and command functions (complexity batch 8)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/ -v", "description": "All CLI tests pass", "expected": "all tests pass", "item_id": "complexity-cli-prompts-w15", "last_result": null, "last_run": null, "seq": 1}, {"command": "make complexity-check", "description": "Complexity check passes and these functions drop below 14", "expected": "PASSED; prompt_phases, prompt_platform_options, prompt_cloud_output_location, check_dependencies no longer in top-20", "item_id": "complexity-cli-prompts-w15", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _parse_phase_input() from prompt_phases in benchmarks.py", "wid": "w1"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _prompt_single_option() from prompt_platform_options in benchmarks.py", "wid": "w2"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _prompt_and_validate_cloud_path() from prompt_cloud_output_location in cloud_storage.py", "wid": "w3"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _show_matrix(), _show_platform(), _show_overview() from check_dependencies in checks.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run complexity check and full test suite to verify no regressions", "wid": "w5"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "only restructure the code", "dont": "change the scoring formula in _rank_by_composite", "why": "results must remain comparable with prior runs"}, {"INSTEAD": "simple if/else with extracted helpers is sufficient", "dont": "create an ABC or strategy pattern for the DuckDB/SQLite dispatch", "why": "only 2 platforms exist"}], "approach": "Follow complexity batches 3-7 pattern. For _run_sql_query, the DuckDB and SQLite\nbranches are nearly identical - factor out shared logic (load tables loop, warmup loop,\nbenchmark loop with row count capture) into a common pattern or extract per-platform\nhelpers that share a _time_query_iterations() utility.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Five core functions at CC=14 have clear decomposition opportunities.\n\nTarget functions:\n1. validate (comparison.py:196) - extract _detect_outliers() and _validate_query_overlap()\n2. _rank_by_composite (ranking.py:361) - extract _compute_raw_scores() and _normalize_scores()\n3. _run_sql_query (benchmark_suite.py:1081) - extract _run_on_duckdb() and _run_on_sqlite()\n (most clear-cut: two 90%-identical platform branches)\n4. convert_csv_to_parquet (data_loader.py:255) - extract _build_write_kwargs() and _resolve_arrow_types()\n5. execute_dry_run (dryrun.py:110) - extract _resolve_execution_mode() and _init_platform_adapter()", "id": "complexity-core-analysis-dataframe-w15", "preserves": ["_rank_by_composite() composite scoring formula and weight handling unchanged", "_run_sql_query() returns identical (mean_time, row_count) tuple for both DuckDB and SQLite", "convert_csv_to_parquet() handles both CSV and TBL files with trailing delimiters", "execute_dry_run() DryRunResult construction and error handling paths unchanged", "validate() error/warning accumulation behavior and return type unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/analysis/comparison.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/analysis/ranking.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/benchmark_suite.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}], "state": "done", "title": "Reduce complexity in core analysis and dataframe functions (complexity batch 9)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/ -v", "description": "Core analysis tests pass", "expected": "all tests pass", "item_id": "complexity-core-analysis-dataframe-w15", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/dataframe/ -v -k 'not integration'", "description": "Dataframe tests pass", "expected": "all tests pass", "item_id": "complexity-core-analysis-dataframe-w15", "last_result": null, "last_run": null, "seq": 2}, {"command": "make complexity-check", "description": "Complexity check passes", "expected": "PASSED; these functions no longer in top-20", "item_id": "complexity-core-analysis-dataframe-w15", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": "Most impactful - near-duplicate platform branches are the primary code smell", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _run_on_duckdb() and _run_on_sqlite() from _run_sql_query in benchmark_suite.py", "wid": "w1"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _compute_raw_scores() and _normalize_scores() from _rank_by_composite in ranking.py", "wid": "w2"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _detect_outliers() and _validate_query_overlap() from validate in comparison.py", "wid": "w3"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _build_write_kwargs() and _resolve_arrow_types() from convert_csv_to_parquet in data_loader.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _resolve_execution_mode() and _init_platform_adapter() from execute_dry_run in dryrun.py", "wid": "w5"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run complexity check and full test suite to verify no regressions", "wid": "w6"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change function signatures or return types", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "merge separate dict constants into a single mega-dict", "why": "one dict per function"}, {"INSTEAD": "(not specified)", "dont": "refactor surrounding functions beyond the target chain", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "use lambdas in dict values when the original used inline expressions; keep readable", "why": "(unstated)"}], "approach": "Pattern: identify the if/elif chain, extract the mapping as a module-level dict constant\n(UPPER_SNAKE_CASE), replace the chain with a single dict.get() or dict[] lookup.\nRaise the same exception on KeyError that the original raised on else/fallthrough.\n\nReference: benchbox/core/tpcds_obt/query_conversion.py already uses this pattern in\nother functions (_post_process dispatch) - follow that style.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "8 functions have high cyclomatic complexity caused purely by if/elif chains performing\ndata lookups. These are the simplest category to fix: replace the chain with a module-level\ndict constant and a single lookup. No logic changes, no API changes, just data vs code\nseparation.\n\nTarget functions:\n benchbox/core/tpcds_obt/query_conversion.py:_update_role_map (19)\n benchbox/core/tpcds_obt/query_conversion.py:_table_from_prefix (18)\n benchbox/core/tpcds/maintenance_test.py:_map_to_maintenance_operation_type (15)\n benchbox/cli/exceptions.py:_handle_cli_error (15)\n benchbox/release/content_validation.py:_is_example_line (15)\n benchbox/utils/database_naming.py:_get_optimizations_suffix (15)\n benchbox/core/metadata_primitives/generator.py:_find_objects_with_prefix (16)\n benchbox/cli/composite_params.py:parse (18, partial)\n\nExpected outcome: all 8 functions drop below warn threshold (15).", "id": "complexity-dict-dispatch-w14", "preserves": ["_handle_cli_error: all existing error types must still trigger the same help output", "_map_to_maintenance_operation_type: mapping must be exact (drives TPC-DS maintenance spec)", "_table_from_prefix: column prefix \u2192 table mapping must be 100% accurate (drives OBT SQL)", "_update_role_map: role map output must be identical for all TPC-DS OBT queries", "parse: CLI parameter parsing output must be byte-for-byte identical for all current inputs"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/composite_params.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/exceptions.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/generator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/maintenance_test.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds_obt/query_conversion.py"}, {"kind": "only_modify", "path_glob": "benchbox/release/content_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/database_naming.py"}], "state": "done", "title": "Replace if/elif lookup chains with dict-based dispatch (complexity batch 1)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-dict-dispatch-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(query_conversion|maintenance_test|exceptions|content_validation|database_naming|generator|composite_params)'", "description": "All 8 target functions drop below warn threshold", "expected": "No WARN entries for the 8 target functions", "item_id": "complexity-dict-dispatch-w14", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "Both are in query_conversion.py. _table_from_prefix has ~15-way if/elif \u2192 dict.\n_update_role_map maps table names \u2192 role prefixes \u2192 dict constant TABLE_TO_ROLE_PREFIX.\nThe role inference calls after the lookup can stay; only the lookup changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _update_role_map and _table_from_prefix with dict constants", "wid": "w1"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/core/tpcds/maintenance_test.py - maps (operation, table) tuples to\nMaintenanceOperationType. Simple dict: OPERATION_TABLE_TO_TYPE = {(\"INSERT\", \"t\"): ...}\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _map_to_maintenance_operation_type with dict mapping", "wid": "w2"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/cli/exceptions.py - 6+ isinstance() checks mapping error types to\nhelp methods. Dict: ERROR_HANDLERS = {PlatformNotFoundError: ..., ...}\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _handle_cli_error isinstance chain with dispatch dict", "wid": "w3"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/release/content_validation.py - multiple if-checks for different\nexample patterns. Compile all patterns into EXAMPLE_PATTERNS = [re.compile(...), ...]\nand use any(p.match(line) for p in EXAMPLE_PATTERNS).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _is_example_line multi-pattern checks with compiled regex list", "wid": "w4"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "_get_optimizations_suffix (database_naming.py): dict mapping (platform, opts) \u2192 suffix.\n_find_objects_with_prefix (metadata_primitives/generator.py): SQL template dict keyed\nby (object_type, dialect).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _get_optimizations_suffix and _find_objects_with_prefix with dicts", "wid": "w5"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/cli/composite_params.py - key dispatch for sample/first/queries uses\nif/elif; replace with TYPE_CONVERTERS = {\"sample\": float, \"first\": int, ...}.\nQuery special-case parsing (edge case) stays inline but extract to helper.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify composite_params.py:parse query/key dispatch with dict mapper", "wid": "w6"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 8 functions below warn threshold", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change the output type or structure of any function (callers depend on these)", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "create a FieldExtractor class", "why": "a list + loop is sufficient and more readable"}, {"INSTEAD": "(not specified)", "dont": "merge the field definitions across functions", "why": "each function gets its own field list"}, {"INSTEAD": "(not specified)", "dont": "skip testing get_enabled_settings", "why": "it drives all tuning decisions"}], "approach": "Three patterns to apply:\n1. Enumeration loop: FIELDS = [(\"src_attr\", \"dst_attr\"), ...]; for s, d in FIELDS: setattr(dst, d, getattr(src, s))\n2. Conditional accumulation: CONDITIONS = [(predicate_fn, value), ...]; result = [v for fn, v in CONDITIONS if fn()]\n3. Optional dict builder: KEYS = [(\"src\", \"dst\"), ...]; return {d: src[s] for s, d in KEYS if s in src}\n\nDo not add abstraction layers beyond what's needed - a simple list + comprehension is sufficient.\nKeep field definitions close to the function (module-level constant or near the function).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "10 functions have high complexity caused by 10-20 nearly-identical conditional blocks\nthat check a field, apply a transformation, and assign to a target. These are pure\nmechanical repetition - not real branching logic - and should be collapsed into either\na loop over a field-definition list or a helper that handles the pattern once.\n\nTarget functions and pattern:\n benchbox/core/dataframe/tuning/interface.py:get_enabled_settings (20) - check condition \u2192 add enum\n benchbox/core/runner/runner.py:_enrich_driver_runtime_metadata (20) - extract adapter field \u2192 assign to result\n benchbox/core/results/schema.py:_build_execution_from_context (18) - extract context field \u2192 dict entry\n benchbox/core/results/schema.py:_build_config_block (17) - extract with fallback \u2192 dict entry\n benchbox/core/schemas.py:populate_driver_metadata (17) - options.get() \u2192 assign if not set\n benchbox/core/schemas.py:to_cli_args (16) - non-default field \u2192 append arg string\n benchbox/core/manifest/io.py:_manifest_v2_to_dict (18) - optional field present \u2192 add to dict\n benchbox/core/tuning/interface.py:get_enabled_tuning_types (16) - check config attr \u2192 add to set\n benchbox/core/dryrun.py:_extract_df_tuning (16) - hasattr + is_default \u2192 build dict\n benchbox/cli/dryrun.py:generate_cli_command (16) - non-default flag \u2192 append to parts list\n\nEach requires a field-definition approach: declare a list/dict of (source_field,\ncondition, transform, target_field) tuples, iterate once.", "id": "complexity-field-extraction-w14", "preserves": ["_build_execution_from_context: result dict must have identical keys/values for all inputs", "_enrich_driver_runtime_metadata: all metadata fields must be populated identically", "generate_cli_command: generated command must be copy-pasteable and functionally identical", "get_enabled_settings: output set must contain exactly the same TuningType values as before", "to_cli_args: CLI arg string output must be identical for all benchmark configurations"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/tuning/interface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/manifest/io.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/schemas.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/interface.py"}], "state": "done", "title": "Extract repetitive field assignment patterns into data-driven helpers (complexity batch 2)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-field-extraction-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(get_enabled_settings|enrich_driver|build_execution|build_config_block|populate_driver|to_cli_args|manifest_v2|get_enabled_tuning|extract_df_tuning|generate_cli_command)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 10 target functions", "item_id": "complexity-field-extraction-w14", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "Both in tuning interface files. Pattern: for each setting, if condition \u2192 add enum.\nReplace with SETTING_CONDITIONS = [(condition_fn, TuningType.X), ...] + list comprehension.\nget_enabled_settings: ~20 if blocks \u2192 1 loop.\nget_enabled_tuning_types: ~8 if blocks \u2192 1 loop.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify get_enabled_settings and get_enabled_tuning_types with field iteration", "wid": "w1"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "benchbox/core/runner/runner.py - 8+ identical extraction blocks:\n if adapter.X is not None: result.X = adapter.X\nDefine METADATA_FIELDS = [\"hostname\", \"driver_version\", ...] and iterate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _enrich_driver_runtime_metadata with field-mapping loop", "wid": "w2"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "Both in benchbox/core/results/schema.py.\n_build_execution_from_context: 12+ conditional blocks \u2192 FIELD_MAP = {\"ctx_key\": \"result_key\"}\n_build_config_block: fallback extraction \u2192 FALLBACK_SOURCES = [(\"compression\", [...]), ...]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _build_execution_from_context and _build_config_block", "wid": "w3"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "populate_driver_metadata: 8+ `if options.get(k): self.field = v` \u2192 loop over OPTION_FIELD_MAP.\nto_cli_args: 12+ non-default checks \u2192 loop over FIELD_CLI_ARGS = [(field, \"--flag\", converter), ...].\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify populate_driver_metadata and to_cli_args in schemas.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "benchbox/core/manifest/io.py - repeated `if key in obj: d[target] = obj[key]` pattern.\nExtract helper: _add_optional(src, dst, key, target=None) and call it per field.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _manifest_v2_to_dict optional-field extraction", "wid": "w5"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "_extract_df_tuning (dryrun.py): hasattr + is_default \u2192 10 identical blocks \u2192 loop.\ngenerate_cli_command (cli/dryrun.py): non-default \u2192 append \u2192 12 blocks \u2192 loop over\nPARAM_CLI_MAP = [(attr, \"--flag\", default), ...].\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _extract_df_tuning and generate_cli_command", "wid": "w6"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 10 functions below warn threshold", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add logging in extracted helpers beyond what's in the original code", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "add retry logic or new error handling in the extracted helpers", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "change TPC generation outputs", "why": "any diff in generated data breaks benchmark reproducibility"}, {"INSTEAD": "(not specified)", "dont": "change the compression format or level when extracting compression helpers", "why": "(unstated)"}], "approach": "For generators: use private methods on the same class; keep artifact paths as instance state.\nFor pipelines: extract IO helpers that take explicit path/connection params (no hidden state).\nFor the streaming chunk loop: use a generator function to yield chunks, making the loop body simpler.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "8 generator and data-loading functions accumulate complexity through mixed concerns:\nartifact management, compression, validation, and loading all interleaved in one\nfunction. Each phase is independent and can be extracted, making the orchestration\nfunction readable and each phase separately testable.\n\nTarget functions:\n benchbox/core/tpcds/generator/manager.py:_generate_local (20)\n benchbox/utils/format_converters/ducklake_converter.py:convert (20)\n benchbox/core/primitives/generator_base.py:_generate_special_test_files (19)\n benchbox/core/tpch/generator.py:_finalize_generation (18)\n benchbox/core/tpcds/generator/streaming.py:_generate_single_table_chunk_streaming (17)\n benchbox/core/tpcdi/benchmark.py:load_data_to_database (17)\n benchbox/platforms/databend/adapter.py:load_data (17)\n benchbox/platforms/influxdb/adapter.py:load_data (17)", "id": "complexity-generators-pipelines-w14", "preserves": ["_finalize_generation: generated file paths must resolve identically for all scale factors", "_generate_local: all output artifacts (files, compression, manifest) must be identical", "_generate_special_test_files: all CSV variants must have identical content to before", "convert (ducklake): row count and schema must match source exactly", "load_data_to_database: all rows must load with identical column types and values"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/primitives/generator_base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/streaming.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/generator.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/influxdb/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/format_converters/ducklake_converter.py"}], "state": "done", "title": "Simplify data generator and pipeline loading functions (complexity batch 5)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-generators-pipelines-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'generator or gen or tpcds or tpch' -q", "description": "Generator unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-generators-pipelines-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(_generate_local|ducklake|special_test_files|finalize_generation|chunk_streaming|load_data_to_database|databend.*load|influxdb.*load)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 8 target functions", "item_id": "complexity-generators-pipelines-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/tpcds/generator/manager.py:220 - phases:\n1. Availability check + smart regeneration decision (lines ~226-254)\n2. Stale artifact cleanup (lines ~265-267)\n3. Sample data vs native generation (lines ~269-291)\n4. Compression (lines ~294-300)\n5. Validation + manifest (lines ~310-313)\nExtract each as a private method. Top-level orchestrates with early returns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _generate_local: separate artifact-check, generation, compress, validate", "wid": "w1"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/utils/format_converters/ducklake_converter.py:49 - clean pipeline:\n- _validate_conversion_inputs(src, dst) \u2192 or raise\n- _setup_duckdb_connection(path, config) \u2192 conn\n- _create_schema_and_tables(conn, src_schema) \u2192 None\n- _transfer_rows(conn, src, progress_cb) \u2192 int\nProgress callback plumbing can use a context manager or be passed explicitly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose ducklake convert(): extract validation, setup, schema, transfer, cleanup", "wid": "w2"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/primitives/generator_base.py:908 - 4+ CSV file variants (errors, nulls,\nquoted fields, PSV) with repetitive patterns. Create FILE_VARIANTS config list:\nFILE_VARIANTS = [{\"name\": \"errors\", \"transform\": inject_errors_fn, ...}, ...]\nReplace per-variant blocks with loop: for v in FILE_VARIANTS: _generate_variant(v).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _generate_special_test_files: data-driven file variant generation", "wid": "w3"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/tpch/generator.py:964 - per-table: 4 file detection strategies tried in\nsequence (compressed, single file, chunks, alternate paths). Extract:\n- _detect_generated_file(table, output_dir) \u2192 Path | None (tries strategies in order)\nMain loop becomes: for table in tables: path = _detect_generated_file(...); if not path: handle_miss()\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _finalize_generation: extract file-detection strategy loop", "wid": "w4"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/tpcds/generator/streaming.py:210 - streaming subprocess with chunk loop.\nExtract:\n- _open_compressed_writer(path, compression) \u2192 context manager\n- _stream_chunks(proc, writer, chunk_size) \u2192 int (returns row count)\nMain function: setup \u2192 stream_chunks() \u2192 post-process checks.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _generate_single_table_chunk_streaming chunk I/O loop", "wid": "w5"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "tpcdi/benchmark.py:load_data_to_database (17):\n- Extract _create_tables(conn, schema) and _load_table_file(conn, table, path, batch_size).\ndatabend/adapter.py:load_data (17):\n- Extract _convert_row_batch(rows, types) \u2192 list[tuple] and _flush_batch(conn, table, batch).\ninfluxdb/adapter.py:load_data (17):\n- Extract TSBS_TABLE_CONFIG = {\"cpu\": {...}, \"disk\": {...}, ...} as module constant.\n- Extract _map_tsbs_fields(row, table_config) \u2192 dict.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose load_data_to_database (tpcdi) and platform load_data functions", "wid": "w6"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 8 functions below warn threshold", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the public render() signature", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "create a base class or mixin for shared helpers", "instead": "(not specified)", "why": "module-level functions are fine"}, {"dont": "extract so many small helpers that reading render() requires jumping between 10 methods", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "optimize the rendering logic while refactoring", "instead": "(not specified)", "why": "behavior first, optimization later"}], "approach": "Extract helper methods as private (prefixed _) on the same class or as module-level\nfunctions if they don't use self. Keep render() as the public entry point that\norchestrates helpers in order.\n\nReference existing decomposed charts: ascii/line_chart.py is well-structured -\nnote how it organizes helpers as private methods. Follow that pattern.\n\nTest approach: capture render() output before and after and compare with assertEqual.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "7 ASCII chart render() functions accumulate complexity by doing layout calculation,\ndata normalization, header rendering, bar/cell rendering, and legend rendering all in\none method. Each concern is independent and can be a private helper method. This\nreduces per-function complexity from 15-20 to well below the warn threshold without\nchanging behavior.\n\nTarget functions:\n benchbox/core/visualization/ascii/heatmap.py:render (20)\n benchbox/core/visualization/ascii/histogram.py:_render_simple_chunk (20)\n benchbox/core/visualization/ascii/histogram.py:_render_grouped_chunk (20)\n benchbox/core/visualization/ascii/diverging_bar.py:render (17)\n benchbox/core/visualization/ascii/summary_box.py:render (17)\n benchbox/core/visualization/ascii/sparkline_table.py:render (15)\n benchbox/cli/commands/visualize.py:_render_ascii_charts (16)\n\nNote: Several render() functions are already approved as required-complexity ignores\n(line_chart, box_plot, scatter_plot, bar_chart, rank_table, cdf_chart). The functions\nlisted above are NOT in that list - their complexity is reducible.", "id": "complexity-ascii-renderers-w14", "preserves": ["All chart output must be pixel-for-pixel identical - test with known data fixtures", "Color codes and ANSI escape sequences must be identical", "Column width calculations must produce the same values", "render() method signatures unchanged (callers pass same arguments)"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/visualize.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/diverging_bar.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/heatmap.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/histogram.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/sparkline_table.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/summary_box.py"}], "state": "done", "title": "Decompose monolithic ASCII visualization render() functions (complexity batch 3)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-ascii-renderers-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'ascii or visual or chart or render' -q", "description": "Visualization-specific tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-ascii-renderers-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(heatmap|histogram|diverging|summary_box|sparkline|visualize)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 7 target functions", "item_id": "complexity-ascii-renderers-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "benchbox/core/visualization/ascii/heatmap.py:render (line 91)\nExtract to private methods on the class:\n- _compute_column_widths(data, headers) \u2192 list[int]\n- _render_title_header(title, total_width) \u2192 list[str]\n- _render_cell(value, normalized, width, colorize) \u2192 str\n- _render_legend(lines) \u2192 list[str]\nMain render() orchestrates via calls to these helpers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose heatmap render(): extract column-width calc, header, cell rendering, legend", "wid": "w1"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "benchbox/core/visualization/ascii/histogram.py - both functions (lines 236, 369)\nBoth share bar-height calculation and y-axis label logic:\n- Extract _compute_bar_height(value, max_value, chart_height) \u2192 int\n- Extract _render_y_axis_label(row_idx, max_value, chart_height) \u2192 str\n- Extract _render_footer_legend(series_names, colors) \u2192 list[str]\n_render_grouped_chunk also: extract _get_series_lookup(data) \u2192 dict\nAfter extraction, _render_grouped_chunk should call _render_simple_chunk for\nshared rendering logic where possible (DRY).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose histogram _render_simple_chunk and _render_grouped_chunk", "wid": "w2"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "diverging_bar.py:render (line 64):\n- Extract _partition_series(data) \u2192 (improvements, regressions)\n- Extract _compute_layout(improvements, regressions, width) \u2192 LayoutConfig\nsummary_box.py:render (line 88):\n- Extract _build_metric_rows(metrics) \u2192 list[str]\n- Extract _pad_to_equal_length(left, right) \u2192 tuple[list, list]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose diverging_bar and summary_box render() functions", "wid": "w3"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": [], "notes": "sparkline_table.py:render (line 63):\n- Extract _normalize_column(col_data) \u2192 list[float]\n- Extract _render_sparkline(values, width) \u2192 str\nvisualize.py:_render_ascii_charts (line 20):\n- Extract _resolve_chart_type(template, chart_type) \u2192 str\n- Replace if/elif chain for chart type validation with dict lookup\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose sparkline_table render() and _render_ascii_charts dispatch", "wid": "w4"}, {"evidence": null, "item_id": "complexity-ascii-renderers-w14", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 7 functions below warn threshold", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add module-wide or directory-wide complexity excludes", "instead": "add only function-level excludes with rationale.", "why": "this hides future regressions"}, {"dont": "collapse platform-specific error handling into generic paths", "instead": "share scaffolding only.", "why": "adapters have distinct semantics"}, {"dont": "lower global complexity thresholds to make checks pass", "instead": "reduce code complexity or justify narrow ignores.", "why": "this weakens enforcement"}, {"dont": "mix behavior changes with complexity-only refactors", "instead": "keep functional behavior stable and test-backed.", "why": "this obscures regressions"}], "approach": "Use an evidence-first workflow:\n1) Build a complete classification matrix for every flagged function.\n2) Apply ignore entries only for truly irreducible complexity with explicit rationale.\n3) Refactor accidental complexity in subsystem batches with focused tests.\n4) Re-run complexity and regression tests after each batch.\n5) Keep threshold policy (`max_complexity = 20`) intact unless a separate policy change is approved.\n\nPreferred refactor strategy is extraction into private helpers with clear contracts and\nreduced branch nesting, not broad architectural rewrites.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "complexity-check-triage-and-remediation", "id": 63, "reason": "Out of scope for this complexity remediation item; requires separate architecture proposal.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'deep redesign' with no scope; explicitly requires a separate architecture proposal that does not exist.", "summary": "Deep redesign of benchmark lifecycle architecture beyond targeted extraction"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "complexity-check-triage-and-remediation", "id": 64, "reason": "Requires credentials/environments and should be handled in dedicated live-test follow-up if unit/integration mocks are insufficient.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague trigger ('if mocks insufficient'); no concrete live-test gap identified, no open item covers general adapter-refactor live validation.", "summary": "Live cloud integration validation for every adapter refactor"}], "deps": [], "description": "`make complexity-check` currently fails with significant complexity debt that mixes\nintentional orchestration logic and accidental branching complexity.\n\nBaseline captured on 2026-02-20:\n- Total functions scanned: 4906\n- Warnings (score >= 15): 106\n- Failures (score > 20): 70\n- Highest offenders: `_extract_common_params` (76), `build_result_payload` (63),\n `render_ascii_chart_from_results` (51), multiple `load_data` methods in cloud\n adapters, and runner/dataframe orchestration methods.\n\nThis TODO delivers a complete decision record for every flagged function (simplify vs\nintentional ignore), implements refactors for accidental complexity, and applies narrow\nignore entries only where complexity is demonstrably irreducible and justified.", "id": "complexity-check-triage-and-remediation", "preserves": ["CLI output contracts for compare/export/visualization commands remain stable.", "Complexity check continues to detect real regressions (no broad suppressions).", "DataFrame benchmark execution parity between pandas-family and expression-family adapters.", "Platform `load_data` semantics, supported file types, and option handling remain backward compatible.", "`benchbox run` lifecycle behavior and phase ordering remain unchanged."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/ or CI/CD configuration"}, {"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "benchbox/**/public API signatures or CLI command interfaces"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml (dependency specifications)"}, {"kind": "do_not_modify", "path_glob": "tests/fixtures/ or test fixture data"}, {"kind": "only_modify", "path_glob": "_project/quality/complexity/"}, {"kind": "only_modify", "path_glob": "benchbox/**/*.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml (complexity config section only)"}, {"kind": "only_modify", "path_glob": "quality/complexity/classification-matrix.yaml"}, {"kind": "only_modify", "path_glob": "scripts/check_complexity.py"}], "state": "done", "title": "Complexity check follow-up: triage intended complexity and remediate accidental complexity", "verifications": [{"command": "uv run -- python scripts/check_complexity.py --no-fail --top 200", "description": "Complexity baseline and classification inventory are complete and consistent", "expected": "Report completes successfully and all functions above warning threshold are represented in the classification matrix.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 1}, {"command": "make complexity-check", "description": "Primary complexity gate passes after remediation and approved ignores", "expected": "PASS with zero functions above max complexity threshold.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q -n 0", "description": "Fast regression suite remains green", "expected": "All fast tests pass with 0 failures.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core tests/unit/platforms tests/unit/cli -q -n 0", "description": "Touched subsystem tests remain green", "expected": "Targeted unit suites pass for modified modules.", "item_id": "complexity-check-triage-and-remediation", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": [], "notes": "Persist current output from `scripts/check_complexity.py --no-fail --top 200` under\n`_project/quality/complexity/` and seed the canonical machine-readable inventory at\n`quality/complexity/classification-matrix.yaml` with all functions at score >= 15,\nincluding score, file, function, and module family.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline complexity artifacts and seed function inventory", "wid": "w1"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Normalize shared load flow into scoped helpers/mixins for adapters with parallel logic\n(Redshift/BigQuery/Databricks/Trino/Presto/Athena/Snowflake) without changing platform-specific semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch C: platform adapter load_data families", "wid": "w10"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Decompose large lifecycle/result-shaping functions into deterministic units with clear\ncontracts and easier test coverage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch D: core runner, result schema, and TPC-DS paths", "wid": "w11"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w10", "w11", "w8", "w9"], "notes": "Validated touched areas with focused suites:\n- `tests/unit/platforms/test_{bigquery,presto,trino,athena,snowflake}_adapter.py`\n- `tests/integration/test_tpch_maintenance_test.py`\n- `tests/unit/core/tpcds_obt/test_tpcds_obt_queries.py`\n- `tests/unit/cli/test_quiet_mode.py`\n- `tests/unit/platforms/test_platform_driver_runtime_contract.py`\nCombined targeted run result: 250 passed, 19 deselected.\nFull `-m fast` run currently reports one order-dependent failure in\n`test_dataframe_adapter_contract_rejects_isolated_runtime_without_binding_support`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run targeted tests for each refactor batch and fix regressions", "wid": "w12"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w12", "w7"], "notes": "Ensure intentional complexity is narrowly allowlisted and accidental complexity\nreductions remove all remaining failures above max threshold.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run complexity checks and confirm pass/fail posture with updated ignores", "wid": "w13"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w13"], "notes": "Before (2026-02-20 baseline): 4906 scanned, 106 warnings, 70 failures.\nAfter remediation: 5048 scanned, 109 warnings, 0 failures.\nNet change: failures reduced by 70; complexity gate now passes.\nUpdated complexity artifact: `_project/quality/complexity/latest-report.txt`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document before/after metrics, update TODO status, and regenerate indexes", "wid": "w14"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w1"], "notes": "Create explicit criteria for `simplify_now`, `ignore_with_rationale`, and\n`defer_with_owner`, including examples and disqualifiers for broad ignores.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define triage rubric for intended vs accidental complexity", "wid": "w2"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w2"], "notes": "Review each failing function in `benchbox/core/` (runner, result shaping, lifecycle)\nand record: decision category, concrete rationale, proposed refactor strategy or exact\nignore key (`file:function`), and required regression tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify failing functions in core runner, result schema, and TPC-DS paths (>20)", "wid": "w3"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w2"], "notes": "Review each failing function in `benchbox/platforms/` and dataframe-related modules.\nRecord same evidence as w3. Includes cloud adapter `load_data` families and\ndataframe runner/mixin hotspots.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify failing functions in platform adapters and dataframe orchestration (>20)", "wid": "w4"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w2"], "notes": "Review remaining failing functions in `benchbox/cli/`, chart rendering, comparison\noutput, and any other modules. Record same evidence as w3.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify failing functions in CLI, visualization, and remaining modules (>20)", "wid": "w5"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w3", "w4", "w5"], "notes": "Classify all warning-band functions and assign priority buckets:\nimmediate, near-term, monitor-only. May span multiple sessions if count exceeds ~40;\nbatch by module family if needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify warning-band functions (15-20) and prioritize remediation queue", "wid": "w6"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Update `[tool.benchbox.complexity].exclude_functions` in `pyproject.toml` only for\napproved `ignore_with_rationale` cases. No module-wide excludes. Every entry must map\nto the classification matrix and include a re-review note.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Finalize ignore candidates and add narrow exclusions with rationale linkage", "wid": "w7"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Split high-complexity render/format functions into smaller pure helpers while\npreserving output formatting and user-facing behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch A: visualization and CLI formatting paths", "wid": "w8"}, {"evidence": null, "item_id": "complexity-check-triage-and-remediation", "needs": ["w6"], "notes": "Reduce branching in dataframe benchmark mixins and unified frame hotspots by extracting\nmode-specific helpers and consolidating repeated decision logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor accidental complexity batch B: dataframe orchestration hotspots", "wid": "w9"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "add logging or validation to the new helpers beyond what's in the original", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "change function signatures visible to callers (CLI commands, test fixtures)", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "merge phases that have different error handling", "instead": "(not specified)", "why": "keep their error paths distinct"}, {"dont": "over-extract: aim for 3-5 helpers per function, not 10", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Pattern: identify the phases of each function using blank-line or comment boundaries.\nExtract each phase to a private function with a clear name describing its role.\nKeep all extracted functions in the same file unless they're generic utilities.\n\nFor wizard steps: prefer a single _wizard_step() helper over one helper per step,\nsince steps are structurally identical (same shape, different data).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "9 CLI and core orchestration functions mix multiple independent concerns (validation,\nloading, business logic, output) in a single large function. Each function has 3-6\ndistinct phases that can be extracted to named private helpers, making the top-level\nfunction a readable orchestration narrative.\n\nTarget functions:\n benchbox/base.py:create_enhanced_benchmark_result (20)\n benchbox/cli/commands/compare.py:_run_file_comparison (20)\n benchbox/cli/tuning.py:run_dataframe_write_wizard (20)\n benchbox/cli/tuning_resolver.py:resolve_tuning (18)\n benchbox/cli/commands/shell.py:shell (18)\n benchbox/core/cost/integration.py:add_cost_estimation_to_results (18)\n benchbox/core/dryrun.py:_extract_ddl_preview (18)\n benchbox/core/dryrun.py:save_dry_run_results (15)\n benchbox/cli/commands/aggregate.py:aggregate (16)", "id": "complexity-cli-orchestration-w14", "preserves": ["_run_file_comparison: comparison output (all formats) must be identical", "add_cost_estimation_to_results: cost values must be identical (floating point exact)", "create_enhanced_benchmark_result: output BenchmarkResult must be structurally identical", "resolve_tuning: tuning resolution logic must produce same config for all inputs", "run_dataframe_write_wizard: wizard UX flow and resulting config must be unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/aggregate.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/compare.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/shell.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/tuning_resolver.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/cost/integration.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}], "state": "done", "title": "Extract phases from monolithic CLI orchestration functions (complexity batch 4)", "verifications": [{"command": "uv run -- python -m pytest tests/integration/ -q --timeout=30", "description": "Integration tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-cli-orchestration-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-cli-orchestration-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(create_enhanced|run_file_comparison|write_wizard|resolve_tuning|shell.*329|add_cost_estimation|extract_ddl|save_dry_run|aggregate.*40)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 9 target functions", "item_id": "complexity-cli-orchestration-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/base.py:588 - Mix of result field extraction, fallback normalization,\nand dict manipulation. Extract:\n- _normalize_benchmark_id(raw) \u2192 str\n- _extract_query_results(raw_queries) \u2192 list[QueryResult]\n- _build_execution_meta(raw) \u2192 ExecutionMeta\nTop-level becomes: normalize \u2192 extract \u2192 build \u2192 return.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose create_enhanced_benchmark_result into factory + normalizer helpers", "wid": "w1"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/cli/commands/compare.py:1082 - four clear phases:\n- _validate_comparison_inputs(file1, file2, opts) \u2192 or raise\n- _load_comparison_files(file1, file2) \u2192 (result1, result2)\n- _compute_comparison(r1, r2, opts) \u2192 ComparisonResult (delegate to existing)\n- _output_comparison(comparison, opts) \u2192 None\nMain function becomes 4 calls + early returns on validation failure.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _run_file_comparison into validate/load/compare/output helpers", "wid": "w2"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/cli/tuning.py:694 - 4+ wizard steps follow identical pattern:\n capability_check \u2192 user_prompt \u2192 config_update\nExtract _wizard_step(capability, prompt, config_key) \u2192 value or None.\nEach step becomes a single function call. Steps are independent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose run_dataframe_write_wizard into per-step handlers", "wid": "w3"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "benchbox/cli/tuning_resolver.py:162 - 4 sequential cases (notuning, auto, explicit path, tuned).\nEach case is self-contained. Extract:\n- _resolve_notuning() \u2192 TuningConfig\n- _resolve_auto(platform, benchmark) \u2192 TuningConfig\n- _resolve_explicit_path(path) \u2192 TuningConfig\n- _resolve_tuned(platform, benchmark) \u2192 TuningConfig\nMain function: mode \u2192 handler dispatch (dict or if/elif, each a 1-liner).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose resolve_tuning into per-mode handler functions", "wid": "w4"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "shell (cli/commands/shell.py:329): extract _resolve_platform_from_args,\n_discover_platform_interactive, _launch_shell_for_platform.\nadd_cost_estimation_to_results (core/cost/integration.py:70): extract\n_validate_cost_inputs, _compute_query_costs, _compute_phase_costs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose shell command and add_cost_estimation_to_results", "wid": "w5"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": [], "notes": "_extract_ddl_preview (dryrun.py:963): extract _build_table_ddl(table, tunings) \u2192 str.\nsave_dry_run_results (dryrun.py:175): extract _save_queries_to_dir, _save_ddl_file.\naggregate (cli/commands/aggregate.py:40): extract _filter_results, _aggregate_metrics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _extract_ddl_preview, save_dry_run_results, and aggregate", "wid": "w6"}, {"evidence": null, "item_id": "complexity-cli-orchestration-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 9 functions below warn threshold", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change public function signatures", "instead": "only extract internal helpers", "why": "callers depend on them"}, {"dont": "move helpers to separate files", "instead": "keep them in the same module", "why": "these are small extractions"}], "approach": "Follow the pattern established in complexity batches 3-7: extract private helper\nfunctions within the same module, preserve the public function signature exactly,\nand ensure the extracted helpers are independently testable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Four CLI functions at CC=14 have extractable sub-concerns that can be\ndecomposed into focused helpers without losing clarity.\n\nTarget functions:\n1. prompt_phases (benchmarks.py:891) - extract _parse_phase_input() for dual-mode parser\n2. prompt_platform_options (benchmarks.py:1387) - extract _prompt_single_option() type-dispatch\n3. prompt_cloud_output_location (cloud_storage.py:21) - extract _prompt_and_validate_cloud_path()\n4. check_dependencies (commands/checks.py:20) - extract _show_matrix(), _show_platform(), _show_overview()", "id": "complexity-cli-prompts-w15", "preserves": ["check_dependencies --matrix, --platform, and default output formats unchanged", "prompt_cloud_output_location() validation logic for cloud:// paths unchanged", "prompt_phases() return type and behavior remain identical (list of phase strings)", "prompt_platform_options() correctly prompts for all option types (choice, bool, int, string)"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/benchmarks.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/cloud_storage.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/checks.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}], "state": "done", "title": "Reduce complexity in CLI prompt and command functions (complexity batch 8)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/ -v", "description": "All CLI tests pass", "expected": "all tests pass", "item_id": "complexity-cli-prompts-w15", "last_result": null, "last_run": null, "seq": 1}, {"command": "make complexity-check", "description": "Complexity check passes and these functions drop below 14", "expected": "PASSED; prompt_phases, prompt_platform_options, prompt_cloud_output_location, check_dependencies no longer in top-20", "item_id": "complexity-cli-prompts-w15", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _parse_phase_input() from prompt_phases in benchmarks.py", "wid": "w1"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _prompt_single_option() from prompt_platform_options in benchmarks.py", "wid": "w2"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _prompt_and_validate_cloud_path() from prompt_cloud_output_location in cloud_storage.py", "wid": "w3"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _show_matrix(), _show_platform(), _show_overview() from check_dependencies in checks.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-cli-prompts-w15", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run complexity check and full test suite to verify no regressions", "wid": "w5"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change the scoring formula in _rank_by_composite", "instead": "only restructure the code", "why": "results must remain comparable with prior runs"}, {"dont": "create an ABC or strategy pattern for the DuckDB/SQLite dispatch", "instead": "simple if/else with extracted helpers is sufficient", "why": "only 2 platforms exist"}], "approach": "Follow complexity batches 3-7 pattern. For _run_sql_query, the DuckDB and SQLite\nbranches are nearly identical - factor out shared logic (load tables loop, warmup loop,\nbenchmark loop with row count capture) into a common pattern or extract per-platform\nhelpers that share a _time_query_iterations() utility.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Five core functions at CC=14 have clear decomposition opportunities.\n\nTarget functions:\n1. validate (comparison.py:196) - extract _detect_outliers() and _validate_query_overlap()\n2. _rank_by_composite (ranking.py:361) - extract _compute_raw_scores() and _normalize_scores()\n3. _run_sql_query (benchmark_suite.py:1081) - extract _run_on_duckdb() and _run_on_sqlite()\n (most clear-cut: two 90%-identical platform branches)\n4. convert_csv_to_parquet (data_loader.py:255) - extract _build_write_kwargs() and _resolve_arrow_types()\n5. execute_dry_run (dryrun.py:110) - extract _resolve_execution_mode() and _init_platform_adapter()", "id": "complexity-core-analysis-dataframe-w15", "preserves": ["_rank_by_composite() composite scoring formula and weight handling unchanged", "_run_sql_query() returns identical (mean_time, row_count) tuple for both DuckDB and SQLite", "convert_csv_to_parquet() handles both CSV and TBL files with trailing delimiters", "execute_dry_run() DryRunResult construction and error handling paths unchanged", "validate() error/warning accumulation behavior and return type unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/analysis/comparison.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/analysis/ranking.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/benchmark_suite.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}], "state": "done", "title": "Reduce complexity in core analysis and dataframe functions (complexity batch 9)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/ -v", "description": "Core analysis tests pass", "expected": "all tests pass", "item_id": "complexity-core-analysis-dataframe-w15", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/dataframe/ -v -k 'not integration'", "description": "Dataframe tests pass", "expected": "all tests pass", "item_id": "complexity-core-analysis-dataframe-w15", "last_result": null, "last_run": null, "seq": 2}, {"command": "make complexity-check", "description": "Complexity check passes", "expected": "PASSED; these functions no longer in top-20", "item_id": "complexity-core-analysis-dataframe-w15", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": "Most impactful - near-duplicate platform branches are the primary code smell", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _run_on_duckdb() and _run_on_sqlite() from _run_sql_query in benchmark_suite.py", "wid": "w1"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _compute_raw_scores() and _normalize_scores() from _rank_by_composite in ranking.py", "wid": "w2"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _detect_outliers() and _validate_query_overlap() from validate in comparison.py", "wid": "w3"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _build_write_kwargs() and _resolve_arrow_types() from convert_csv_to_parquet in data_loader.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _resolve_execution_mode() and _init_platform_adapter() from execute_dry_run in dryrun.py", "wid": "w5"}, {"evidence": null, "item_id": "complexity-core-analysis-dataframe-w15", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run complexity check and full test suite to verify no regressions", "wid": "w6"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change function signatures or return types", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "merge separate dict constants into a single mega-dict", "instead": "(not specified)", "why": "one dict per function"}, {"dont": "refactor surrounding functions beyond the target chain", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "use lambdas in dict values when the original used inline expressions; keep readable", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Pattern: identify the if/elif chain, extract the mapping as a module-level dict constant\n(UPPER_SNAKE_CASE), replace the chain with a single dict.get() or dict[] lookup.\nRaise the same exception on KeyError that the original raised on else/fallthrough.\n\nReference: benchbox/core/tpcds_obt/query_conversion.py already uses this pattern in\nother functions (_post_process dispatch) - follow that style.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "8 functions have high cyclomatic complexity caused purely by if/elif chains performing\ndata lookups. These are the simplest category to fix: replace the chain with a module-level\ndict constant and a single lookup. No logic changes, no API changes, just data vs code\nseparation.\n\nTarget functions:\n benchbox/core/tpcds_obt/query_conversion.py:_update_role_map (19)\n benchbox/core/tpcds_obt/query_conversion.py:_table_from_prefix (18)\n benchbox/core/tpcds/maintenance_test.py:_map_to_maintenance_operation_type (15)\n benchbox/cli/exceptions.py:_handle_cli_error (15)\n benchbox/release/content_validation.py:_is_example_line (15)\n benchbox/utils/database_naming.py:_get_optimizations_suffix (15)\n benchbox/core/metadata_primitives/generator.py:_find_objects_with_prefix (16)\n benchbox/cli/composite_params.py:parse (18, partial)\n\nExpected outcome: all 8 functions drop below warn threshold (15).", "id": "complexity-dict-dispatch-w14", "preserves": ["_handle_cli_error: all existing error types must still trigger the same help output", "_map_to_maintenance_operation_type: mapping must be exact (drives TPC-DS maintenance spec)", "_table_from_prefix: column prefix \u2192 table mapping must be 100% accurate (drives OBT SQL)", "_update_role_map: role map output must be identical for all TPC-DS OBT queries", "parse: CLI parameter parsing output must be byte-for-byte identical for all current inputs"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/composite_params.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/exceptions.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/generator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/maintenance_test.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds_obt/query_conversion.py"}, {"kind": "only_modify", "path_glob": "benchbox/release/content_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/database_naming.py"}], "state": "done", "title": "Replace if/elif lookup chains with dict-based dispatch (complexity batch 1)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-dict-dispatch-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(query_conversion|maintenance_test|exceptions|content_validation|database_naming|generator|composite_params)'", "description": "All 8 target functions drop below warn threshold", "expected": "No WARN entries for the 8 target functions", "item_id": "complexity-dict-dispatch-w14", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "Both are in query_conversion.py. _table_from_prefix has ~15-way if/elif \u2192 dict.\n_update_role_map maps table names \u2192 role prefixes \u2192 dict constant TABLE_TO_ROLE_PREFIX.\nThe role inference calls after the lookup can stay; only the lookup changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _update_role_map and _table_from_prefix with dict constants", "wid": "w1"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/core/tpcds/maintenance_test.py - maps (operation, table) tuples to\nMaintenanceOperationType. Simple dict: OPERATION_TABLE_TO_TYPE = {(\"INSERT\", \"t\"): ...}\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _map_to_maintenance_operation_type with dict mapping", "wid": "w2"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/cli/exceptions.py - 6+ isinstance() checks mapping error types to\nhelp methods. Dict: ERROR_HANDLERS = {PlatformNotFoundError: ..., ...}\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _handle_cli_error isinstance chain with dispatch dict", "wid": "w3"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/release/content_validation.py - multiple if-checks for different\nexample patterns. Compile all patterns into EXAMPLE_PATTERNS = [re.compile(...), ...]\nand use any(p.match(line) for p in EXAMPLE_PATTERNS).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _is_example_line multi-pattern checks with compiled regex list", "wid": "w4"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "_get_optimizations_suffix (database_naming.py): dict mapping (platform, opts) \u2192 suffix.\n_find_objects_with_prefix (metadata_primitives/generator.py): SQL template dict keyed\nby (object_type, dialect).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace _get_optimizations_suffix and _find_objects_with_prefix with dicts", "wid": "w5"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": [], "notes": "benchbox/cli/composite_params.py - key dispatch for sample/first/queries uses\nif/elif; replace with TYPE_CONVERTERS = {\"sample\": float, \"first\": int, ...}.\nQuery special-case parsing (edge case) stays inline but extract to helper.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify composite_params.py:parse query/key dispatch with dict mapper", "wid": "w6"}, {"evidence": null, "item_id": "complexity-dict-dispatch-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 8 functions below warn threshold", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the output type or structure of any function (callers depend on these)", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "create a FieldExtractor class", "instead": "(not specified)", "why": "a list + loop is sufficient and more readable"}, {"dont": "merge the field definitions across functions", "instead": "(not specified)", "why": "each function gets its own field list"}, {"dont": "skip testing get_enabled_settings", "instead": "(not specified)", "why": "it drives all tuning decisions"}], "approach": "Three patterns to apply:\n1. Enumeration loop: FIELDS = [(\"src_attr\", \"dst_attr\"), ...]; for s, d in FIELDS: setattr(dst, d, getattr(src, s))\n2. Conditional accumulation: CONDITIONS = [(predicate_fn, value), ...]; result = [v for fn, v in CONDITIONS if fn()]\n3. Optional dict builder: KEYS = [(\"src\", \"dst\"), ...]; return {d: src[s] for s, d in KEYS if s in src}\n\nDo not add abstraction layers beyond what's needed - a simple list + comprehension is sufficient.\nKeep field definitions close to the function (module-level constant or near the function).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "10 functions have high complexity caused by 10-20 nearly-identical conditional blocks\nthat check a field, apply a transformation, and assign to a target. These are pure\nmechanical repetition - not real branching logic - and should be collapsed into either\na loop over a field-definition list or a helper that handles the pattern once.\n\nTarget functions and pattern:\n benchbox/core/dataframe/tuning/interface.py:get_enabled_settings (20) - check condition \u2192 add enum\n benchbox/core/runner/runner.py:_enrich_driver_runtime_metadata (20) - extract adapter field \u2192 assign to result\n benchbox/core/results/schema.py:_build_execution_from_context (18) - extract context field \u2192 dict entry\n benchbox/core/results/schema.py:_build_config_block (17) - extract with fallback \u2192 dict entry\n benchbox/core/schemas.py:populate_driver_metadata (17) - options.get() \u2192 assign if not set\n benchbox/core/schemas.py:to_cli_args (16) - non-default field \u2192 append arg string\n benchbox/core/manifest/io.py:_manifest_v2_to_dict (18) - optional field present \u2192 add to dict\n benchbox/core/tuning/interface.py:get_enabled_tuning_types (16) - check config attr \u2192 add to set\n benchbox/core/dryrun.py:_extract_df_tuning (16) - hasattr + is_default \u2192 build dict\n benchbox/cli/dryrun.py:generate_cli_command (16) - non-default flag \u2192 append to parts list\n\nEach requires a field-definition approach: declare a list/dict of (source_field,\ncondition, transform, target_field) tuples, iterate once.", "id": "complexity-field-extraction-w14", "preserves": ["_build_execution_from_context: result dict must have identical keys/values for all inputs", "_enrich_driver_runtime_metadata: all metadata fields must be populated identically", "generate_cli_command: generated command must be copy-pasteable and functionally identical", "get_enabled_settings: output set must contain exactly the same TuningType values as before", "to_cli_args: CLI arg string output must be identical for all benchmark configurations"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/tuning/interface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/manifest/io.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/schemas.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/interface.py"}], "state": "done", "title": "Extract repetitive field assignment patterns into data-driven helpers (complexity batch 2)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-field-extraction-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(get_enabled_settings|enrich_driver|build_execution|build_config_block|populate_driver|to_cli_args|manifest_v2|get_enabled_tuning|extract_df_tuning|generate_cli_command)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 10 target functions", "item_id": "complexity-field-extraction-w14", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "Both in tuning interface files. Pattern: for each setting, if condition \u2192 add enum.\nReplace with SETTING_CONDITIONS = [(condition_fn, TuningType.X), ...] + list comprehension.\nget_enabled_settings: ~20 if blocks \u2192 1 loop.\nget_enabled_tuning_types: ~8 if blocks \u2192 1 loop.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify get_enabled_settings and get_enabled_tuning_types with field iteration", "wid": "w1"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "benchbox/core/runner/runner.py - 8+ identical extraction blocks:\n if adapter.X is not None: result.X = adapter.X\nDefine METADATA_FIELDS = [\"hostname\", \"driver_version\", ...] and iterate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _enrich_driver_runtime_metadata with field-mapping loop", "wid": "w2"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "Both in benchbox/core/results/schema.py.\n_build_execution_from_context: 12+ conditional blocks \u2192 FIELD_MAP = {\"ctx_key\": \"result_key\"}\n_build_config_block: fallback extraction \u2192 FALLBACK_SOURCES = [(\"compression\", [...]), ...]\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _build_execution_from_context and _build_config_block", "wid": "w3"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "populate_driver_metadata: 8+ `if options.get(k): self.field = v` \u2192 loop over OPTION_FIELD_MAP.\nto_cli_args: 12+ non-default checks \u2192 loop over FIELD_CLI_ARGS = [(field, \"--flag\", converter), ...].\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify populate_driver_metadata and to_cli_args in schemas.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "benchbox/core/manifest/io.py - repeated `if key in obj: d[target] = obj[key]` pattern.\nExtract helper: _add_optional(src, dst, key, target=None) and call it per field.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _manifest_v2_to_dict optional-field extraction", "wid": "w5"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": [], "notes": "_extract_df_tuning (dryrun.py): hasattr + is_default \u2192 10 identical blocks \u2192 loop.\ngenerate_cli_command (cli/dryrun.py): non-default \u2192 append \u2192 12 blocks \u2192 loop over\nPARAM_CLI_MAP = [(attr, \"--flag\", default), ...].\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _extract_df_tuning and generate_cli_command", "wid": "w6"}, {"evidence": null, "item_id": "complexity-field-extraction-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 10 functions below warn threshold", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add logging in extracted helpers beyond what's in the original code", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "add retry logic or new error handling in the extracted helpers", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "change TPC generation outputs", "instead": "(not specified)", "why": "any diff in generated data breaks benchmark reproducibility"}, {"dont": "change the compression format or level when extracting compression helpers", "instead": "(not specified)", "why": "(unstated)"}], "approach": "For generators: use private methods on the same class; keep artifact paths as instance state.\nFor pipelines: extract IO helpers that take explicit path/connection params (no hidden state).\nFor the streaming chunk loop: use a generator function to yield chunks, making the loop body simpler.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "8 generator and data-loading functions accumulate complexity through mixed concerns:\nartifact management, compression, validation, and loading all interleaved in one\nfunction. Each phase is independent and can be extracted, making the orchestration\nfunction readable and each phase separately testable.\n\nTarget functions:\n benchbox/core/tpcds/generator/manager.py:_generate_local (20)\n benchbox/utils/format_converters/ducklake_converter.py:convert (20)\n benchbox/core/primitives/generator_base.py:_generate_special_test_files (19)\n benchbox/core/tpch/generator.py:_finalize_generation (18)\n benchbox/core/tpcds/generator/streaming.py:_generate_single_table_chunk_streaming (17)\n benchbox/core/tpcdi/benchmark.py:load_data_to_database (17)\n benchbox/platforms/databend/adapter.py:load_data (17)\n benchbox/platforms/influxdb/adapter.py:load_data (17)", "id": "complexity-generators-pipelines-w14", "preserves": ["_finalize_generation: generated file paths must resolve identically for all scale factors", "_generate_local: all output artifacts (files, compression, manifest) must be identical", "_generate_special_test_files: all CSV variants must have identical content to before", "convert (ducklake): row count and schema must match source exactly", "load_data_to_database: all rows must load with identical column types and values"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/primitives/generator_base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/generator/streaming.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/generator.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/influxdb/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/format_converters/ducklake_converter.py"}], "state": "done", "title": "Simplify data generator and pipeline loading functions (complexity batch 5)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-generators-pipelines-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'generator or gen or tpcds or tpch' -q", "description": "Generator unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-generators-pipelines-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(_generate_local|ducklake|special_test_files|finalize_generation|chunk_streaming|load_data_to_database|databend.*load|influxdb.*load)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 8 target functions", "item_id": "complexity-generators-pipelines-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/tpcds/generator/manager.py:220 - phases:\n1. Availability check + smart regeneration decision (lines ~226-254)\n2. Stale artifact cleanup (lines ~265-267)\n3. Sample data vs native generation (lines ~269-291)\n4. Compression (lines ~294-300)\n5. Validation + manifest (lines ~310-313)\nExtract each as a private method. Top-level orchestrates with early returns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _generate_local: separate artifact-check, generation, compress, validate", "wid": "w1"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/utils/format_converters/ducklake_converter.py:49 - clean pipeline:\n- _validate_conversion_inputs(src, dst) \u2192 or raise\n- _setup_duckdb_connection(path, config) \u2192 conn\n- _create_schema_and_tables(conn, src_schema) \u2192 None\n- _transfer_rows(conn, src, progress_cb) \u2192 int\nProgress callback plumbing can use a context manager or be passed explicitly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose ducklake convert(): extract validation, setup, schema, transfer, cleanup", "wid": "w2"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/primitives/generator_base.py:908 - 4+ CSV file variants (errors, nulls,\nquoted fields, PSV) with repetitive patterns. Create FILE_VARIANTS config list:\nFILE_VARIANTS = [{\"name\": \"errors\", \"transform\": inject_errors_fn, ...}, ...]\nReplace per-variant blocks with loop: for v in FILE_VARIANTS: _generate_variant(v).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _generate_special_test_files: data-driven file variant generation", "wid": "w3"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/tpch/generator.py:964 - per-table: 4 file detection strategies tried in\nsequence (compressed, single file, chunks, alternate paths). Extract:\n- _detect_generated_file(table, output_dir) \u2192 Path | None (tries strategies in order)\nMain loop becomes: for table in tables: path = _detect_generated_file(...); if not path: handle_miss()\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _finalize_generation: extract file-detection strategy loop", "wid": "w4"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "benchbox/core/tpcds/generator/streaming.py:210 - streaming subprocess with chunk loop.\nExtract:\n- _open_compressed_writer(path, compression) \u2192 context manager\n- _stream_chunks(proc, writer, chunk_size) \u2192 int (returns row count)\nMain function: setup \u2192 stream_chunks() \u2192 post-process checks.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify _generate_single_table_chunk_streaming chunk I/O loop", "wid": "w5"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": [], "notes": "tpcdi/benchmark.py:load_data_to_database (17):\n- Extract _create_tables(conn, schema) and _load_table_file(conn, table, path, batch_size).\ndatabend/adapter.py:load_data (17):\n- Extract _convert_row_batch(rows, types) \u2192 list[tuple] and _flush_batch(conn, table, batch).\ninfluxdb/adapter.py:load_data (17):\n- Extract TSBS_TABLE_CONFIG = {\"cpu\": {...}, \"disk\": {...}, ...} as module constant.\n- Extract _map_tsbs_fields(row, table_config) \u2192 dict.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose load_data_to_database (tpcdi) and platform load_data functions", "wid": "w6"}, {"evidence": null, "item_id": "complexity-generators-pipelines-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 8 functions below warn threshold", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Following the complexity-check evaluation of the top-20 functions at CC=14,\nfive functions were classified as \"required complexity\" - their branching is\ninherent to the problem domain and cannot be meaningfully reduced without\nadding indirection that hurts readability.\n\nThese should be added to pyproject.toml [tool.benchbox.complexity] exclude_functions\nwith rationale comments, matching the existing pattern established in previous batches.\n\nFunctions to exclude:\n1. benchbox/cli/commands/show_plan.py:show_plan (CC=14)\n - Error handling (4 except branches) + 3-way format dispatch; irreducible for a CLI command\n2. benchbox/cli/commands/tuning_group.py:_display_config_settings_table (CC=14)\n - Guard count proportional to number of config fields; no reducible structure\n3. benchbox/core/query_plans/parsers/base.py:_harmonize_operator_type (CC=14)\n - 12-way keyword\u2192enum mapping; branching count equals enum member count\n4. benchbox/core/concurrency/analyzer.py:analyze_contention (CC=14)\n - Mutually-exclusive classification decision tree; genuinely a classifier\n5. benchbox/core/metadata_primitives/complexity.py:__post_init__ (CC=14)\n - N independent field validation guards; inherent to dataclass validation", "id": "complexity-ignore-required-w15", "preserves": [], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Add complexity exclusions for 5 functions with inherent required complexity", "verifications": [{"command": "make complexity-check", "description": "Complexity check still passes after adding exclusions", "expected": "PASSED: all functions within complexity limit", "item_id": "complexity-ignore-required-w15", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "complexity-ignore-required-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 5 exclude_functions entries to pyproject.toml with rationale comments", "wid": "w1"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add functions that are reducible", "why": "only genuinely irreducible cases documented above"}, {"INSTEAD": "(not specified)", "dont": "add module-level exclusions; per-function ignores only", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "remove or modify existing approved ignores", "why": "(unstated)"}], "approach": "Follow the existing pattern in pyproject.toml [tool.benchbox.complexity] exclude_functions.\nGroup new entries by the same category headings used for existing ones.\nEach entry: \"file/path.py:function_name\", # NN - rationale\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "19 functions flagged by make complexity-check have complexity that is genuinely\nirreducible - TPC spec implementations, interactive REPL state machines, multi-platform\nAPI dispatch, AST traversal, and thread-safety patterns. These must be added to the\nexclude_functions list in pyproject.toml with rationale comments so the warning count\naccurately reflects addressable complexity only.\n\nFunctions to add:\nScore 20:\n benchbox/core/tpcdi/etl/batch.py:_execute_parallel_pipeline\n benchbox/platforms/credentials/athena.py:setup_athena_credentials\n benchbox/platforms/credentials/bigquery.py:setup_bigquery_credentials\n benchbox/platforms/dataframe/unified_frame.py:sort\n\nScore 19:\n benchbox/platforms/dataframe/unified_frame.py:_apply_datafusion_post_ops\n\nScore 18:\n benchbox/core/runner/dataframe_runner.py:_execute_dataframe_queries\n benchbox/core/tpcds/throughput_test.py:run\n benchbox/core/tpcds_obt/query_conversion.py:_rewrite_select\n benchbox/core/tpch/maintenance_test.py:_execute_rf1\n benchbox/core/tpch/power_test.py:run\n benchbox/core/tpch/throughput_test.py:run\n\nScore 17:\n benchbox/core/expected_results/registry.py:_get_benchmark_results\n benchbox/core/query_plans/parsers/datafusion.py:_convert_to_logical_operator\n benchbox/core/tpch/throughput_test.py:_execute_stream\n benchbox/core/transaction_primitives/benchmark.py:setup\n benchbox/platforms/base/adapter.py:capture_query_plan\n\nScore 16:\n benchbox/cli/commands/shell.py:_launch_sqlite_shell\n benchbox/platforms/credentials/snowflake.py:setup_snowflake_credentials\n\nScore 15:\n benchbox/cli/commands/shell.py:_launch_duckdb_shell", "id": "complexity-ignores-w14", "preserves": ["All 19 functions remain behaviorally unchanged - this is annotation only", "Existing approved ignores in exclude_functions list are untouched"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Mark required-complexity functions as approved ignores in pyproject.toml", "verifications": [{"command": "uv run -- python scripts/check_complexity.py --top 150", "description": "Complexity check passes with reduced warning count", "expected": "PASSED: all functions within complexity limit of 20; warnings count drops by ~19", "item_id": "complexity-ignores-w14", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "complexity-ignores-w14", "needs": [], "notes": "Add these with rationale comments (pattern: # NN - brief rationale):\n- _execute_parallel_pipeline: async parallel ETL orchestration with future management\n- setup_athena_credentials: interactive credential wizard; branching is UX-required\n- setup_bigquery_credentials: same as Athena\n- sort: multi-platform dispatch (Polars/PySpark/DataFusion) with different sort APIs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add score-20 TPC/platform required ignores to pyproject.toml", "wid": "w1"}, {"evidence": null, "item_id": "complexity-ignores-w14", "needs": ["w1"], "notes": "Add these with rationale:\n- _apply_datafusion_post_ops (19): literal vs multi-agg arithmetic with distinct semantics\n- _execute_dataframe_queries (18): TPC power test methodology; phases are spec-ordered\n- tpcds/throughput_test.py:run (18): TPC-DS spec; threading is mandatory\n- _rewrite_select (18): AST rewriting with dependency ordering (role_map \u2192 column rewrite)\n- _execute_rf1 (18): TPC-H RF1 spec; SQL building depends on prior step results\n- tpch/power_test.py:run (18): TPC-H power test; spec mandates permutation order\n- tpch/throughput_test.py:run (18): TPC-H throughput; spec mandates TTT measurement\n- _get_benchmark_results (17): thread-safe single-flight cache pattern\n- _convert_to_logical_operator (17): AST node conversion with operator-specific properties\n- _execute_stream (17): TPC-H stream; 22 queries in spec permutation order\n- transaction_primitives/benchmark.py:setup (17): spec-required sequential dependency chain\n- capture_query_plan (17): multi-level filter chain (ID, iteration, sampling, EXPLAIN)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add score 17-19 TPC spec and AST ignores", "wid": "w2"}, {"evidence": null, "item_id": "complexity-ignores-w14", "needs": ["w2"], "notes": "- _launch_sqlite_shell (16): user input dispatch loop; state machine is REPL-inherent\n- setup_snowflake_credentials (16): interactive wizard; multi-step validation with dependencies\n- _launch_duckdb_shell (15): same pattern as SQLite shell\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add score 15-16 interactive REPL and credential wizard ignores", "wid": "w3"}, {"evidence": null, "item_id": "complexity-ignores-w14", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify warning count drops and complexity-check still passes", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add async to sync functions or remove async from async functions", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "change the JSON response structure of any MCP tool or resource", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "merge analytics impl functions", "why": "keep them separate (each serves a different MCP tool)"}, {"INSTEAD": "(not specified)", "dont": "rename MCP tool/resource/prompt names", "why": "they're part of the public API"}], "approach": "MCP registration functions: use factory functions that close over data, returning the\ninner handler. This is the same pattern as existing MCP tools in discovery.py\n(already approved as ignore, but the factory approach is worth borrowing).\n\nAnalytics functions: three-phase pattern - load files, compute metric, format output.\nExtract each phase. Avoid shared mutable state between phases.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "8 MCP tool and analytics functions mix phase logic (file enumeration, metric extraction,\naggregation, formatting) in single long functions. Extracting phases to private helpers\nreduces complexity while keeping the MCP-registered tool function as a readable summary.\n\nTarget functions:\n benchbox/mcp/resources/registry.py:register_all_resources (20)\n benchbox/mcp/tools/analytics.py:_get_performance_trends_impl (20)\n benchbox/mcp/tools/analytics.py:_aggregate_results_impl (18)\n benchbox/mcp/tools/analytics.py:_detect_regressions_impl (17)\n benchbox/mcp/tools/analytics.py:get_query_plan (15)\n benchbox/mcp/prompts/registry.py:register_all_prompts (15)\n benchbox/mcp/tools/benchmark.py:_run_benchmark_impl (16)\n benchbox/mcp/tools/benchmark.py:_validate_config_impl (15)", "id": "complexity-mcp-analytics-w14", "preserves": ["_detect_regressions_impl: regression detection logic and threshold comparison unchanged", "_get_performance_trends_impl: trend values and structure must be numerically identical", "_validate_config_impl: validation error messages must be identical (MCP clients may parse them)", "register_all_prompts: all prompt names, descriptions, and argument specs unchanged", "register_all_resources: all MCP resource URIs and JSON shapes must be identical"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/mcp/prompts/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/analytics.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}], "state": "done", "title": "Decompose MCP tool implementations and analytics functions (complexity batch 6)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-mcp-analytics-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'mcp or analytics or benchmark_tool' -q", "description": "MCP-specific tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-mcp-analytics-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(register_all_resources|performance_trends|aggregate_results|detect_regressions|get_query_plan|register_all_prompts|run_benchmark_impl|validate_config_impl)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 8 target functions", "item_id": "complexity-mcp-analytics-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "benchbox/mcp/resources/registry.py:34 - nested function definitions for\nlist_benchmarks, get_benchmark, list_platforms, get_platform each follow\nidentical fetch \u2192 JSON structure pattern. Extract resource factories:\n- _make_list_resource(fetch_fn, formatter_fn) \u2192 Callable\n- _make_get_resource(fetch_fn, formatter_fn) \u2192 Callable\nRegistration becomes: server.register_resource(uri, _make_list_resource(...))\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose register_all_resources: extract per-resource handler factories", "wid": "w1"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "benchbox/mcp/tools/analytics.py:435 - phases:\n- _load_result_files(result_dir, platform_filter) \u2192 list[RunResult]\n- _extract_metric(result, metric_name) \u2192 float | None\n- _compute_trends(grouped_runs, metric) \u2192 TrendResult\nMain function: load \u2192 filter \u2192 extract per run \u2192 group \u2192 compute trends.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _get_performance_trends_impl into load/filter/extract/aggregate phases", "wid": "w2"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "_aggregate_results_impl (analytics.py:566):\n- Extract _build_group_key(result, group_by) \u2192 str (3-way dispatch \u2192 dict)\n- Extract _extract_timings(result) \u2192 list[float]\n_detect_regressions_impl (analytics.py:278):\n- Extract _load_result_pair(file1, file2) \u2192 (RunResult, RunResult)\n- Extract _detect_query_regression(q1, q2, threshold) \u2192 Regression | None\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _aggregate_results_impl and _detect_regressions_impl", "wid": "w3"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "get_query_plan (analytics.py:101): extract _resolve_result_file(name) \u2192 Path,\n _extract_plan(result, query_id, format) \u2192 str\nregister_all_prompts (prompts/registry.py:20): similar to resources - extract\n _make_benchmark_prompt(template_fn) factory; repeated @mcp.prompt() patterns.\n_run_benchmark_impl (benchmark.py:239): extract _validate_and_build_config,\n _execute_and_format_result.\n_validate_config_impl (benchmark.py:545): extract per-phase validators as small helpers:\n _validate_platform, _validate_mode, _validate_benchmark each return error | None.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify get_query_plan, register_all_prompts, _run_benchmark_impl, _validate_config_impl", "wid": "w4"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 8 functions below warn threshold", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "only restructure the code", "dont": "change TPC-DS spec-mandated behavior in run_streams or maintenance_test", "why": "TPC compliance depends on exact execution semantics"}, {"INSTEAD": "keep them in the same module", "dont": "create new files for extracted helpers", "why": "these are small private functions"}], "approach": "For _compare_operator_properties, use a COMPARE_DISPATCH: dict[str, Callable] pattern\nmapping operator type strings to small comparison functions. For _format_operator_properties,\nthe _format_truncated_list(label, items, max_display=3) helper eliminates the repeated\ntruncation pattern. For run_streams, the key insight is that the sequential loop body\nand the concurrent StreamExecutor._run_single_stream do the same thing - factor into one.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Six core functions at CC=14 in query plans and TPC-DS modules can be decomposed.\n\nTarget functions:\n1. _compare_operator_properties (comparison.py:288) - replace if/elif dispatch with registry dict\n2. _format_operator_properties (visualization.py:297) - extract _format_truncated_list() helper\n3. load_operations_catalog (primitives/catalog/loader.py:24) - extract _parse_operation_entry()\n4. run_streams (tpcds/benchmark/runner.py:687) - extract sequential stream runner, share with concurrent\n5. run (tpcds/maintenance_test.py:131) - split loop from finalization logic\n6. run_official_benchmark (tpcds/official_benchmark.py:97) - extract _extract_metric() helper", "id": "complexity-plans-tpcds-w15", "preserves": ["QphDS@Size composite metric calculation in run_official_benchmark unchanged", "TPC-DS maintenance test success criteria (>=90% success rate) unchanged", "_compare_operator_properties() returns identical diff dict for all operator types", "_format_operator_properties() truncation behavior (max 3 items + '... and N more')", "load_operations_catalog() validation error messages and error types unchanged", "run_streams() concurrent and sequential execution produce identical result structures"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/comparison.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/visualization.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/maintenance_test.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/official_benchmark.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Reduce complexity in query plan and TPC-DS functions (complexity batch 10)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/query_plans/ -v", "description": "Query plan tests pass", "expected": "all tests pass", "item_id": "complexity-plans-tpcds-w15", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k tpcds", "description": "TPC-DS tests pass", "expected": "all tests pass", "item_id": "complexity-plans-tpcds-w15", "last_result": null, "last_run": null, "seq": 2}, {"command": "make complexity-check", "description": "Complexity check passes", "expected": "PASSED; these functions no longer in top-20", "item_id": "complexity-plans-tpcds-w15", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace if/elif chain in _compare_operator_properties with dispatch dict pattern", "wid": "w1"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _format_truncated_list() from _format_operator_properties in visualization.py", "wid": "w2"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _parse_operation_entry() from load_operations_catalog in primitives/catalog/loader.py", "wid": "w3"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _run_single_stream() and _aggregate_stream_results() from run_streams in runner.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _run_maintenance_loop() and _finalize_maintenance_result() from run() in maintenance_test.py", "wid": "w5"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _extract_metric() helper from run_official_benchmark in official_benchmark.py", "wid": "w6"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run complexity check and full test suite to verify no regressions", "wid": "w7"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "add functions that are reducible", "instead": "(not specified)", "why": "only genuinely irreducible cases documented above"}, {"dont": "add module-level exclusions; per-function ignores only", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "remove or modify existing approved ignores", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Follow the existing pattern in pyproject.toml [tool.benchbox.complexity] exclude_functions.\nGroup new entries by the same category headings used for existing ones.\nEach entry: \"file/path.py:function_name\", # NN - rationale\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "19 functions flagged by make complexity-check have complexity that is genuinely\nirreducible - TPC spec implementations, interactive REPL state machines, multi-platform\nAPI dispatch, AST traversal, and thread-safety patterns. These must be added to the\nexclude_functions list in pyproject.toml with rationale comments so the warning count\naccurately reflects addressable complexity only.\n\nFunctions to add:\nScore 20:\n benchbox/core/tpcdi/etl/batch.py:_execute_parallel_pipeline\n benchbox/platforms/credentials/athena.py:setup_athena_credentials\n benchbox/platforms/credentials/bigquery.py:setup_bigquery_credentials\n benchbox/platforms/dataframe/unified_frame.py:sort\n\nScore 19:\n benchbox/platforms/dataframe/unified_frame.py:_apply_datafusion_post_ops\n\nScore 18:\n benchbox/core/runner/dataframe_runner.py:_execute_dataframe_queries\n benchbox/core/tpcds/throughput_test.py:run\n benchbox/core/tpcds_obt/query_conversion.py:_rewrite_select\n benchbox/core/tpch/maintenance_test.py:_execute_rf1\n benchbox/core/tpch/power_test.py:run\n benchbox/core/tpch/throughput_test.py:run\n\nScore 17:\n benchbox/core/expected_results/registry.py:_get_benchmark_results\n benchbox/core/query_plans/parsers/datafusion.py:_convert_to_logical_operator\n benchbox/core/tpch/throughput_test.py:_execute_stream\n benchbox/core/transaction_primitives/benchmark.py:setup\n benchbox/platforms/base/adapter.py:capture_query_plan\n\nScore 16:\n benchbox/cli/commands/shell.py:_launch_sqlite_shell\n benchbox/platforms/credentials/snowflake.py:setup_snowflake_credentials\n\nScore 15:\n benchbox/cli/commands/shell.py:_launch_duckdb_shell", "id": "complexity-ignores-w14", "preserves": ["All 19 functions remain behaviorally unchanged - this is annotation only", "Existing approved ignores in exclude_functions list are untouched"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Mark required-complexity functions as approved ignores in pyproject.toml", "verifications": [{"command": "uv run -- python scripts/check_complexity.py --top 150", "description": "Complexity check passes with reduced warning count", "expected": "PASSED: all functions within complexity limit of 20; warnings count drops by ~19", "item_id": "complexity-ignores-w14", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "complexity-ignores-w14", "needs": [], "notes": "Add these with rationale comments (pattern: # NN - brief rationale):\n- _execute_parallel_pipeline: async parallel ETL orchestration with future management\n- setup_athena_credentials: interactive credential wizard; branching is UX-required\n- setup_bigquery_credentials: same as Athena\n- sort: multi-platform dispatch (Polars/PySpark/DataFusion) with different sort APIs\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add score-20 TPC/platform required ignores to pyproject.toml", "wid": "w1"}, {"evidence": null, "item_id": "complexity-ignores-w14", "needs": ["w1"], "notes": "Add these with rationale:\n- _apply_datafusion_post_ops (19): literal vs multi-agg arithmetic with distinct semantics\n- _execute_dataframe_queries (18): TPC power test methodology; phases are spec-ordered\n- tpcds/throughput_test.py:run (18): TPC-DS spec; threading is mandatory\n- _rewrite_select (18): AST rewriting with dependency ordering (role_map \u2192 column rewrite)\n- _execute_rf1 (18): TPC-H RF1 spec; SQL building depends on prior step results\n- tpch/power_test.py:run (18): TPC-H power test; spec mandates permutation order\n- tpch/throughput_test.py:run (18): TPC-H throughput; spec mandates TTT measurement\n- _get_benchmark_results (17): thread-safe single-flight cache pattern\n- _convert_to_logical_operator (17): AST node conversion with operator-specific properties\n- _execute_stream (17): TPC-H stream; 22 queries in spec permutation order\n- transaction_primitives/benchmark.py:setup (17): spec-required sequential dependency chain\n- capture_query_plan (17): multi-level filter chain (ID, iteration, sampling, EXPLAIN)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add score 17-19 TPC spec and AST ignores", "wid": "w2"}, {"evidence": null, "item_id": "complexity-ignores-w14", "needs": ["w2"], "notes": "- _launch_sqlite_shell (16): user input dispatch loop; state machine is REPL-inherent\n- setup_snowflake_credentials (16): interactive wizard; multi-step validation with dependencies\n- _launch_duckdb_shell (15): same pattern as SQLite shell\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add score 15-16 interactive REPL and credential wizard ignores", "wid": "w3"}, {"evidence": null, "item_id": "complexity-ignores-w14", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify warning count drops and complexity-check still passes", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add async to sync functions or remove async from async functions", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "change the JSON response structure of any MCP tool or resource", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "merge analytics impl functions", "instead": "(not specified)", "why": "keep them separate (each serves a different MCP tool)"}, {"dont": "rename MCP tool/resource/prompt names", "instead": "(not specified)", "why": "they're part of the public API"}], "approach": "MCP registration functions: use factory functions that close over data, returning the\ninner handler. This is the same pattern as existing MCP tools in discovery.py\n(already approved as ignore, but the factory approach is worth borrowing).\n\nAnalytics functions: three-phase pattern - load files, compute metric, format output.\nExtract each phase. Avoid shared mutable state between phases.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "8 MCP tool and analytics functions mix phase logic (file enumeration, metric extraction,\naggregation, formatting) in single long functions. Extracting phases to private helpers\nreduces complexity while keeping the MCP-registered tool function as a readable summary.\n\nTarget functions:\n benchbox/mcp/resources/registry.py:register_all_resources (20)\n benchbox/mcp/tools/analytics.py:_get_performance_trends_impl (20)\n benchbox/mcp/tools/analytics.py:_aggregate_results_impl (18)\n benchbox/mcp/tools/analytics.py:_detect_regressions_impl (17)\n benchbox/mcp/tools/analytics.py:get_query_plan (15)\n benchbox/mcp/prompts/registry.py:register_all_prompts (15)\n benchbox/mcp/tools/benchmark.py:_run_benchmark_impl (16)\n benchbox/mcp/tools/benchmark.py:_validate_config_impl (15)", "id": "complexity-mcp-analytics-w14", "preserves": ["_detect_regressions_impl: regression detection logic and threshold comparison unchanged", "_get_performance_trends_impl: trend values and structure must be numerically identical", "_validate_config_impl: validation error messages must be identical (MCP clients may parse them)", "register_all_prompts: all prompt names, descriptions, and argument specs unchanged", "register_all_resources: all MCP resource URIs and JSON shapes must be identical"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/mcp/prompts/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/resources/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/analytics.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}], "state": "done", "title": "Decompose MCP tool implementations and analytics functions (complexity batch 6)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-mcp-analytics-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'mcp or analytics or benchmark_tool' -q", "description": "MCP-specific tests pass", "expected": "all tests pass, 0 failures", "item_id": "complexity-mcp-analytics-w14", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/check_complexity.py --top 150 2>&1 | grep -E '(register_all_resources|performance_trends|aggregate_results|detect_regressions|get_query_plan|register_all_prompts|run_benchmark_impl|validate_config_impl)'", "description": "Target functions below warn threshold", "expected": "No WARN entries for the 8 target functions", "item_id": "complexity-mcp-analytics-w14", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "benchbox/mcp/resources/registry.py:34 - nested function definitions for\nlist_benchmarks, get_benchmark, list_platforms, get_platform each follow\nidentical fetch \u2192 JSON structure pattern. Extract resource factories:\n- _make_list_resource(fetch_fn, formatter_fn) \u2192 Callable\n- _make_get_resource(fetch_fn, formatter_fn) \u2192 Callable\nRegistration becomes: server.register_resource(uri, _make_list_resource(...))\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose register_all_resources: extract per-resource handler factories", "wid": "w1"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "benchbox/mcp/tools/analytics.py:435 - phases:\n- _load_result_files(result_dir, platform_filter) \u2192 list[RunResult]\n- _extract_metric(result, metric_name) \u2192 float | None\n- _compute_trends(grouped_runs, metric) \u2192 TrendResult\nMain function: load \u2192 filter \u2192 extract per run \u2192 group \u2192 compute trends.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _get_performance_trends_impl into load/filter/extract/aggregate phases", "wid": "w2"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "_aggregate_results_impl (analytics.py:566):\n- Extract _build_group_key(result, group_by) \u2192 str (3-way dispatch \u2192 dict)\n- Extract _extract_timings(result) \u2192 list[float]\n_detect_regressions_impl (analytics.py:278):\n- Extract _load_result_pair(file1, file2) \u2192 (RunResult, RunResult)\n- Extract _detect_query_regression(q1, q2, threshold) \u2192 Regression | None\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decompose _aggregate_results_impl and _detect_regressions_impl", "wid": "w3"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": [], "notes": "get_query_plan (analytics.py:101): extract _resolve_result_file(name) \u2192 Path,\n _extract_plan(result, query_id, format) \u2192 str\nregister_all_prompts (prompts/registry.py:20): similar to resources - extract\n _make_benchmark_prompt(template_fn) factory; repeated @mcp.prompt() patterns.\n_run_benchmark_impl (benchmark.py:239): extract _validate_and_build_config,\n _execute_and_format_result.\n_validate_config_impl (benchmark.py:545): extract per-phase validators as small helpers:\n _validate_platform, _validate_mode, _validate_benchmark each return error | None.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify get_query_plan, register_all_prompts, _run_benchmark_impl, _validate_config_impl", "wid": "w4"}, {"evidence": null, "item_id": "complexity-mcp-analytics-w14", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke tests and complexity check; verify all 8 functions below warn threshold", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change TPC-DS spec-mandated behavior in run_streams or maintenance_test", "instead": "only restructure the code", "why": "TPC compliance depends on exact execution semantics"}, {"dont": "create new files for extracted helpers", "instead": "keep them in the same module", "why": "these are small private functions"}], "approach": "For _compare_operator_properties, use a COMPARE_DISPATCH: dict[str, Callable] pattern\nmapping operator type strings to small comparison functions. For _format_operator_properties,\nthe _format_truncated_list(label, items, max_display=3) helper eliminates the repeated\ntruncation pattern. For run_streams, the key insight is that the sequential loop body\nand the concurrent StreamExecutor._run_single_stream do the same thing - factor into one.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Six core functions at CC=14 in query plans and TPC-DS modules can be decomposed.\n\nTarget functions:\n1. _compare_operator_properties (comparison.py:288) - replace if/elif dispatch with registry dict\n2. _format_operator_properties (visualization.py:297) - extract _format_truncated_list() helper\n3. load_operations_catalog (primitives/catalog/loader.py:24) - extract _parse_operation_entry()\n4. run_streams (tpcds/benchmark/runner.py:687) - extract sequential stream runner, share with concurrent\n5. run (tpcds/maintenance_test.py:131) - split loop from finalization logic\n6. run_official_benchmark (tpcds/official_benchmark.py:97) - extract _extract_metric() helper", "id": "complexity-plans-tpcds-w15", "preserves": ["QphDS@Size composite metric calculation in run_official_benchmark unchanged", "TPC-DS maintenance test success criteria (>=90% success rate) unchanged", "_compare_operator_properties() returns identical diff dict for all operator types", "_format_operator_properties() truncation behavior (max 3 items + '... and N more')", "load_operations_catalog() validation error messages and error types unchanged", "run_streams() concurrent and sequential execution produce identical result structures"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/comparison.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/visualization.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/maintenance_test.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/official_benchmark.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Reduce complexity in query plan and TPC-DS functions (complexity batch 10)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/query_plans/ -v", "description": "Query plan tests pass", "expected": "all tests pass", "item_id": "complexity-plans-tpcds-w15", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k tpcds", "description": "TPC-DS tests pass", "expected": "all tests pass", "item_id": "complexity-plans-tpcds-w15", "last_result": null, "last_run": null, "seq": 2}, {"command": "make complexity-check", "description": "Complexity check passes", "expected": "PASSED; these functions no longer in top-20", "item_id": "complexity-plans-tpcds-w15", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace if/elif chain in _compare_operator_properties with dispatch dict pattern", "wid": "w1"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _format_truncated_list() from _format_operator_properties in visualization.py", "wid": "w2"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _parse_operation_entry() from load_operations_catalog in primitives/catalog/loader.py", "wid": "w3"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _run_single_stream() and _aggregate_stream_results() from run_streams in runner.py", "wid": "w4"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _run_maintenance_loop() and _finalize_maintenance_result() from run() in maintenance_test.py", "wid": "w5"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _extract_metric() helper from run_official_benchmark in official_benchmark.py", "wid": "w6"}, {"evidence": null, "item_id": "complexity-plans-tpcds-w15", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run complexity check and full test suite to verify no regressions", "wid": "w7"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:00Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #385 of df-support-metadata-primitives.\nDeferred: Complexity stress testing (wide tables, large catalogs) in DataFrame mode\nReason deferred: DDL generation (CREATE TABLE with 1000 columns) is SQL-specific. DataFrame equivalent (constructing wide DataFrames) is a separate design problem.", "id": "complexity-stress-testing-wide-tables-large-catalogs-in-data", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Complexity stress testing (wide tables, large catalogs) in DataFrame mode", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add new abstraction layers in validation", "why": "extract to private methods, not new classes"}, {"INSTEAD": "(not specified)", "dont": "change the output of get_query", "why": "permutation order is spec-mandated"}, {"INSTEAD": "(not specified)", "dont": "merge functions that serve different purposes just because they look similar", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "skip the smoke test between sub-areas", "why": "catch regressions early"}], "approach": "Work sub-area by sub-area; each is mostly independent of the others.\nAfter each sub-area, run smoke tests before moving on.\n\nFor near-duplicate functions (qphh/calculate_qphh, get_query tpch/tpcds):\nextract shared logic to a _shared_*() helper; do NOT merge the public functions.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": ["complexity-dict-dispatch-w14", "complexity-field-extraction-w14"], "description": "~32 remaining functions with complexity 15-19 that don't fit cleanly into earlier batches.\nGrouped into 6 sub-areas for sequential implementation. Each sub-area shares a common\nrefactoring pattern.\n\nSub-area A - Validation functions (8 functions):\n benchbox/core/tpc_validation.py:validate (TimingValidator, 16)\n benchbox/core/tpc_validation.py:validate (MetricsValidator, 15)\n benchbox/core/tpch/reporting.py:_validate_result (15)\n benchbox/core/dataframe/validation.py:compare_dataframes (17)\n benchbox/utils/data_validation.py:validate_data_directory (16)\n benchbox/core/query_plans/insights.py:compute_complexity_score (15)\n benchbox/core/query_plans/parsers/postgresql.py:_extract_operator_specific_info (16)\n benchbox/core/query_plans/visualization.py:render_comparison (15)\n\nSub-area B - CLI output and runner helpers (7 functions):\n benchbox/cli/output.py:_display_platform_result (19)\n benchbox/core/runner/dataframe_runner.py:_get_queries_for_benchmark (19)\n benchbox/core/tpcds/parameter_extractor.py:_extract_common_params (19)\n benchbox/core/tpch_skew/benchmark.py:compare_with_uniform (19)\n benchbox/core/ai_primitives/benchmark.py:run_benchmark (17)\n benchbox/core/concurrency/analyzer.py:analyze_scaling (16)\n benchbox/core/results/loader.py:reconstruct_benchmark_results (15)\n\nSub-area C - CLI display and tuning (6 functions):\n benchbox/cli/commands/df_tuning.py:create_sample (17)\n benchbox/cli/commands/df_tuning.py:show_defaults (16)\n benchbox/cli/commands/tuning_group.py:show_defaults (16)\n benchbox/cli/commands/results.py:show_cli (16)\n benchbox/cli/commands/run.py:setup_verbose_logging (16)\n benchbox/cli/platform.py:numbered_platform_select (15)\n\nSub-area D - CLI metrics and comparison (4 functions):\n benchbox/cli/commands/calculate_qphh.py:calculate_qphh (17)\n benchbox/cli/commands/metrics.py:qphh (17)\n benchbox/cli/commands/compare.py:_format_html_comparison (17)\n benchbox/cli/exceptions.py:_handle_cli_error (15)\n\nSub-area E - Platform and utility functions (7 functions):\n benchbox/platforms/databricks/adapter.py:_auto_detect_databricks_config (16)\n benchbox/platforms/base/data_loading.py:get_data_source (16)\n benchbox/platforms/firebolt.py:__init__ (15)\n benchbox/platforms/base/adapter.py:apply_ctas_sort (15)\n benchbox/platforms/credentials/athena.py:validate_athena_credentials (15)\n benchbox/utils/file_format.py:has_trailing_delimiter (15)\n benchbox/core/ai_primitives/dataframe_operations.py:extract_entities (16)\n\nSub-area F - Benchmark query helpers and write primitives (5 functions):\n benchbox/core/tpcds/benchmark/runner.py:get_query (15)\n benchbox/core/tpch/benchmark.py:get_query (16)\n benchbox/core/write_primitives/benchmark.py:execute_operation (15)\n benchbox/core/benchmark_mixins.py:load_data_to_database (15)\n benchbox/core/dataframe/data_loader.py:convert_csv_to_parquet (15)\n benchbox/core/dataframe/data_loader.py:_convert_data (15)\n\nAlso includes release functions:\n benchbox/release/workflow.py:_should_exclude_file (17)\n benchbox/release/sync.py:cmd_status (15)\n\nTotals ~39 functions, priority ordered A\u2192F. Stop when complexity check shows\nall functions below warn threshold.", "id": "complexity-validation-misc-w14", "preserves": ["_display_platform_result: all display fields and formatting must be identical", "calculate_qphh / qphh: metric calculation must be numerically identical", "compare_dataframes: any difference between DataFrames must still raise the same errors", "get_query (tpch/tpcds): query permutation logic must be spec-compliant and unchanged", "tpc_validation validate(): validation issue counts and messages must be identical"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/calculate_qphh.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/compare.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/df_tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/results.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/tuning_group.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/output.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/platform.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ai_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ai_primitives/dataframe_operations.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_mixins.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/concurrency/analyzer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/insights.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/parsers/postgresql.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/visualization.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpc_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/parameter_extractor.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/reporting.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch_skew/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/athena.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/release/sync.py"}, {"kind": "only_modify", "path_glob": "benchbox/release/workflow.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/data_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/file_format.py"}], "state": "done", "title": "Reduce complexity in validation, analysis, and remaining scattered functions (complexity batch 7)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass after each sub-area", "expected": "all tests pass, 0 failures", "item_id": "complexity-validation-misc-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_complexity.py --top 150", "description": "Full complexity check shows all warnings resolved or approved", "expected": "PASSED: all functions within complexity limit of 20; Warnings count significantly reduced", "item_id": "complexity-validation-misc-w14", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "tpc_validation.py both validate() methods: extract check_X() private methods per\nvalidation rule. Pattern: repeated try-except-add_issue \u2192 list of validator functions.\n_validate_result (tpch/reporting.py): same - extract per-category check helpers.\ncompare_dataframes: extract _check_row_count, _check_columns, _check_values phases.\n_extract_operator_specific_info (postgresql): strategy dict keyed by operator type.\ncompute_complexity_score: extract _analyze_node recursive helper as standalone function.\nrender_comparison: extract _render_diff_section, _render_operator_counts helpers.\nvalidate_data_directory: extract _validate_with_manifest, _validate_without_manifest.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area A: Extract validate() helper methods and simplify query plan parsers", "wid": "w1"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "_display_platform_result: extract validation status lookup dict, extract _format_query_row.\n_get_queries_for_benchmark: replace benchmark if/elif with registry dict lookup.\n_extract_common_params: unify 3 extraction loops into single generic loop.\ncompare_with_uniform: extract _validate_inputs, _run_uniform_comparison helpers.\nrun_benchmark (ai_primitives): extract _check_platform, _select_queries, _run_with_cost.\nanalyze_scaling: extract single-result vs multi-result calculation to shared helper.\nreconstruct_benchmark_results: extract per-section loaders as small helpers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area B: Simplify CLI output and runner helper functions", "wid": "w2"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "create_sample (df_tuning): replace profile branches with PROFILE_CONFIG dict.\nshow_defaults (df_tuning, tuning_group): extract _add_table_rows(table, config_group).\nshow_cli (results): use early returns for default-value checks; extract display helper.\nsetup_verbose_logging: replace level if/elif with LEVEL_MAP = {\"DEBUG\": logging.DEBUG, ...}.\nnumbered_platform_select: extract _group_platforms(platforms) \u2192 (enabled, available, missing).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area C: Simplify CLI display/tuning and platform select functions", "wid": "w3"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "calculate_qphh and qphh are near-identical - extract shared _load_and_compute_qphh helper.\n_format_html_comparison: extract _html_summary_table(), _html_query_table() helpers.\n_handle_cli_error: if already done in dict-dispatch batch, skip; else do it here.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area D: Simplify CLI metrics, compare, and exception handling", "wid": "w4"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "_auto_detect_databricks_config: extract _select_warehouse(warehouses) \u2192 warehouse.\nget_data_source: unify v1/v2 manifest handling with shared file-loading helper.\nfirebolt __init__: extract _detect_deployment_mode(config) \u2192 str.\napply_ctas_sort: extract _extract_sort_columns(tuning, table) \u2192 list[str].\nvalidate_athena_credentials: extract _check_boto3_available(), _build_connection_kwargs().\nhas_trailing_delimiter: unify compressed vs uncompressed paths with shared reader.\nextract_entities (ai_primitives): extract _spacy_extraction, _regex_fallback as helpers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area E: Simplify platform adapters and utility functions", "wid": "w5"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "get_query (tpch, tpcds): validation blocks are near-identical - extract shared\n_validate_query_params(query_id, stream_id, benchmark_name) helper.\nexecute_operation (write_primitives): extract _setup_operation, _run_sql_operation.\nload_data_to_database (benchmark_mixins): extract _create_schema, _insert_rows.\nconvert_csv_to_parquet and _convert_data: extract _build_type_map(schema) \u2192 dict.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area F: Simplify benchmark query helpers and write primitives", "wid": "w6"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "_should_exclude_file: extract guard functions and apply as list.\ncmd_status: extract _print_file_section(files, header, limit) helper (3 call sites).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area G: Release functions", "wid": "w7"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Final complexity check - verify all remaining WARN functions are resolved or approved", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add new abstraction layers in validation", "instead": "(not specified)", "why": "extract to private methods, not new classes"}, {"dont": "change the output of get_query", "instead": "(not specified)", "why": "permutation order is spec-mandated"}, {"dont": "merge functions that serve different purposes just because they look similar", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "skip the smoke test between sub-areas", "instead": "(not specified)", "why": "catch regressions early"}], "approach": "Work sub-area by sub-area; each is mostly independent of the others.\nAfter each sub-area, run smoke tests before moving on.\n\nFor near-duplicate functions (qphh/calculate_qphh, get_query tpch/tpcds):\nextract shared logic to a _shared_*() helper; do NOT merge the public functions.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": ["complexity-dict-dispatch-w14", "complexity-field-extraction-w14"], "description": "~32 remaining functions with complexity 15-19 that don't fit cleanly into earlier batches.\nGrouped into 6 sub-areas for sequential implementation. Each sub-area shares a common\nrefactoring pattern.\n\nSub-area A - Validation functions (8 functions):\n benchbox/core/tpc_validation.py:validate (TimingValidator, 16)\n benchbox/core/tpc_validation.py:validate (MetricsValidator, 15)\n benchbox/core/tpch/reporting.py:_validate_result (15)\n benchbox/core/dataframe/validation.py:compare_dataframes (17)\n benchbox/utils/data_validation.py:validate_data_directory (16)\n benchbox/core/query_plans/insights.py:compute_complexity_score (15)\n benchbox/core/query_plans/parsers/postgresql.py:_extract_operator_specific_info (16)\n benchbox/core/query_plans/visualization.py:render_comparison (15)\n\nSub-area B - CLI output and runner helpers (7 functions):\n benchbox/cli/output.py:_display_platform_result (19)\n benchbox/core/runner/dataframe_runner.py:_get_queries_for_benchmark (19)\n benchbox/core/tpcds/parameter_extractor.py:_extract_common_params (19)\n benchbox/core/tpch_skew/benchmark.py:compare_with_uniform (19)\n benchbox/core/ai_primitives/benchmark.py:run_benchmark (17)\n benchbox/core/concurrency/analyzer.py:analyze_scaling (16)\n benchbox/core/results/loader.py:reconstruct_benchmark_results (15)\n\nSub-area C - CLI display and tuning (6 functions):\n benchbox/cli/commands/df_tuning.py:create_sample (17)\n benchbox/cli/commands/df_tuning.py:show_defaults (16)\n benchbox/cli/commands/tuning_group.py:show_defaults (16)\n benchbox/cli/commands/results.py:show_cli (16)\n benchbox/cli/commands/run.py:setup_verbose_logging (16)\n benchbox/cli/platform.py:numbered_platform_select (15)\n\nSub-area D - CLI metrics and comparison (4 functions):\n benchbox/cli/commands/calculate_qphh.py:calculate_qphh (17)\n benchbox/cli/commands/metrics.py:qphh (17)\n benchbox/cli/commands/compare.py:_format_html_comparison (17)\n benchbox/cli/exceptions.py:_handle_cli_error (15)\n\nSub-area E - Platform and utility functions (7 functions):\n benchbox/platforms/databricks/adapter.py:_auto_detect_databricks_config (16)\n benchbox/platforms/base/data_loading.py:get_data_source (16)\n benchbox/platforms/firebolt.py:__init__ (15)\n benchbox/platforms/base/adapter.py:apply_ctas_sort (15)\n benchbox/platforms/credentials/athena.py:validate_athena_credentials (15)\n benchbox/utils/file_format.py:has_trailing_delimiter (15)\n benchbox/core/ai_primitives/dataframe_operations.py:extract_entities (16)\n\nSub-area F - Benchmark query helpers and write primitives (5 functions):\n benchbox/core/tpcds/benchmark/runner.py:get_query (15)\n benchbox/core/tpch/benchmark.py:get_query (16)\n benchbox/core/write_primitives/benchmark.py:execute_operation (15)\n benchbox/core/benchmark_mixins.py:load_data_to_database (15)\n benchbox/core/dataframe/data_loader.py:convert_csv_to_parquet (15)\n benchbox/core/dataframe/data_loader.py:_convert_data (15)\n\nAlso includes release functions:\n benchbox/release/workflow.py:_should_exclude_file (17)\n benchbox/release/sync.py:cmd_status (15)\n\nTotals ~39 functions, priority ordered A\u2192F. Stop when complexity check shows\nall functions below warn threshold.", "id": "complexity-validation-misc-w14", "preserves": ["_display_platform_result: all display fields and formatting must be identical", "calculate_qphh / qphh: metric calculation must be numerically identical", "compare_dataframes: any difference between DataFrames must still raise the same errors", "get_query (tpch/tpcds): query permutation logic must be spec-compliant and unchanged", "tpc_validation validate(): validation issue counts and messages must be identical"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/calculate_qphh.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/compare.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/df_tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/results.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/tuning_group.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/output.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/platform.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ai_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ai_primitives/dataframe_operations.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_mixins.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/concurrency/analyzer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/insights.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/parsers/postgresql.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_plans/visualization.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpc_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/parameter_extractor.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/reporting.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch_skew/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/credentials/athena.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/release/sync.py"}, {"kind": "only_modify", "path_glob": "benchbox/release/workflow.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/data_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/file_format.py"}], "state": "done", "title": "Reduce complexity in validation, analysis, and remaining scattered functions (complexity batch 7)", "verifications": [{"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke tests pass after each sub-area", "expected": "all tests pass, 0 failures", "item_id": "complexity-validation-misc-w14", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_complexity.py --top 150", "description": "Full complexity check shows all warnings resolved or approved", "expected": "PASSED: all functions within complexity limit of 20; Warnings count significantly reduced", "item_id": "complexity-validation-misc-w14", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "tpc_validation.py both validate() methods: extract check_X() private methods per\nvalidation rule. Pattern: repeated try-except-add_issue \u2192 list of validator functions.\n_validate_result (tpch/reporting.py): same - extract per-category check helpers.\ncompare_dataframes: extract _check_row_count, _check_columns, _check_values phases.\n_extract_operator_specific_info (postgresql): strategy dict keyed by operator type.\ncompute_complexity_score: extract _analyze_node recursive helper as standalone function.\nrender_comparison: extract _render_diff_section, _render_operator_counts helpers.\nvalidate_data_directory: extract _validate_with_manifest, _validate_without_manifest.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area A: Extract validate() helper methods and simplify query plan parsers", "wid": "w1"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "_display_platform_result: extract validation status lookup dict, extract _format_query_row.\n_get_queries_for_benchmark: replace benchmark if/elif with registry dict lookup.\n_extract_common_params: unify 3 extraction loops into single generic loop.\ncompare_with_uniform: extract _validate_inputs, _run_uniform_comparison helpers.\nrun_benchmark (ai_primitives): extract _check_platform, _select_queries, _run_with_cost.\nanalyze_scaling: extract single-result vs multi-result calculation to shared helper.\nreconstruct_benchmark_results: extract per-section loaders as small helpers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area B: Simplify CLI output and runner helper functions", "wid": "w2"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "create_sample (df_tuning): replace profile branches with PROFILE_CONFIG dict.\nshow_defaults (df_tuning, tuning_group): extract _add_table_rows(table, config_group).\nshow_cli (results): use early returns for default-value checks; extract display helper.\nsetup_verbose_logging: replace level if/elif with LEVEL_MAP = {\"DEBUG\": logging.DEBUG, ...}.\nnumbered_platform_select: extract _group_platforms(platforms) \u2192 (enabled, available, missing).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area C: Simplify CLI display/tuning and platform select functions", "wid": "w3"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "calculate_qphh and qphh are near-identical - extract shared _load_and_compute_qphh helper.\n_format_html_comparison: extract _html_summary_table(), _html_query_table() helpers.\n_handle_cli_error: if already done in dict-dispatch batch, skip; else do it here.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area D: Simplify CLI metrics, compare, and exception handling", "wid": "w4"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "_auto_detect_databricks_config: extract _select_warehouse(warehouses) \u2192 warehouse.\nget_data_source: unify v1/v2 manifest handling with shared file-loading helper.\nfirebolt __init__: extract _detect_deployment_mode(config) \u2192 str.\napply_ctas_sort: extract _extract_sort_columns(tuning, table) \u2192 list[str].\nvalidate_athena_credentials: extract _check_boto3_available(), _build_connection_kwargs().\nhas_trailing_delimiter: unify compressed vs uncompressed paths with shared reader.\nextract_entities (ai_primitives): extract _spacy_extraction, _regex_fallback as helpers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area E: Simplify platform adapters and utility functions", "wid": "w5"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "get_query (tpch, tpcds): validation blocks are near-identical - extract shared\n_validate_query_params(query_id, stream_id, benchmark_name) helper.\nexecute_operation (write_primitives): extract _setup_operation, _run_sql_operation.\nload_data_to_database (benchmark_mixins): extract _create_schema, _insert_rows.\nconvert_csv_to_parquet and _convert_data: extract _build_type_map(schema) \u2192 dict.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area F: Simplify benchmark query helpers and write primitives", "wid": "w6"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": [], "notes": "_should_exclude_file: extract guard functions and apply as list.\ncmd_status: extract _print_file_section(files, header, limit) helper (3 call sites).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sub-area G: Release functions", "wid": "w7"}, {"evidence": null, "item_id": "complexity-validation-misc-w14", "needs": ["w1", "w2", "w3", "w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Final complexity check - verify all remaining WARN functions are resolved or approved", "wid": "w8"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2025-01-12T00:00:00Z", "completed_pr": null, "created_at": "2025-01-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Systematic review of human-authored documentation files in the BenchBox project\nto ensure accuracy, completeness, and quality. Each document will be\nreviewed using /docs-review, with all recommended fixes applied.\n\nFile Scope (249 total files):\n- Human-authored docs: ~170 files (IN SCOPE - full review)\n- Auto-generated _tags/: ~47 files (EXCLUDED - spot-check only)\n- Auto-generated reference/python-api/: ~30 files (EXCLUDED - verify build works)\n- Build artifacts (_build/, _static/): 2 files (EXCLUDED)\n\nThis audit addresses documentation quality issues discovered during\nthe performance.md review, which revealed critical issues including:\n- Non-existent CLI flags documented\n- Invalid import statements in code examples\n- Broken Unicode characters in headers\n- Misleading performance claims\n- Missing prerequisites sections\n\nThe goal is to bring all documentation to a consistent quality standard\nwith verified code examples, accurate CLI references, and working links.\n\nCommit Strategy: Per-phase commits for reviewability. Each phase completion\ngets its own commit with summary of changes made.", "id": "comprehensive-docs-quality-audit", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Comprehensive Documentation Quality Audit", "verifications": [], "work": [], "worktree": "docs-experience"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Comprehensive platform adapter testing implemented with extensive coverage across all database platforms\"\n- \"**Implementation Summary**:\"\n- \"**Test Categories Implemented**:\"\n- \"**Implementation Date**: August 17, 2025 (commit `9cee3d3`)\"\n- \"**Minor Remaining Gaps**:\"\n- \"**Files Affected**: All platform adapter test files (`tests/unit/platforms/test_*_adapter.py`), comprehensive test suite infrastructure\"\n- \"**Impact**: \u2705 Ensures robust platform adapter functionality across all database platforms, provides confidence for production deployments, enables comprehensive cross-platform compatibility validation with extensive error handling and optimization testing\"", "id": "comprehensive-platform-adapter-testing-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Comprehensive Platform Adapter Testing \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-09T00:00:00Z", "deferrals": [], "deps": [], "description": "Conducted a thorough review of all tests in the BenchBox test suite to ensure each test\nserves a distinct purpose and provides meaningful validation.\n\nAUDIT FINDINGS:\nThe initial estimate of ~1,362 anti-patterns was significantly overestimated. After\ndetailed analysis, the actual issues found were:\n\n1. Enum count tests: 10 instances (FIXED - replaced with structural validation)\n2. Tautological 'is not None' on imports: 19 instances (FIXED in test_circular_imports.py)\n3. Other 'is not None' assertions: ~570 instances (VALID - guard functions that return Optional)\n4. isinstance checks: ~630 instances (VALID - most have follow-up assertions)\n\nKey insight: Functions like parse_explain_output(), detect_error_type(), get_parser()\nlegitimately return None for invalid input, making 'is not None' assertions meaningful\nguards rather than tautologies.\n\nTest suite scope (actual):\n- 5,309 test methods across 353 test files\n- Unit tests: 279 files\n- Integration tests: 43 files\n- Validation/Performance/Root: 31 files", "id": "comprehensive-test-quality-audit", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Comprehensive Test Quality Audit: Ensure Meaningful and Distinct Test Coverage", "verifications": [], "work": [], "worktree": "test-quality-review"} @@ -224,100 +224,100 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Performance and Efficiency", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Successfully implemented comprehensive data compression support during generation to reduce storage requirements and improve I/O performance\"\n- \"**Implementation Summary**:\"\n- \"**Technical Features**:\"\n- \"**Performance Results** (tested on SSB date table):\"\n- \"**Files Affected**:\"\n- \"**Impact**: \u2705 Enables 6-8x reduction in storage requirements for generated data, significantly improves data transfer speeds, and reduces I/O load during benchmark setup. Particularly valuable for large scale factors and CI/CD environments with storage constraints.\"", "id": "compressed-data-generation-completed-2", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Compressed Data Generation \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Data Quality and Compatibility Issues", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"Priority: High\"\n- \"Status: Completed\"\n- \"Description: Implemented durable manifest-based data generation across benchmarks and hardened compressed output handling. No raw `.dat`/`.tbl` artifacts remain when compression is enabled; empty compressed files are disallowed; validators now prefer manifest and rebuild when missing.\"\n- \"Solution Delivered:\"\n- \"Files Affected:\"\n- \"Impact: Increases reliability of data reuse, accelerates preflight checks (manifest vs scan), and ensures consistent artifact formats for loaders and validators across all supported benchmarks.\"", "id": "compressed-data-generation-gaps-all-benchmarks", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Compressed Data Generation Gaps (all benchmarks) \u2705", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-09T00:00:00Z", "deferrals": [], "deps": ["throughput-stream-count-wiring-defect"], "description": "Documentation gaps/inconsistencies found while reviewing the concurrency\nsurface. These are low-severity on their own but compound user confusion about\na feature that (per the sibling TODOs) already behaves unexpectedly.\n\n- `--streams` is documented in docs/usage/faq.md and docs/concepts/workflow.md\n but is ABSENT from the canonical CLI reference docs/reference/cli/run.md.\n (Note: throughput-stream-count-wiring-defect must land first, since today\n --streams is effectively ignored -- do not document it as working until it\n does.)\n- Config default mismatch: docs/reference/cli/configuration.md documents\n `execution.concurrent_queries.max_concurrent` default = 2, while\n docs/advanced/power-run-concurrent-queries.md shows an example with\n max_concurrent = 4. Reconcile to the real default.\n- docs/advanced/power-run-concurrent-queries.md references\n `_project/TPC_SPECIFICATION_COMPLIANCE.md`, which does not exist. Fix or\n remove the dangling link.\n- Ensure the documented DuckDB single-connection/per-cursor caveat and the\n TPC-H(0.99)/TPC-DS(0.70) success-gate divergence stay in sync with whatever\n the code changes in the sibling TODOs settle on.", "id": "concurrency-docs-alignment", "preserves": ["Existing accurate caveats (DuckDB cursor sharing, success-gate divergence) remain documented."], "prior_art": [{"concept": "docs/advanced/power-run-concurrent-queries.md \u2014 the guide with the 4-vs-2 example and the dangling link.", "decision": "reuse", "path": "docs/advanced/power-run-concurrent-queries.md"}, {"concept": "docs/benchmarks/tpc-h.md / tpc-ds.md \u2014 where the DuckDB and permutation caveats live.", "decision": "reuse", "path": "docs/benchmarks/tpc-h.md"}, {"concept": "docs/reference/cli/configuration.md \u2014 the max_concurrent default of record.", "decision": "reuse", "path": "docs/reference/cli/configuration.md"}, {"concept": "docs/reference/cli/run.md \u2014 canonical CLI reference missing --streams.", "decision": "reuse", "path": "docs/reference/cli/run.md"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "docs/"}], "state": "done", "title": "Align concurrency/throughput documentation with actual behavior", "verifications": [{"command": "make docs-linkcheck", "description": "Docs link-check passes and no dangling internal references remain", "expected": "no broken internal links", "item_id": "concurrency-docs-alignment", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "concurrency-docs-alignment", "needs": [], "notes": "Verified the real default in code: benchbox/utils/config_helpers.py:78\nand benchbox/cli/config.py:50/280 all default\nexecution.concurrent_queries.max_concurrent to 2, matching\ndocs/reference/cli/configuration.md:297 (already correct). The\nmismatch was entirely in\ndocs/advanced/power-run-concurrent-queries.md's illustrative YAML\nexample, which showed max_concurrent: 4 with nothing marking it as a\nnon-default example. Added an explicit callout above the block\n(\"illustrative example, not the shipped defaults\") linking to\nConfiguration reference -> `execution`, plus inline \"(default: N)\"\ncomments on every field in that YAML block. Python-code examples\nfurther down that pass max_concurrent=4/level as explicit function\narguments were left as-is -- self-evidently overrides, not claimed\ndefaults.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile max_concurrent default across configuration.md and the guide", "wid": "w1"}, {"evidence": null, "item_id": "concurrency-docs-alignment", "needs": [], "notes": "That file never existed in docs/ or _project/. Replaced the dangling\nreference in docs/advanced/power-run-concurrent-queries.md with links\nto the docs that actually cover TPC compliance/permutations:\ndocs/guides/tpc/tpc-h-official-guide.md#tpc-h-specification-compliance,\ndocs/guides/tpc/tpc-ds-official-guide.md, and the shared\ndocs/guides/tpc/tpc-patterns-usage.md. All three targets confirmed to\nexist; scripts/check_doc_relative_links.py baseline unchanged (43, no\nnew broken links) after the edit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix/remove the dangling _project/TPC_SPECIFICATION_COMPLIANCE.md reference", "wid": "w2"}, {"evidence": null, "item_id": "concurrency-docs-alignment", "needs": [], "notes": "throughput-stream-count-wiring-defect (sibling #1) is merged to\ndevelop. Verified the actual CLI surface directly (`benchbox run\n--help-topic all` and `benchbox run-official --help`): plain `run`\n(including `run --official`) has NO `--streams`/`--concurrency`\noption at all -- confirmed live: `benchbox run --platform duckdb\n--benchmark tpch --scale 1 --phases throughput --streams 4` errors\nwith \"Error: No such option: --streams\". Only the deprecated\n`run-official` command exposes `--streams INTEGER` (required when\n--phases includes throughput). Since docs/reference/cli/run.md is\nexclusively the `run` command's reference, documenting --streams as\na `run` option would itself be inaccurate. Added a note under the\n`throughput` phase bullet stating plain `run` has no such flag, plus\na full \"Deprecated: `run-official`\" section documenting its real\noption list (including --streams) accurately.\n\nExtended fix (same \"don't document --streams as working where it\nisn't\" goal, and directly named in this TODO's own description):\ndocs/usage/faq.md documented `benchbox run ... --streams 4` (the\nnon-official command) as working -- verified it errors identically.\nFixed to the working `run-official ... --streams 4` form.\ndocs/concepts/workflow.md's compliance-workflow example used\n`run-official ... --test-type power/throughput`, but `--test-type`\nis not a real run-official option (the real option is --phases) --\nverified against run_official.py's click options and a live\n`run-official tpch --platform duckdb --scale 1 --phases power --seed\n42 --output ` invocation (parsed and executed with no \"no such\noption\" error). Fixed to --phases; both files now cross-link the new\nCLI reference section.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document --streams in run.md once throughput-stream-count-wiring-defect lands", "wid": "w3"}, {"evidence": null, "item_id": "concurrency-docs-alignment", "needs": [], "notes": "DEFERRED per explicit handoff instructions for this run: sibling\nTODOs tpcds-throughput-permutation-compliance and\nthroughput-timeout-leak-and-success-gates own the code behind the\nDuckDB single-connection/per-cursor caveat and the TPC-H(0.99)/\nTPC-DS(0.70) success-gate divergence, and are updating\ndocs/benchmarks/tpc-ds.md and\ndocs/development/throughput-result-alignment.md in parallel in\nsibling worktrees. This run did not touch either file\n(conflict-avoidance) and did not re-derive what those caveats should\nsay -- that decision belongs to whichever of those two TODOs lands\nthe underlying behavior change. Do not reopen this item without\nfirst checking whether those TODOs already reconciled the caveats.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep the DuckDB single-connection/success-gate divergence caveats in sync with sibling TODOs", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "delete code that a passing test depends on without first proving the test is asserting fake success (the stub case).", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-09T00:00:00Z", "deferrals": [], "deps": [], "description": "There are three overlapping concurrent-stream executors plus a dead stub. This\nfragmentation is a maintenance and correctness hazard: fixes to one path (the\nproduction one) do not reach the others, and it is not obvious which is live.\n\n- LIVE: benchbox/core/throughput/runner.py:StreamRunner \u2014 the real executor\n used by the CLI/adapter throughput path.\n- EFFECTIVELY DEAD: benchbox/utils/execution_manager.py:ConcurrentQueryExecutor\n (+ PowerRunExecutor) \u2014 config-driven, reachable only through the stubbed\n run_concurrent_streams paths below.\n- UNUSED EXTERNALLY: benchbox/core/tpc_patterns.py:StreamExecutor \u2014 a full\n concurrent framework instantiated only inside its own module; external\n consumers import only generate_official_benchmark_audit_trail from it.\n- NON-EXECUTING STUB: benchbox/core/tpch/streams.py:TPCHStreamRunner.run_stream\n (~lines 431-454) counts `-- Query` comment lines and marks every query\n successful WITHOUT executing SQL. Any path through\n run_concurrent_streams (tpch/streams.py, and the parallel\n tpcds/benchmark/runner.py:988-1011 path) measures nothing real. A test or\n caller that reaches this reports fake success.\n\nFix: make StreamRunner the single throughput executor; delete or clearly\nquarantine the dead/unused executors; and either remove the non-executing\nstub or make it raise NotImplementedError so it cannot silently report fake\nsuccess.", "id": "concurrency-executor-consolidation", "preserves": ["The production StreamRunner throughput path behavior.", "generate_official_benchmark_audit_trail (the one thing external code imports from tpc_patterns)."], "prior_art": [{"concept": "_project/DONE/unknown/planning/tpc-h-stream-processing-implementation.yaml \u2014 the completed item that originally filled these stubs; context for why they exist.", "decision": "reuse", "path": "_project/DONE/unknown/planning/tpc-h-stream-processing-implementation.yaml"}, {"concept": "benchbox/core/throughput/runner.py:StreamRunner \u2014 the canonical executor to standardize on.", "decision": "reuse", "path": "benchbox/core/throughput/runner.py:StreamRunner"}, {"concept": "benchbox/core/tpc_patterns.py:StreamExecutor \u2014 unused framework to retire or fold in.", "decision": "reuse", "path": "benchbox/core/tpc_patterns.py:StreamExecutor"}, {"concept": "benchbox/utils/execution_manager.py:ConcurrentQueryExecutor \u2014 legacy path to retire.", "decision": "reuse", "path": "benchbox/utils/execution_manager.py:ConcurrentQueryExecutor"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpc_patterns.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/streams.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/execution_manager.py"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Consolidate overlapping concurrent executors and remove the non-executing stream stub", "verifications": [{"command": "uv run -- python -m pytest tests/unit -k 'stream' -v", "description": "No production/test path reaches a non-executing stream stub", "expected": "stream tests pass against real or explicitly-not-implemented paths", "item_id": "concurrency-executor-consolidation", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": [], "notes": "grep + call-graph; captured to /tmp/concurrency-consolidation-w0.log. StreamRunner is the only live executor; ConcurrentQueryExecutor/PowerRunExecutor and the tpc_patterns.py framework have zero production callers; TPCHStreamRunner.run_stream and TPCDSBenchmark.run_streams/_execute_single_stream are non-executing comment-counting stubs reachable only from tests.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Map every caller of each executor and the stub; confirm which are truly reachable in production/tests", "wid": "w0"}, {"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": ["w0"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Neutralize the non-executing run_stream stub (raise NotImplementedError or route to real execution)", "wid": "w1"}, {"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": ["w0"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire ConcurrentQueryExecutor / PowerRunExecutor and StreamExecutor, or fold their unique value into StreamRunner", "wid": "w2"}, {"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests/docs that reference retired paths", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "delete code that a passing test depends on without first proving the test is asserting fake success (the stub case).", "instead": "(not specified)", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-09T00:00:00Z", "deferrals": [], "deps": [], "description": "There are three overlapping concurrent-stream executors plus a dead stub. This\nfragmentation is a maintenance and correctness hazard: fixes to one path (the\nproduction one) do not reach the others, and it is not obvious which is live.\n\n- LIVE: benchbox/core/throughput/runner.py:StreamRunner \u2014 the real executor\n used by the CLI/adapter throughput path.\n- EFFECTIVELY DEAD: benchbox/utils/execution_manager.py:ConcurrentQueryExecutor\n (+ PowerRunExecutor) \u2014 config-driven, reachable only through the stubbed\n run_concurrent_streams paths below.\n- UNUSED EXTERNALLY: benchbox/core/tpc_patterns.py:StreamExecutor \u2014 a full\n concurrent framework instantiated only inside its own module; external\n consumers import only generate_official_benchmark_audit_trail from it.\n- NON-EXECUTING STUB: benchbox/core/tpch/streams.py:TPCHStreamRunner.run_stream\n (~lines 431-454) counts `-- Query` comment lines and marks every query\n successful WITHOUT executing SQL. Any path through\n run_concurrent_streams (tpch/streams.py, and the parallel\n tpcds/benchmark/runner.py:988-1011 path) measures nothing real. A test or\n caller that reaches this reports fake success.\n\nFix: make StreamRunner the single throughput executor; delete or clearly\nquarantine the dead/unused executors; and either remove the non-executing\nstub or make it raise NotImplementedError so it cannot silently report fake\nsuccess.", "id": "concurrency-executor-consolidation", "preserves": ["The production StreamRunner throughput path behavior.", "generate_official_benchmark_audit_trail (the one thing external code imports from tpc_patterns)."], "prior_art": [{"concept": "_project/DONE/unknown/planning/tpc-h-stream-processing-implementation.yaml \u2014 the completed item that originally filled these stubs; context for why they exist.", "decision": "reuse", "path": "_project/DONE/unknown/planning/tpc-h-stream-processing-implementation.yaml"}, {"concept": "benchbox/core/throughput/runner.py:StreamRunner \u2014 the canonical executor to standardize on.", "decision": "reuse", "path": "benchbox/core/throughput/runner.py:StreamRunner"}, {"concept": "benchbox/core/tpc_patterns.py:StreamExecutor \u2014 unused framework to retire or fold in.", "decision": "reuse", "path": "benchbox/core/tpc_patterns.py:StreamExecutor"}, {"concept": "benchbox/utils/execution_manager.py:ConcurrentQueryExecutor \u2014 legacy path to retire.", "decision": "reuse", "path": "benchbox/utils/execution_manager.py:ConcurrentQueryExecutor"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpc_patterns.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/benchmark/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/streams.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/execution_manager.py"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Consolidate overlapping concurrent executors and remove the non-executing stream stub", "verifications": [{"command": "uv run -- python -m pytest tests/unit -k 'stream' -v", "description": "No production/test path reaches a non-executing stream stub", "expected": "stream tests pass against real or explicitly-not-implemented paths", "item_id": "concurrency-executor-consolidation", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": [], "notes": "grep + call-graph; captured to /tmp/concurrency-consolidation-w0.log. StreamRunner is the only live executor; ConcurrentQueryExecutor/PowerRunExecutor and the tpc_patterns.py framework have zero production callers; TPCHStreamRunner.run_stream and TPCDSBenchmark.run_streams/_execute_single_stream are non-executing comment-counting stubs reachable only from tests.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Map every caller of each executor and the stub; confirm which are truly reachable in production/tests", "wid": "w0"}, {"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": ["w0"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Neutralize the non-executing run_stream stub (raise NotImplementedError or route to real execution)", "wid": "w1"}, {"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": ["w0"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire ConcurrentQueryExecutor / PowerRunExecutor and StreamExecutor, or fold their unique value into StreamRunner", "wid": "w2"}, {"evidence": null, "item_id": "concurrency-executor-consolidation", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests/docs that reference retired paths", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Enterprise Operations", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Multi-threaded query execution testing with queueing analysis and\nresource monitoring.", "id": "concurrency-testing-framework", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Concurrency Testing Framework", "verifications": [], "work": [], "worktree": "enterprise-ops-concurrency"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "New Feature Requests", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Add comprehensive concurrent query execution system for throughput testing and scalability analysis\"\n- \"**Tasks**:\"", "id": "concurrent-queries-configuration-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Concurrent Queries Configuration \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (Should-Fix Before Release)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Added driver version fields to configs/results, created a runtime version manager with optional auto-install, and updated CLI/reporting pipelines to record both requested and resolved driver builds.", "id": "configurable-platform-version-pinning", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Configurable Platform Version Pinning", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Bug Fixes", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Fixed \\\"multiple values for keyword argument 'connection'\\\" errors when executing TPC power/throughput/maintenance tests through CLI\"\n- \"**Context**: The CLI orchestrator was passing connection parameters both explicitly (connection=connection) and within run_config dictionary, causing Python to receive duplicate keyword arguments when calling platform adapter test methods\"\n- \"**Root Cause**: In CLI orchestrator methods (`_execute_power_test`, `_execute_throughput_test`, `_execute_maintenance_test`), calls like `platform_adapter.run_power_test(benchmark, connection=connection, **run_config)` failed when run_config also contained a 'connection' key\"\n- \"**Tasks Completed**:\"\n- \"**Files Affected**:\"\n- \"**Impact**: \u2705 **RESOLVED** - TPC test methods (power, throughput, maintenance) now execute successfully without parameter conflicts, ensuring reliable benchmark execution through CLI interface\"\n- \"**Validation**: SSB power test execution progresses past connection error to actual benchmark execution phase\"", "id": "connection-parameter-conflicts-in-tpc-test-methods", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Connection Parameter Conflicts in TPC Test Methods", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:40Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #215 of move-duckdb-to-optional-extras.\nDeferred: Consider whether other core drivers (polars, clickhouse-connect, psycopg2-binary) should similarly be made optional\nReason deferred: Out of scope for this item - focus only on duckdb, which is the platform with the most active version-testing use case", "id": "consider-whether-other-core-drivers-polars-clickhouse-connec", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Consider whether other core drivers (polars, clickhouse-connect, psycopg2-binary) should similarly be made optional", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-27T00:00:00Z", "completed_pr": null, "created_at": "2026-01-27T00:00:00Z", "deferrals": [], "deps": [], "description": "The codebase has ~15 locations that duplicate benchmark name variant detection\nusing `or` chains like `\"tpch\" in x or \"tpc-h\" in x`. These should use the\ncentralized `normalize_benchmark_id()` utility introduced in commit 8117828c.\n\nThis is a code hygiene issue, not a bug. The current code works correctly but:\n1. Duplicates logic that could drift out of sync\n2. Is harder to extend when adding new benchmarks\n3. Inconsistent with the now-canonical normalize_benchmark_id() pattern\n\nNote: Some locations already use the cleaner tuple-membership pattern\n`benchmark.lower() in (\"tpch\", \"tpc-h\")` which is acceptable and performant.", "id": "consolidate-benchmark-name-detection", "preserves": [], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Consolidate benchmark name detection to use normalize_benchmark_id()", "verifications": [], "work": [], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "only change internal implementations", "dont": "change public API signatures during consolidation", "why": "downstream code and tests depend on them"}, {"INSTEAD": "always read the actual implementations to confirm true duplication", "dont": "consolidate code that only looks structurally similar", "why": "AST hashing ignores identifiers"}, {"INSTEAD": "(not specified)", "dont": "consolidate platform adapter interface methods (from_config, apply_constraint_configuration)", "why": "these are intended per-platform implementations even when structurally identical"}, {"INSTEAD": "use mixins or standalone utility functions instead", "dont": "create deep inheritance hierarchies", "why": "the adapter hierarchy is already 3 levels deep"}], "approach": "Work in phases, each independently testable and committable:\n\nPhase 1 (w1-w2): Extract shared infrastructure for the primitives fork. These are the\nhighest-value targets with byte-for-byte identical code. Create new shared modules,\nupdate imports, verify with existing tests.\n\nPhase 2 (w3-w5): Consolidate benchmark base class methods and query infrastructure.\nThese are lower-risk refactors touching well-tested code paths.\n\nPhase 3 (w6-w8): Minor utility extraction, ignore-list configuration, and threshold\nadjustment. This is the cleanup phase.\n\nFor each consolidation:\n1. Read both/all copies side-by-side to confirm true equivalence\n2. Create the shared module with the canonical implementation\n3. Update all call sites to use the shared version\n4. Run targeted tests for affected modules\n5. Run make duplicate-check-verbose to confirm line count drop\n6. Commit\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "consolidate-duplicate-code", "id": 65, "reason": "The two benchmarks share generators and catalogs but have distinct query sets and validation logic. Full unification requires deeper analysis of behavioral differences.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: transaction_primitives + write_primitives already share generators/catalogs (consolidate-duplicate-code done); full module unification of two benchmarks with intentionally distinct query sets/validation is speculative, unscoped, and the note itself hedges it needs 'deeper analysis'.", "summary": "Full transaction_primitives/write_primitives module unification"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "consolidate-duplicate-code", "id": 66, "reason": "These are similar (~79 lines each) but have platform-specific error handling and cursor semantics. Consolidation requires careful testing against live platforms.", "resolution": "promoted", "resolved_item": "consolidate-firebolt-presto-trino-execute-query-implementati", "resolved_reason": null, "summary": "Consolidate Firebolt/Presto/Trino execute_query implementations"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "consolidate-duplicate-code", "id": 67, "reason": "Protocol methods in protocols.py have structural duplication but serve as the interface contract. Changing them risks breaking the protocol hierarchy.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: protocols.py DataFrame protocol methods are the interface contract; structural similarity across Protocol declarations is by-design. consolidate-duplicate-code (DONE) correctly excluded them. Consolidation is inappropriate, not deferred work.", "summary": "Unify DataFrame protocol abstract method definitions"}], "deps": [], "description": "The `make duplicate-check-verbose` tool reports 440 duplicate function groups totalling\n~17,595 duplicated lines (threshold: 20,000). After careful analysis of the top 30 groups,\n13 groups are genuine parallel implementations that should be consolidated, while 17 are\nintended boilerplate (interface implementations, structural patterns, or facade delegations)\nthat should be marked for ignore.\n\nThis work reduces the duplicated-line count by an estimated 2,500-3,000 lines through\ntargeted consolidation of the highest-value groups, while configuring the duplicate checker\nto suppress known-intentional patterns so future regressions are caught at a lower threshold.\n\nKey consolidation targets (by estimated line savings):\n- transaction_primitives/write_primitives generator fork (~530 lines)\n- catalog loaders (transaction/write/metadata/read primitives) (~310 lines)\n- benchmark base class (generate_data, load_data_to_database) (~210 lines)\n- query manager base class (get_query, _load_queries pattern) (~140 lines)\n- DataFrame query registry list_*_queries functions (~120 lines)\n- platform adapter apply_constraint_configuration / apply_unified_tuning (~360 lines via ignore)", "id": "consolidate-duplicate-code", "preserves": ["All benchmark tests pass (uv run -- python -m pytest tests/unit/ -v --timeout=120)", "All existing platform adapter tests pass (uv run -- python -m pytest tests/unit/platforms/ -v)", "Catalog loader tests pass (uv run -- python -m pytest tests/ -k 'catalog or loader' -v)", "DataFrame query registry tests pass (uv run -- python -m pytest tests/ -k 'registry or dataframe_quer' -v)", "DuckDB TPC-H smoke test passes (benchbox run --platform duckdb --benchmark tpch --scale 0.01)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/protocols.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/*/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/amplab/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/amplab/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_mixins.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/clickbench/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/query_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/h2odb/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_analytics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_etl.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/generator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/utils/tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/generator.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/check_duplicate_code.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Consolidate duplicate code identified by AST structural clone detection", "verifications": [{"command": "uv run -- python -m pytest tests/unit/ -v --timeout=120", "description": "All unit tests pass after each work unit", "expected": "all tests pass, 0 failures", "item_id": "consolidate-duplicate-code", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose 2>&1 | tail -1", "description": "Duplicate line count reduced below 15500", "expected": "PASS: duplicated lines within threshold (value below 15500)", "item_id": "consolidate-duplicate-code", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke test passes", "expected": "all fast-marked tests pass", "item_id": "consolidate-duplicate-code", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create benchbox/core/primitives/generator_base.py with shared methods from\ntransaction_primitives/generator.py and write_primitives/generator.py.\nBoth generators inherit from this base. Methods to extract:\n- _generate_special_test_files (~159 lines, byte-for-byte identical)\n- generate_bulk_load_files (~107 lines, byte-for-byte identical)\nEstimated savings: ~530 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create shared primitives generator base class consolidating _generate_special_test_files and generate_bulk_load_files from transaction_primitives and write_primitives generators", "wid": "w1"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": ["w1"], "notes": "Create benchbox/core/primitives/catalog/loader.py with generic\nload_operations_catalog() and load_query_catalog() functions.\nThe 4 existing loaders become thin wrappers passing their error class and package.\nUnify identical dataclass definitions (ValidationQuery, WriteOperation,\nMetadataQuery, PrimitiveQuery) where field structures match.\nEstimated savings: ~310 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create shared catalog loader for transaction/write/metadata/read primitives catalog YAML parsing", "wid": "w2"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create a DataGenerationMixin in benchbox/core/benchmark_mixins.py (standalone,\nnot modifying the existing base class hierarchy):\n- generate_data(): 5 copies differ only in TABLES reference.\n Mixin provides the method; each benchmark defines _get_table_schema() -> dict.\n- load_data_to_database(): 3 copies (clickbench, read_primitives, ssb).\n Parameterize the schema dict and provide as mixin method.\nEach benchmark class adds the mixin to its bases and implements _get_table_schema().\nEstimated savings: ~210 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract common benchmark mixin with generate_data() and load_data_to_database()", "wid": "w3"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create benchbox/core/query_manager.py (or add to existing base) with:\n- get_query(query_id, params) template method\n- Abstract _generate_default_params(query_id)\nSubclass in SSB, AMPLab, TPC-DI sub-managers (4 copies eliminated).\nEstimated savings: ~140 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ParameterizedQueryManager base class with shared get_query() method", "wid": "w4"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "The filtering logic in list_*_queries (family/category filter over get_all_queries)\nis duplicated 9 times across dataframe_queries/registry.py files.\nMove this logic to QueryRegistry.list_queries(family, category) method.\nEach registry file's list_*_queries becomes a one-liner delegation.\nEstimated savings: ~120 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add list_queries() method to QueryRegistry class, simplify 9 registry wrapper functions", "wid": "w5"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create a small utility in benchbox/core/utils/tuning.py:\n def extract_constraint_flags(tuning_config) -> tuple[bool, bool]\nUsed by 5 benchmark get_create_tables_sql implementations.\nMinor savings (~50 lines) but reduces a common pattern to one line.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract tuning config unpacking utility for get_create_tables_sql methods", "wid": "w6"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": "After consolidation work is complete, update pyproject.toml\n[tool.benchbox.duplicates] section or the check_duplicate_code.py script\nto suppress known-intentional patterns:\n- Platform adapter interface methods (from_config, apply_constraint_configuration,\n apply_unified_tuning, supports_tuning_type)\n- DataFrame adapter cast_date/cast_string implementations\n- Facade/delegation one-liners\n- Structural patterns (_load_queries, _generate_default_params)\nLower the threshold from 20,000 to catch future regressions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Configure duplicate checker ignore patterns for intended boilerplate groups", "wid": "w7"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": ["w7"], "notes": "Run: uv run -- python -m pytest -x -v\nRun: make duplicate-check-verbose\nVerify duplicated lines dropped by ~2,500-3,000 from baseline 17,595.\nUpdate threshold in pyproject.toml to new level + 10% buffer.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite, update duplicate threshold, verify line count reduction", "wid": "w8"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "only change the internal implementation", "dont": "change function signatures", "why": "callers depend on the current API"}, {"INSTEAD": "follow the is_tpc_format() function style", "dont": "create a generic format detector class", "why": "simple predicate functions match the existing codebase pattern"}], "approach": null, "blocked_reason": null, "category": "Code Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-01-27T00:00:00Z", "deferrals": [], "deps": [], "description": "The codebase has inconsistent file format detection patterns that fail to handle:\n1. Compressed files: `data.parquet.zst`, `data.csv.gz`\n2. Sharded files: `data.parquet.1`, `data.csv.2`, `data.tbl.1.zst`\n\nSome locations use `is_tpc_format()` which correctly handles compression, but then\ncheck `.parquet` or `.csv` directly on the suffix which breaks for compressed/sharded files.\n\nThe existing `is_tpc_format()` utility correctly iterates through suffixes skipping\ncompression extensions. We need analogous utilities for parquet and csv detection,\nthen update all locations using direct suffix checks.\n\nThis is a follow-up to the TPC delimiter detection consolidation work.", "id": "consolidate-file-format-detection", "preserves": ["All existing platform tests continue to pass", "Existing file format detection behavior for uncompressed files unchanged", "is_tpc_format() API and return values unchanged"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}], "state": "done", "title": "Consolidate file format detection to handle compressed and sharded files", "verifications": [{"command": "uv run -- python -m pytest -m fast --tb=short -q", "description": "All fast tests pass after refactoring", "expected": "all tests pass, 0 failures", "item_id": "consolidate-file-format-detection", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run ruff check benchbox/", "description": "Lint passes", "expected": "no errors", "item_id": "consolidate-file-format-detection", "last_result": null, "last_run": null, "seq": 2}], "work": [], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change public API signatures during consolidation", "instead": "only change internal implementations", "why": "downstream code and tests depend on them"}, {"dont": "consolidate code that only looks structurally similar", "instead": "always read the actual implementations to confirm true duplication", "why": "AST hashing ignores identifiers"}, {"dont": "consolidate platform adapter interface methods (from_config, apply_constraint_configuration)", "instead": "(not specified)", "why": "these are intended per-platform implementations even when structurally identical"}, {"dont": "create deep inheritance hierarchies", "instead": "use mixins or standalone utility functions instead", "why": "the adapter hierarchy is already 3 levels deep"}], "approach": "Work in phases, each independently testable and committable:\n\nPhase 1 (w1-w2): Extract shared infrastructure for the primitives fork. These are the\nhighest-value targets with byte-for-byte identical code. Create new shared modules,\nupdate imports, verify with existing tests.\n\nPhase 2 (w3-w5): Consolidate benchmark base class methods and query infrastructure.\nThese are lower-risk refactors touching well-tested code paths.\n\nPhase 3 (w6-w8): Minor utility extraction, ignore-list configuration, and threshold\nadjustment. This is the cleanup phase.\n\nFor each consolidation:\n1. Read both/all copies side-by-side to confirm true equivalence\n2. Create the shared module with the canonical implementation\n3. Update all call sites to use the shared version\n4. Run targeted tests for affected modules\n5. Run make duplicate-check-verbose to confirm line count drop\n6. Commit\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "consolidate-duplicate-code", "id": 65, "reason": "The two benchmarks share generators and catalogs but have distinct query sets and validation logic. Full unification requires deeper analysis of behavioral differences.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: transaction_primitives + write_primitives already share generators/catalogs (consolidate-duplicate-code done); full module unification of two benchmarks with intentionally distinct query sets/validation is speculative, unscoped, and the note itself hedges it needs 'deeper analysis'.", "summary": "Full transaction_primitives/write_primitives module unification"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "consolidate-duplicate-code", "id": 66, "reason": "These are similar (~79 lines each) but have platform-specific error handling and cursor semantics. Consolidation requires careful testing against live platforms.", "resolution": "promoted", "resolved_item": "consolidate-firebolt-presto-trino-execute-query-implementati", "resolved_reason": null, "summary": "Consolidate Firebolt/Presto/Trino execute_query implementations"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "consolidate-duplicate-code", "id": 67, "reason": "Protocol methods in protocols.py have structural duplication but serve as the interface contract. Changing them risks breaking the protocol hierarchy.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: protocols.py DataFrame protocol methods are the interface contract; structural similarity across Protocol declarations is by-design. consolidate-duplicate-code (DONE) correctly excluded them. Consolidation is inappropriate, not deferred work.", "summary": "Unify DataFrame protocol abstract method definitions"}], "deps": [], "description": "The `make duplicate-check-verbose` tool reports 440 duplicate function groups totalling\n~17,595 duplicated lines (threshold: 20,000). After careful analysis of the top 30 groups,\n13 groups are genuine parallel implementations that should be consolidated, while 17 are\nintended boilerplate (interface implementations, structural patterns, or facade delegations)\nthat should be marked for ignore.\n\nThis work reduces the duplicated-line count by an estimated 2,500-3,000 lines through\ntargeted consolidation of the highest-value groups, while configuring the duplicate checker\nto suppress known-intentional patterns so future regressions are caught at a lower threshold.\n\nKey consolidation targets (by estimated line savings):\n- transaction_primitives/write_primitives generator fork (~530 lines)\n- catalog loaders (transaction/write/metadata/read primitives) (~310 lines)\n- benchmark base class (generate_data, load_data_to_database) (~210 lines)\n- query manager base class (get_query, _load_queries pattern) (~140 lines)\n- DataFrame query registry list_*_queries functions (~120 lines)\n- platform adapter apply_constraint_configuration / apply_unified_tuning (~360 lines via ignore)", "id": "consolidate-duplicate-code", "preserves": ["All benchmark tests pass (uv run -- python -m pytest tests/unit/ -v --timeout=120)", "All existing platform adapter tests pass (uv run -- python -m pytest tests/unit/platforms/ -v)", "Catalog loader tests pass (uv run -- python -m pytest tests/ -k 'catalog or loader' -v)", "DataFrame query registry tests pass (uv run -- python -m pytest tests/ -k 'registry or dataframe_quer' -v)", "DuckDB TPC-H smoke test passes (benchbox run --platform duckdb --benchmark tpch --scale 0.01)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/protocols.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/*/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/amplab/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/amplab/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_mixins.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/clickbench/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/query_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/h2odb/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_analytics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_etl.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/generator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/utils/tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/catalog/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/generator.py"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/check_duplicate_code.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Consolidate duplicate code identified by AST structural clone detection", "verifications": [{"command": "uv run -- python -m pytest tests/unit/ -v --timeout=120", "description": "All unit tests pass after each work unit", "expected": "all tests pass, 0 failures", "item_id": "consolidate-duplicate-code", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose 2>&1 | tail -1", "description": "Duplicate line count reduced below 15500", "expected": "PASS: duplicated lines within threshold (value below 15500)", "item_id": "consolidate-duplicate-code", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke test passes", "expected": "all fast-marked tests pass", "item_id": "consolidate-duplicate-code", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create benchbox/core/primitives/generator_base.py with shared methods from\ntransaction_primitives/generator.py and write_primitives/generator.py.\nBoth generators inherit from this base. Methods to extract:\n- _generate_special_test_files (~159 lines, byte-for-byte identical)\n- generate_bulk_load_files (~107 lines, byte-for-byte identical)\nEstimated savings: ~530 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create shared primitives generator base class consolidating _generate_special_test_files and generate_bulk_load_files from transaction_primitives and write_primitives generators", "wid": "w1"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": ["w1"], "notes": "Create benchbox/core/primitives/catalog/loader.py with generic\nload_operations_catalog() and load_query_catalog() functions.\nThe 4 existing loaders become thin wrappers passing their error class and package.\nUnify identical dataclass definitions (ValidationQuery, WriteOperation,\nMetadataQuery, PrimitiveQuery) where field structures match.\nEstimated savings: ~310 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create shared catalog loader for transaction/write/metadata/read primitives catalog YAML parsing", "wid": "w2"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create a DataGenerationMixin in benchbox/core/benchmark_mixins.py (standalone,\nnot modifying the existing base class hierarchy):\n- generate_data(): 5 copies differ only in TABLES reference.\n Mixin provides the method; each benchmark defines _get_table_schema() -> dict.\n- load_data_to_database(): 3 copies (clickbench, read_primitives, ssb).\n Parameterize the schema dict and provide as mixin method.\nEach benchmark class adds the mixin to its bases and implements _get_table_schema().\nEstimated savings: ~210 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract common benchmark mixin with generate_data() and load_data_to_database()", "wid": "w3"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create benchbox/core/query_manager.py (or add to existing base) with:\n- get_query(query_id, params) template method\n- Abstract _generate_default_params(query_id)\nSubclass in SSB, AMPLab, TPC-DI sub-managers (4 copies eliminated).\nEstimated savings: ~140 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create ParameterizedQueryManager base class with shared get_query() method", "wid": "w4"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "The filtering logic in list_*_queries (family/category filter over get_all_queries)\nis duplicated 9 times across dataframe_queries/registry.py files.\nMove this logic to QueryRegistry.list_queries(family, category) method.\nEach registry file's list_*_queries becomes a one-liner delegation.\nEstimated savings: ~120 duplicated lines.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add list_queries() method to QueryRegistry class, simplify 9 registry wrapper functions", "wid": "w5"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": [], "notes": "Create a small utility in benchbox/core/utils/tuning.py:\n def extract_constraint_flags(tuning_config) -> tuple[bool, bool]\nUsed by 5 benchmark get_create_tables_sql implementations.\nMinor savings (~50 lines) but reduces a common pattern to one line.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract tuning config unpacking utility for get_create_tables_sql methods", "wid": "w6"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": "After consolidation work is complete, update pyproject.toml\n[tool.benchbox.duplicates] section or the check_duplicate_code.py script\nto suppress known-intentional patterns:\n- Platform adapter interface methods (from_config, apply_constraint_configuration,\n apply_unified_tuning, supports_tuning_type)\n- DataFrame adapter cast_date/cast_string implementations\n- Facade/delegation one-liners\n- Structural patterns (_load_queries, _generate_default_params)\nLower the threshold from 20,000 to catch future regressions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Configure duplicate checker ignore patterns for intended boilerplate groups", "wid": "w7"}, {"evidence": null, "item_id": "consolidate-duplicate-code", "needs": ["w7"], "notes": "Run: uv run -- python -m pytest -x -v\nRun: make duplicate-check-verbose\nVerify duplicated lines dropped by ~2,500-3,000 from baseline 17,595.\nUpdate threshold in pyproject.toml to new level + 10% buffer.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite, update duplicate threshold, verify line count reduction", "wid": "w8"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change function signatures", "instead": "only change the internal implementation", "why": "callers depend on the current API"}, {"dont": "create a generic format detector class", "instead": "follow the is_tpc_format() function style", "why": "simple predicate functions match the existing codebase pattern"}], "approach": null, "blocked_reason": null, "category": "Code Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-01-27T00:00:00Z", "deferrals": [], "deps": [], "description": "The codebase has inconsistent file format detection patterns that fail to handle:\n1. Compressed files: `data.parquet.zst`, `data.csv.gz`\n2. Sharded files: `data.parquet.1`, `data.csv.2`, `data.tbl.1.zst`\n\nSome locations use `is_tpc_format()` which correctly handles compression, but then\ncheck `.parquet` or `.csv` directly on the suffix which breaks for compressed/sharded files.\n\nThe existing `is_tpc_format()` utility correctly iterates through suffixes skipping\ncompression extensions. We need analogous utilities for parquet and csv detection,\nthen update all locations using direct suffix checks.\n\nThis is a follow-up to the TPC delimiter detection consolidation work.", "id": "consolidate-file-format-detection", "preserves": ["All existing platform tests continue to pass", "Existing file format detection behavior for uncompressed files unchanged", "is_tpc_format() API and return values unchanged"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}], "state": "done", "title": "Consolidate file format detection to handle compressed and sharded files", "verifications": [{"command": "uv run -- python -m pytest -m fast --tb=short -q", "description": "All fast tests pass after refactoring", "expected": "all tests pass, 0 failures", "item_id": "consolidate-file-format-detection", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run ruff check benchbox/", "description": "Lint passes", "expected": "no errors", "item_id": "consolidate-file-format-detection", "last_result": null, "last_run": null, "seq": 2}], "work": [], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:33:25Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #66 of consolidate-duplicate-code.\nDeferred: Consolidate Firebolt/Presto/Trino execute_query implementations\nReason deferred: These are similar (~79 lines each) but have platform-specific error handling and cursor semantics. Consolidation requires careful testing against live platforms.", "id": "consolidate-firebolt-presto-trino-execute-query-implementati", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Consolidate Firebolt/Presto/Trino execute_query implementations", "verifications": [], "work": [], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (COMPLETED)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Successfully consolidated all linting tools to use only ruff, eliminating redundant tools (black, isort, pycodestyle, flake8) and simplifying the development workflow.", "id": "consolidate-linting-tools", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Consolidate Linting Tools", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-22T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "21 platform adapters each have a _build__config function that performs\nthe same 4-step operation: (1) instantiate CredentialManager, (2) merge\noptions/saved_creds/explicit_options/overrides, (3) extract fields into a dict,\n(4) return DatabaseConfig(**config_dict).\n\nbenchbox/platforms/base/config_utils.py already contains build_platform_config()\nimplementing this exact pattern, but only 3 of 21 platforms use it\n(velox.py:761, lakesail.py:811, azure_synapse.py:1300).\n\nThe credential merge order bug (commit d41c56980) required patching 18 separate\nfiles because each had its own inline copy. If the builders used the shared helper,\nthe same fix would have been one edit.\n\nPlatforms are divided into four groups based on consolidation difficulty:\n\n Group A (5 platforms) - clean drop-in, no complications:\n databend, clickhouse_cloud, spark, snowflake, databricks\n\n Group B (4 platforms) - minor audit needed before consolidating:\n trino, presto, starburst (conditional schema key), firebolt (deployment_mode sourcing)\n\n Group C (3 platforms) - need base_options parameter added to helper:\n pg_duckdb, pg_mooncake, timescaledb (pre-seed merged_options with {\"schema\": \"public\"})\n\n Group D (6 platforms) - investigation gate required before any code:\n singlestore, doris, starrocks (os.environ fallbacks + int() coercions)\n athena (dual-source region: region or aws_region)\n bigquery (multi-step staging_root derivation from default_output_location)\n redshift (admin_database=\"dev\" hardcoded default, conditional database handling)\n\nGroups C and D must complete investigation work units before any consolidation\nis attempted. The investigation may conclude that some Group D platforms should\nstay inline - that outcome is acceptable if properly documented.", "id": "consolidate-platform-config-builders-to-shared-helper", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Consolidate platform config builders to use shared build_platform_config helper", "verifications": [], "work": [{"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": [], "notes": "Three specific questions to answer before w4 can proceed:\n\nQ1: trino/presto/starburst conditional schema\n Current code: `if \"schema\" in overrides and overrides[\"schema\"]: config_dict[\"schema\"] = ...`\n build_platform_config would emit schema=None when not set.\n Action: grep each adapter's __init__ and from_config() for schema access patterns.\n - config.get(\"schema\") \u2192 safe (None is handled)\n - config[\"schema\"] \u2192 NOT safe (KeyError if key absent, or TypeError if None used as schema)\n Read: benchbox/platforms/trino.py TrinoPlatform.from_config()\n benchbox/platforms/presto.py PrestoPlatform.from_config()\n benchbox/platforms/starburst.py StarburstPlatform.from_config()\n If all three use config.get(\"schema\"), include \"schema\" in platform_fields (emitting None is safe).\n If any uses config[\"schema\"], exclude \"schema\" from platform_fields and note it in w4.\n\nQ2: databricks missing benchmark/scale_factor/tuning_config\n build_platform_config always adds these three fields (from overrides).\n Current _build_databricks_config omits them - they'd be added as None.\n Action: Check DatabaseConfig Pydantic model for Optional annotations on these fields.\n Read: benchbox/core/schemas.py DatabaseConfig definition.\n If Optional, adding them as None is safe \u2192 databricks can go in Group A (w3).\n If required/non-Optional, databricks needs special handling in w4.\n\nQ3: firebolt deployment_mode sourcing\n Current: \"deployment_mode\": overrides.get(\"deployment_mode\") or options.get(\"deployment_mode\")\n With helper: merged_options.get(\"deployment_mode\") - since overrides is merged last, this\n should be equivalent. Verify by tracing: if deployment_mode is in overrides, it appears in\n merged_options (overrides.update last). If it's only in options, it also appears in merged_options.\n Confirm there's no case where options.get() would find it but merged_options.get() wouldn't.\n\nRecord findings as annotations in w4 notes before marking w1 done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit Group B adapters - confirm conditional-schema and missing benchmark fields are safe to normalize", "wid": "w1"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": [], "notes": "File: benchbox/platforms/base/config_utils.py\n\nChange: Add `base_options: dict[str, Any] | None = None` as last parameter before options.\nChange: Replace `merged_options: dict[str, Any] = {}` with `merged_options: dict[str, Any] = dict(base_options or {})`\n\nThat's the only code change needed. All existing callers pass no base_options - behavior unchanged.\n\nTest: tests/unit/core/test_platform_config_utils.py (already exists - add a test case)\nTest case: call build_platform_config with base_options={\"schema\": \"public\"} and empty\ncredentials \u2192 confirm schema=\"public\" appears in result.options. Then call with\nsaved_creds={\"schema\": \"custom\"} \u2192 confirm saved_creds win over base_options (priority invariant).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend build_platform_config with base_options parameter", "wid": "w2"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": [], "notes": "Five platforms where every config_dict entry is a plain merged_options.get(field).\nNOTE: clickhouse_cloud, snowflake, databricks all have the credential merge order bug\n(saved_creds updated before options). Consolidation to build_platform_config fixes this\nas a side-effect - each commit message should note the merge-order fix.\n\nFor each platform, the swap is:\n1. Remove: CredentialManager import, cred_manager, saved_creds, explicit_options, merged_options,\n name, driver_package, config_dict, and return DatabaseConfig(**config_dict) lines\n2. Add: from benchbox.platforms.base.config_utils import build_platform_config (local import)\n3. Add: return build_platform_config(platform_type=..., credential_key=..., ...)\n\nPlatform details:\n databend: credential_key=\"databend\", driver=\"databend-driver\", display=\"Databend\"\n fields: host, port, username, password, database, dsn, warehouse, ssl, disable_result_cache\n clickhouse_cloud: credential_key=\"clickhouse-cloud\", driver=\"clickhouse-connect\", display=\"ClickHouse Cloud\"\n fields: host, password, username, database, oauth_token, s3_staging_url, s3_region,\n gcs_staging_url, max_memory_usage, max_execution_time, disable_result_cache, compression\n spark: credential_key=\"spark\", driver=\"pyspark\", display=\"Apache Spark\"\n fields: master, app_name, deploy_mode, driver_memory, executor_memory, executor_cores,\n num_executors, shuffle_partitions, broadcast_threshold, adaptive_enabled,\n table_format, enable_hive, spark_config, java_home, staging_root\n snowflake: credential_key=\"snowflake\", driver=\"snowflake-connector-python\", display=\"Snowflake\"\n fields: account, warehouse, schema, username, password, role, authenticator,\n private_key_path, private_key_passphrase\n databricks: credential_key=\"databricks\", driver=\"databricks-sql-connector\", display=\"Databricks\"\n fields: server_hostname, http_path, access_token, catalog, schema,\n uc_catalog, uc_schema, uc_volume, staging_root\n\nCommit per platform (5 commits), or group if they're truly trivial after first one succeeds.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate Group A - databend, clickhouse_cloud, spark, snowflake, databricks", "wid": "w3"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": ["w1"], "notes": "w1 FINDINGS:\n Q1 (schema): ALL SAFE. trino/presto/starburst from_config() all use\n `if \"schema\" in config and config[\"schema\"]:` - handles None correctly.\n Include \"schema\" in platform_fields for all three.\n Q2 (databricks benchmark fields): DatabaseConfig uses extra=\"allow\" - benchmark,\n scale_factor, tuning_config are extra fields with no validators. Safe as None.\n Databricks promoted to Group A (w3).\n Q3 (firebolt deployment_mode): Equivalent. Include in platform_fields.\n\ntrino: credential_key=\"trino\", default_display_name=\"Trino\", default_driver_package=\"trino\"\n fields: host, port, catalog, username, password, http_scheme, verify_ssl, ssl_cert_path,\n session_properties, query_timeout, timezone, table_format, staging_root, schema\npresto: credential_key=\"presto\", default_display_name=\"Presto\", default_driver_package=\"presto-python-client\"\n fields: host, port, catalog, username, password, http_scheme, verify_ssl, ssl_cert_path,\n session_properties, query_timeout, table_format, staging_root, schema\nstarburst: credential_key=\"starburst\", default_display_name=\"Starburst\", default_driver_package=\"trino\"\n fields: host, port, catalog, username, password, role, http_scheme, verify_ssl,\n ssl_cert_path, session_properties, query_timeout, timezone, table_format, staging_root, schema\nfirebolt: credential_key=\"firebolt\", default_display_name=\"Firebolt\", default_driver_package=\"firebolt-sdk\"\n fields: url, client_id, client_secret, account_name, engine_name, api_endpoint,\n deployment_mode, s3_staging_url, s3_region\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate Group B - trino, presto, starburst, firebolt (after w1 audit)", "wid": "w4"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": [], "notes": "These platforms pre-seed merged_options with {\"schema\": \"public\"} and use .get(field, default)\nthroughout the builder. The investigation goal is confirming that build_platform_config with\nbase_options={\"schema\": \"public\"} is sufficient - no data is lost.\n\nKey finding already established (read from pg_duckdb.py:130-154):\n- pg_duckdb from_config() applies defaults for host=\"localhost\", port=5432, username=\"postgres\",\n sslmode=\"prefer\", work_mem=\"256MB\", maintenance_work_mem=\"512MB\", etc.\n- pg_duckdb registry (benchbox/platforms/__init__.py:914) registers host=\"localhost\",\n port=\"5432\", username=\"postgres\" as option defaults - so options already contains them.\n- The builder's .get(field, default) is the middle (redundant) layer of a triple-layer default:\n registry default \u2192 builder .get(default) \u2192 from_config .get(default)\n\nAction for each platform:\n1. Identify which .get(field, default) entries in the builder have NO corresponding registry\n option spec default (those are truly load-bearing).\n2. For any load-bearing defaults not in the registry: decide whether to add them to the registry\n (preferred) or keep the inline builder for that platform (fallback).\n3. Confirm schema is the only field needing base_options (since it may not have a registry default).\n\nRead:\n benchbox/platforms/pg_duckdb.py: _build_pg_duckdb_config + from_config()\n benchbox/platforms/pg_mooncake.py: same\n benchbox/platforms/timescaledb.py: same\n benchbox/platforms/__init__.py: option spec registrations for pg-duckdb, pg-mooncake, timescaledb\n\nw5 FINDINGS:\n - schema is already registered with default=\"public\" on all three platforms. base_options={\"schema\": \"public\"}\n is still useful in w6 to preserve merged_options[\"schema\"] when the builder is called directly with empty options,\n but it is not the only load-bearing default.\n - All three platforms have additional top-level builder defaults that are not yet registered:\n admin_database, sslmode, work_mem, maintenance_work_mem, effective_cache_size,\n max_parallel_workers_per_gather.\n - pg_duckdb also has a load-bearing compare_native=False builder default; this should be registered so the helper\n can keep emitting False instead of None.\n - pg_mooncake storage_mode and timescaledb chunk/compression fields are already covered by registry defaults.\n\nExpected outcome: all three platforms consolidate cleanly with base_options={\"schema\": \"public\"}.\nAction: register the missing Group C defaults first as a registry-only change, then proceed with w6.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Investigate Group C (pg_duckdb, pg_mooncake, timescaledb) - triple-layer defaults and base_options sufficiency", "wid": "w5"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": ["w2", "w5"], "notes": "After w5 confirms readiness and w2 adds base_options support.\n\nEach platform swaps to:\n return build_platform_config(\n platform_type=\"pg-duckdb\", # or pg-mooncake / timescaledb\n credential_key=\"pg-duckdb\",\n default_display_name=\"pg_duckdb\",\n default_driver_package=\"psycopg\",\n base_options={\"schema\": \"public\"},\n platform_fields=[...],\n options=options,\n overrides=overrides,\n info=info,\n )\n\nw6 FINDINGS:\n - base_options had to seed all direct-call builder defaults, not just schema. The shared helper receives\n already-defaulted registry values in the normal CLI path, but unit tests also call the builders directly\n with options={}, so host/port/username and each platform's non-None fallback fields must be pre-seeded.\n - pg_duckdb base_options: host, port, username, schema, admin_database, sslmode, work_mem,\n maintenance_work_mem, effective_cache_size, max_parallel_workers_per_gather, force_execution,\n postgres_scan_threads, compare_native.\n - pg_mooncake base_options: host, port, username, schema, admin_database, sslmode, work_mem,\n maintenance_work_mem, effective_cache_size, max_parallel_workers_per_gather, storage_mode.\n - timescaledb base_options: host, port, username, schema, admin_database, sslmode, work_mem,\n maintenance_work_mem, effective_cache_size, max_parallel_workers_per_gather, chunk_interval,\n compression_enabled, compression_after.\n - Verification: `uv run ruff check ...` passed on the touched files; focused Group C adapter+hook tests passed;\n `uv run -- python -m pytest -m fast -q -n 0` completed with 21020 passed, 1 skipped, 2105 deselected.\n\nplatform_fields for pg_duckdb: host, port, username, password, database, admin_database,\n sslmode, work_mem, maintenance_work_mem, effective_cache_size, max_parallel_workers_per_gather,\n force_execution, postgres_scan_threads, compare_native, duckdb_db_path\n(The redundant .get(field, default) in the builder are removed; from_config() still handles defaults.)\n\nRun fast tests after each platform. The test at\ntests/unit/platforms/test_pg_duckdb_adapter.py (or equivalent) must continue to pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate Group C (pg_duckdb, pg_mooncake, timescaledb) using base_options", "wid": "w6"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": [], "notes": "These builders use os.environ.get() and int() coercions for field defaults:\n singlestore: SINGLESTORE_HOST, SINGLESTORE_PORT, SINGLESTORE_USER, SINGLESTORE_PASSWORD\n doris: DORIS_HOST, DORIS_PORT (int), DORIS_HTTP_PORT (int), DORIS_BE_HTTP_PORT (int),\n DORIS_USER/DORIS_USERNAME, DORIS_PASSWORD, DORIS_DATABASE\n starrocks: STARROCKS_HOST, STARROCKS_PORT (int), STARROCKS_USER, STARROCKS_PASSWORD,\n STARROCKS_DATABASE, STARROCKS_HTTP_PORT (int)\n\nInvestigation questions:\n1. Are these platforms' PlatformOptionSpec defaults currently hardcoded strings\n (e.g., default=\"localhost\") or do they already read from env vars?\n Read: benchbox/platforms/__init__.py option spec registrations for singlestore, doris, starrocks\n2. Could PlatformOptionSpec.default be changed to a callable or env-var-aware value?\n Or could the default= argument read os.environ at registration time?\n Read: benchbox/cli/platform_hooks.py PlatformOptionSpec definition\n3. Are there existing unit tests that mock the env vars and assert specific values?\n These tests constrain how safe the migration is.\n4. The int() coercions in doris/starrocks (e.g., int(merged_options.get(\"port\") or \"9030\")):\n are these needed because the registry/options dict stores port as a string?\n If so, this is a type-coercion concern separate from the env-var question.\n\nDecision framework:\n- If env-var fallbacks CAN move to registry defaults cleanly: add to w9 as consolidation candidates\n- If env-var fallbacks require PlatformOptionSpec changes that affect other platforms: too risky, leave inline\n- If int() coercions are needed because of string\u2192int type conversion: note in w9 as a blocker\n (build_platform_config would return string where adapter expects int - a silent type bug)\n\nw7 FINDINGS:\n - PlatformOptionSpec.default is a static value. PlatformHookRegistry.get_default_options() returns that raw value\n unchanged; there is no callable/env-aware default hook today.\n - Reading os.environ at registration time would only snapshot the environment once, but the existing tests patch env vars\n around builder/direct-construction calls after import. That means moving env fallbacks into today's registry mechanism\n would break current behavior and tests.\n - Existing tests explicitly lock in env-var resolution:\n singlestore -> tests/unit/platforms/test_singlestore_adapter.py\n doris -> tests/unit/platforms/test_doris_adapter.py\n starrocks -> tests/unit/platforms/test_starrocks_adapter.py\n - int() coercions are load-bearing:\n singlestore coerces port\n doris coerces port/http_port/be_http_port\n starrocks coerces port/http_port\n\nVerdicts:\n - singlestore: Can consolidate: NO. Reason: builder-level env fallback and port coercion are runtime behaviors, not static defaults.\n - doris: Can consolidate: NO. Reason: env fallback plus multi-port int coercion are load-bearing and covered by tests.\n - starrocks: Can consolidate: NO. Reason: env fallback plus int coercion for FE/HTTP ports are load-bearing and covered by tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Investigate Group D env-var platforms (singlestore, doris, starrocks) - can env-var fallbacks move to registered defaults?", "wid": "w7"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": [], "notes": "ATHENA - dual-source region\n Current: \"region\": merged_options.get(\"region\") or merged_options.get(\"aws_region\")\n Options:\n (a) Pre-process before calling helper:\n if not options.get(\"region\") and options.get(\"aws_region\"):\n options = {**options, \"region\": options[\"aws_region\"]}\n return build_platform_config(..., platform_fields=[\"region\", ...])\n (b) Normalize in registry: register aws_region as an alias for region at registration time\n (c) Leave inline\n Evaluate: does the dual-source appear only in _build_athena_config, or does AthenaAdapter\n also access aws_region directly from config? Read athena.py AthenaAdapter.__init__ / from_config.\n If the adapter only uses config.get(\"region\"), option (a) is safe.\n\nBIGQUERY - multi-step staging_root derivation\n Current: ~15 lines of logic resolving staging_root from overrides, then default_output_location\n in merged_options, then parsing with get_cloud_path_info() to extract storage_bucket/storage_prefix.\n Options:\n (a) Extract as _resolve_bigquery_staging(merged_options, overrides) \u2192 (staging_root, bucket, prefix)\n Then call build_platform_config with staging_root/storage_bucket/storage_prefix\n injected into options before the call.\n (b) Leave inline - the logic is load-bearing and extracting it doesn't reduce complexity\n Evaluate: is there test coverage for the staging_root fallback logic in test_bigquery_adapter.py?\n If yes, extraction is safe (tests catch regressions). If no coverage, document as higher risk.\n\nREDSHIFT - admin_database default + conditional database\n Current: \"admin_database\": merged_options.get(\"admin_database\", \"dev\")\n Options:\n (a) Register \"dev\" as the registry default for admin_database option spec - then options\n will contain it and merged_options.get(\"admin_database\") returns \"dev\" without the fallback\n (b) Leave inline\n Also check: build_platform_config already handles \"if database in overrides\" the same way\n as the inline code - verify they're truly equivalent.\n Read: benchbox/platforms/__init__.py for redshift option spec registration.\n\nw8 FINDINGS:\n - athena: AthenaAdapter.__init__ already treats region and aws_region as aliases (`config.get(\"region\") or config.get(\"aws_region\")`).\n The builder's only unique logic is emitting top-level region from either source.\n - bigquery: staging_root/default_output_location parsing is covered by targeted builder tests in both\n tests/unit/platforms/test_bigquery_adapter.py and tests/unit/platforms/test_bigquery_adapter_coverage.py.\n The logic can be preserved as a post-processing step over the merged options emitted by the shared helper.\n - redshift: the builder is otherwise a plain field projection. The only special cases are\n admin_database defaulting to \"dev\" and the existing \"database only when explicitly overridden\" rule.\n build_platform_config already matches the conditional database behavior, so only admin_database needs additive seeding.\n\nVerdicts:\n - athena: Consolidation path: (a). Risk: low. Action: use build_platform_config for the plain fields, then set config.region from config.options[\"region\"] or config.options[\"aws_region\"].\n - bigquery: Consolidation path: (a). Risk: medium. Action: keep the staging resolver logic, but run it after build_platform_config using config.options as the merged source of truth.\n - redshift: Consolidation path: (a). Risk: low. Action: consolidate to build_platform_config with base_options={\"admin_database\": \"dev\"}.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Investigate Group D complex platforms (athena, bigquery, redshift) - can unique logic be pre-processed?", "wid": "w8"}, {"evidence": null, "item_id": "consolidate-platform-config-builders-to-shared-helper", "needs": ["w7", "w8"], "notes": "Investigation-driven implementation plan:\n singlestore: stay inline; add explanatory comment citing env-var fallback + int coercion\n doris: stay inline; add explanatory comment citing env-var fallback + multi-port int coercion\n starrocks: stay inline; add explanatory comment citing env-var fallback + int coercion\n athena: consolidate via build_platform_config, then normalize config.region from config.options[\"region\"] or [\"aws_region\"]\n bigquery: consolidate via build_platform_config, then resolve staging_root/storage_bucket/storage_prefix from config.options\n redshift: consolidate via build_platform_config with base_options={\"admin_database\": \"dev\"}\n\nFor every platform that stays inline after investigation, add this comment at the top\nof the function body (immediately after the docstring):\n # Intentionally inline - not using build_platform_config because: \n # See _project/DONE/main/.../consolidate-platform-config-builders-to-shared-helper.yaml\n\nThis prevents future contributors from \"cleaning up\" these builders in a way that\naccidentally re-introduces credential merge order bugs or silent type coercion errors.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Group D consolidation for platforms that cleared investigation gates (w7 + w8)", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-28T00:00:00Z", "completed_pr": null, "created_at": "2026-01-28T00:00:00Z", "deferrals": [], "deps": [], "description": "Multiple modules implement nearly identical logic for extracting data from\nbenchmark result JSON files, with duplicated schema version detection and\nfield mapping. This creates maintenance burden and risk of inconsistent\nbehavior when the schema evolves.\n\nDiscovered during fix for empty chart generation (ResultPlotter._normalize_dict\nwas using v1.x schema paths while actual output uses v2.x schema).\n\nThe fix added v2.x support to result_plotter.py, but now there are 5+ places\nwith similar extraction logic that should be consolidated.", "id": "consolidate-result-normalization", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Consolidate Duplicated Result JSON Normalization Logic", "verifications": [], "work": [], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-27T00:00:00Z", "deferrals": [], "deps": [], "description": "The codebase has ~15 locations that duplicate TPC file format detection\nusing patterns like `\".tbl\" in file_str or \".dat\" in file_str`.\n\nNew utility functions were added in commit XXXX:\n- `is_tpc_format(path)` - returns True for .tbl/.dat files\n- `get_delimiter_for_file(path)` - returns \"|\" for TPC formats, \",\" otherwise\n- `TPC_FORMAT_EXTENSIONS` - frozenset({\".tbl\", \".dat\"})\n\nThese should replace the duplicated or-chains across platform adapters.", "id": "consolidate-tpc-delimiter-detection", "preserves": [], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Consolidate TPC delimiter detection to use get_delimiter_for_file()", "verifications": [], "work": [], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "unconditionally add dummy columns in DataFrame adapters", "why": "it breaks when the file has no trailing delimiter (extra NaN column)"}, {"INSTEAD": "(not specified)", "dont": "use delimiter == '|' as a proxy for has_trailing_delimiter", "why": "not all pipe-delimited files have trailing pipes"}, {"INSTEAD": "(not specified)", "dont": "use naive 'ends with delimiter' check", "why": "it incorrectly flags TPC-DS files like time_dim.dat that end with | as field content"}], "approach": "The smart field-count detection from FormatConverter._needs_trailing_dummy_column() is the\ncanonical approach: read the first non-empty line, split by delimiter, compare field count\nto expected column count. This correctly handles both TPC-H files (which have trailing |)\nand TPC-DS files like time_dim.dat (which do not). Move this logic to file_format.py as\nthe single source of truth, then update all 13 call sites to use it. For SQL adapters,\nonly standardize the column name -- the unconditional add-then-drop pattern is safe there.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-09T00:00:00Z", "completed_pr": null, "created_at": "2026-02-09T00:00:00Z", "deferrals": [], "deps": [], "description": "TPC benchmark data files (.tbl/.dat) use | as a field terminator (not separator),\ncreating an extra empty field when split. This is currently handled in 13+ locations\nacross 9 files using 3 different detection approaches and 3 different dummy column\nnames (_trailing_, _trailing_delimiter, _trailing_ignore). The inconsistency caused\nregressions (e.g., time_dim.dat conversion failure where the naive \"ends with |\"\ncheck incorrectly added a dummy column).\n\nApproach: Extract the smart field-count detection logic into benchbox/utils/file_format.py\nas a shared utility. Update all 13 locations to use the shared constant and, where\nappropriate, the shared detection function. Remove has_trailing_delimiter from\nFileFormatInfo (it's always False -- dead code).", "id": "consolidate-trailing-delimiter-handling", "preserves": ["All existing TPC-H and TPC-DS data loading must continue to work identically", "Compressed file support in trailing delimiter detection (via CompressionManager)", "DuckDB SQL EXCLUDE pattern and PyArrow include_columns filtering remain unconditional (they handle both with/without trailing delimiter)", "Schema-less fallback for has_trailing_delimiter when column_names is None"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/cudf.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/cudf_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/datafusion_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/modin_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pyspark_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/polars_platform.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/file_format.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/format_converters/base.py"}], "state": "done", "title": "Consolidate trailing delimiter handling into shared utility", "verifications": [{"command": "uv run ruff check .", "description": "Lint passes", "expected": null, "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "All fast tests pass", "expected": null, "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run benchbox run --platform datafusion --benchmark tpcds --scale 0.01 --mode dataframe", "description": "TPC-DS time_dim loads correctly via DataFusion DataFrame mode", "expected": null, "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -r '_trailing_' benchbox/ --include='*.py'", "description": "No stale trailing column references remain", "expected": "Only references to TRAILING_DUMMY_COLUMN or its value '_trailing_delimiter_'", "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": [], "notes": "Add to benchbox/utils/file_format.py:\n- TRAILING_DUMMY_COLUMN = \"_trailing_delimiter_\" -- canonical constant\n- has_trailing_delimiter(path, delimiter, column_names=None) -> bool -- moved from\n FormatConverter._needs_trailing_dummy_column(), with schema-less fallback (checks\n if first line ends with delimiter when column_names is None)\n- get_column_names_with_trailing(column_names, has_trailing) -> list[str] -- helper\n that appends the dummy column name when needed\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add shared utility to file_format.py (TRAILING_DUMMY_COLUMN, has_trailing_delimiter(), get_column_names_with_trailing())", "wid": "w1"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/core/dataframe/data_loader.py\n- Delete _needs_trailing_dummy_column() static method\n- Replace calls with file_format.has_trailing_delimiter()\n- Replace \"_trailing_\" with TRAILING_DUMMY_COLUMN\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update FormatConverter in data_loader.py to use shared utility", "wid": "w2"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/platforms/dataframe/datafusion_df.py\n- Delete duplicated _needs_trailing_dummy_column() instance method\n- Replace calls with file_format.has_trailing_delimiter()\n- Replace \"_trailing_\" with TRAILING_DUMMY_COLUMN\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DataFusion DF adapter to use shared utility", "wid": "w3"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "Files: pandas_df.py, modin_df.py, cudf_df.py, dask_df.py\nReplace the unconditional add-then-drop pattern:\n Before: if is_tpc_format(path) and names: extended_names = names + [\"_trailing_\"]\n After: if is_tpc_format(path) and names and has_trailing_delimiter(path, delimiter, names):\n extended_names = names + [TRAILING_DUMMY_COLUMN]\nUpdate the drop check to use TRAILING_DUMMY_COLUMN.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update pandas-family DataFrame adapters (pandas_df, modin_df, cudf_df, dask_df)", "wid": "w4"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/platforms/dataframe/pyspark_df.py\n- Bug fix: Replace path_str.endswith(\".tbl\") with is_tpc_format(path) (currently misses .dat)\n- Apply same smart detection + constant pattern as Phase 4\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix PySpark DF adapter bug (.tbl-only check misses .dat) and update to shared utility", "wid": "w5"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "Files:\n- benchbox/utils/format_converters/base.py -- change \"_trailing_delimiter\" -> TRAILING_DUMMY_COLUMN\n- benchbox/platforms/base/data_loading.py -- change \"_trailing_ignore\" -> TRAILING_DUMMY_COLUMN\nKeep the unconditional add-then-drop pattern here (DuckDB SQL EXCLUDE and PyArrow\ninclude_columns handle both cases robustly).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Standardize column names in SQL adapters (format_converters/base.py, data_loading.py)", "wid": "w6"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/platforms/base/utils.py\n- Remove has_trailing_delimiter field from FileFormatInfo (always False, misleading)\n- Update detect_file_format() to return 2-field tuple\nDownstream updates:\n- benchbox/platforms/polars_platform.py -- remove 3rd element from _detect_file_format(),\n use shared detection in _load_csv()\n- benchbox/platforms/datafusion.py -- simplify _detect_csv_format() to return just delimiter\n- benchbox/platforms/spark.py -- remove dead format_info.has_trailing_delimiter check\n- benchbox/platforms/cudf.py -- remove dead format_info.has_trailing_delimiter check\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove dead has_trailing_delimiter from FileFormatInfo and update downstream consumers", "wid": "w7"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w2", "w3", "w4", "w5", "w6", "w7"], "notes": "- tests/unit/utils/test_file_format.py -- add tests for has_trailing_delimiter(),\n TRAILING_DUMMY_COLUMN, get_column_names_with_trailing()\n- Update any test referencing \"_trailing_\", \"_trailing_ignore\", or \"_trailing_delimiter\"\n to use the constant\n- Update tests consuming FileFormatInfo for the removed field\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests for shared utility, constant references, and FileFormatInfo changes", "wid": "w8"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "unconditionally add dummy columns in DataFrame adapters", "instead": "(not specified)", "why": "it breaks when the file has no trailing delimiter (extra NaN column)"}, {"dont": "use delimiter == '|' as a proxy for has_trailing_delimiter", "instead": "(not specified)", "why": "not all pipe-delimited files have trailing pipes"}, {"dont": "use naive 'ends with delimiter' check", "instead": "(not specified)", "why": "it incorrectly flags TPC-DS files like time_dim.dat that end with | as field content"}], "approach": "The smart field-count detection from FormatConverter._needs_trailing_dummy_column() is the\ncanonical approach: read the first non-empty line, split by delimiter, compare field count\nto expected column count. This correctly handles both TPC-H files (which have trailing |)\nand TPC-DS files like time_dim.dat (which do not). Move this logic to file_format.py as\nthe single source of truth, then update all 13 call sites to use it. For SQL adapters,\nonly standardize the column name -- the unconditional add-then-drop pattern is safe there.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-09T00:00:00Z", "completed_pr": null, "created_at": "2026-02-09T00:00:00Z", "deferrals": [], "deps": [], "description": "TPC benchmark data files (.tbl/.dat) use | as a field terminator (not separator),\ncreating an extra empty field when split. This is currently handled in 13+ locations\nacross 9 files using 3 different detection approaches and 3 different dummy column\nnames (_trailing_, _trailing_delimiter, _trailing_ignore). The inconsistency caused\nregressions (e.g., time_dim.dat conversion failure where the naive \"ends with |\"\ncheck incorrectly added a dummy column).\n\nApproach: Extract the smart field-count detection logic into benchbox/utils/file_format.py\nas a shared utility. Update all 13 locations to use the shared constant and, where\nappropriate, the shared detection function. Remove has_trailing_delimiter from\nFileFormatInfo (it's always False -- dead code).", "id": "consolidate-trailing-delimiter-handling", "preserves": ["All existing TPC-H and TPC-DS data loading must continue to work identically", "Compressed file support in trailing delimiter detection (via CompressionManager)", "DuckDB SQL EXCLUDE pattern and PyArrow include_columns filtering remain unconditional (they handle both with/without trailing delimiter)", "Schema-less fallback for has_trailing_delimiter when column_names is None"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/cudf.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/cudf_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/datafusion_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/modin_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pyspark_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/polars_platform.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/file_format.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/format_converters/base.py"}], "state": "done", "title": "Consolidate trailing delimiter handling into shared utility", "verifications": [{"command": "uv run ruff check .", "description": "Lint passes", "expected": null, "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "All fast tests pass", "expected": null, "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run benchbox run --platform datafusion --benchmark tpcds --scale 0.01 --mode dataframe", "description": "TPC-DS time_dim loads correctly via DataFusion DataFrame mode", "expected": null, "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -r '_trailing_' benchbox/ --include='*.py'", "description": "No stale trailing column references remain", "expected": "Only references to TRAILING_DUMMY_COLUMN or its value '_trailing_delimiter_'", "item_id": "consolidate-trailing-delimiter-handling", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": [], "notes": "Add to benchbox/utils/file_format.py:\n- TRAILING_DUMMY_COLUMN = \"_trailing_delimiter_\" -- canonical constant\n- has_trailing_delimiter(path, delimiter, column_names=None) -> bool -- moved from\n FormatConverter._needs_trailing_dummy_column(), with schema-less fallback (checks\n if first line ends with delimiter when column_names is None)\n- get_column_names_with_trailing(column_names, has_trailing) -> list[str] -- helper\n that appends the dummy column name when needed\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add shared utility to file_format.py (TRAILING_DUMMY_COLUMN, has_trailing_delimiter(), get_column_names_with_trailing())", "wid": "w1"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/core/dataframe/data_loader.py\n- Delete _needs_trailing_dummy_column() static method\n- Replace calls with file_format.has_trailing_delimiter()\n- Replace \"_trailing_\" with TRAILING_DUMMY_COLUMN\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update FormatConverter in data_loader.py to use shared utility", "wid": "w2"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/platforms/dataframe/datafusion_df.py\n- Delete duplicated _needs_trailing_dummy_column() instance method\n- Replace calls with file_format.has_trailing_delimiter()\n- Replace \"_trailing_\" with TRAILING_DUMMY_COLUMN\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DataFusion DF adapter to use shared utility", "wid": "w3"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "Files: pandas_df.py, modin_df.py, cudf_df.py, dask_df.py\nReplace the unconditional add-then-drop pattern:\n Before: if is_tpc_format(path) and names: extended_names = names + [\"_trailing_\"]\n After: if is_tpc_format(path) and names and has_trailing_delimiter(path, delimiter, names):\n extended_names = names + [TRAILING_DUMMY_COLUMN]\nUpdate the drop check to use TRAILING_DUMMY_COLUMN.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update pandas-family DataFrame adapters (pandas_df, modin_df, cudf_df, dask_df)", "wid": "w4"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/platforms/dataframe/pyspark_df.py\n- Bug fix: Replace path_str.endswith(\".tbl\") with is_tpc_format(path) (currently misses .dat)\n- Apply same smart detection + constant pattern as Phase 4\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix PySpark DF adapter bug (.tbl-only check misses .dat) and update to shared utility", "wid": "w5"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "Files:\n- benchbox/utils/format_converters/base.py -- change \"_trailing_delimiter\" -> TRAILING_DUMMY_COLUMN\n- benchbox/platforms/base/data_loading.py -- change \"_trailing_ignore\" -> TRAILING_DUMMY_COLUMN\nKeep the unconditional add-then-drop pattern here (DuckDB SQL EXCLUDE and PyArrow\ninclude_columns handle both cases robustly).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Standardize column names in SQL adapters (format_converters/base.py, data_loading.py)", "wid": "w6"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w1"], "notes": "File: benchbox/platforms/base/utils.py\n- Remove has_trailing_delimiter field from FileFormatInfo (always False, misleading)\n- Update detect_file_format() to return 2-field tuple\nDownstream updates:\n- benchbox/platforms/polars_platform.py -- remove 3rd element from _detect_file_format(),\n use shared detection in _load_csv()\n- benchbox/platforms/datafusion.py -- simplify _detect_csv_format() to return just delimiter\n- benchbox/platforms/spark.py -- remove dead format_info.has_trailing_delimiter check\n- benchbox/platforms/cudf.py -- remove dead format_info.has_trailing_delimiter check\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove dead has_trailing_delimiter from FileFormatInfo and update downstream consumers", "wid": "w7"}, {"evidence": null, "item_id": "consolidate-trailing-delimiter-handling", "needs": ["w2", "w3", "w4", "w5", "w6", "w7"], "notes": "- tests/unit/utils/test_file_format.py -- add tests for has_trailing_delimiter(),\n TRAILING_DUMMY_COLUMN, get_column_names_with_trailing()\n- Update any test referencing \"_trailing_\", \"_trailing_ignore\", or \"_trailing_delimiter\"\n to use the constant\n- Update tests consuming FileFormatInfo for the removed field\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests for shared utility, constant references, and FileFormatInfo changes", "wid": "w8"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-14T00:00:00Z", "completed_pr": null, "created_at": "2025-12-14T00:00:00Z", "deferrals": [], "deps": [], "description": "The CLI currently has two separate tuning command structures that should be unified:\n\nCurrent (inconsistent):\n- `benchbox create-sample-tuning --platform duckdb` (top-level, 20 chars)\n- `benchbox df-tuning create-sample --platform polars` (nested group)\n- `benchbox df-tuning validate config.yaml`\n- `benchbox df-tuning show-defaults --platform polars`\n\nProposed (unified):\n- `benchbox tuning init --platform duckdb` (SQL platforms)\n- `benchbox tuning init --platform polars --mode dataframe` (DataFrame platforms)\n- `benchbox tuning validate config.yaml --platform polars`\n- `benchbox tuning defaults --platform polars`\n\nThis consolidation improves discoverability, reduces cognitive load, and provides\na consistent interface for all tuning-related operations.", "id": "consolidate-tuning-commands", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Consolidate tuning commands under unified `tuning` group", "verifications": [], "work": [], "worktree": "ux-improvements"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Legal and Licensing", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Update copyright and license declarations throughout the project to ensure proper attribution and TPC compliance\"\n- \"**Tasks**:\"\n- \"**Specific TPC Copyright Requirements**:\"\n- \"**Files Affected**: All source files (.py), documentation (.md), configuration files, examples, tests, package metadata\"\n- \"**Impact**: \u2705 Ensures legal compliance, proper attribution, protects project intellectual property, and maintains TPC benchmark trademark compliance. Successfully added copyright headers to 323+ Python files, 108+ documentation files, and comprehensive legal documentation including COPYRIGHT.md with TPC Fair Use Policy compliance.\"", "id": "copyright-and-license-declaration-updates-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Copyright and License Declaration Updates \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "bump read_model_version in the DDL comment without the matching pipeline change", "why": "record it as an item-5 obligation to avoid a half-applied version."}, {"INSTEAD": "(not specified)", "dont": "make funding NULLable in the read model", "why": "default 'unspecified' keeps the badge/chip logic total."}, {"INSTEAD": "(not specified)", "dont": "put the vendor/funding derivation logic in a DuckDB view (violates the bare-column G-11 rule)", "why": "it belongs in the pipeline/Python."}, {"INSTEAD": "(not specified)", "dont": "re-derive label->visibility here; import the mapping from provenance.py.", "why": "(unstated)"}], "approach": "Keep the inventory's sidecar-presence inference for internal vs community, and\nlayer the vendor signal on top exactly as the item-3 gate resolves it (e.g.\nbundle under results-data/bundles/vendor/ -> vendor-supplied). Read funding\nfrom the bundle's provenance.funding first, then the manifest, defaulting to\n'unspecified'. For the DDL, funding is a plain column; is_ranking_eligible is\nalready a precomputed boolean, so the only change is which visibility states\nmap to true \u2014 encode that from provenance.py's mapping so the DDL, inventory,\nand pipeline agree. Because the pipeline that POPULATES the DB is gitignored,\nthis item changes the SCHEMA + tests + inventory only and hands the populate\nlogic + read_model_version bump to item 5.\n", "blocked_reason": null, "category": "Feature", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": ["bundle-schema-and-run-cli", "provenance-vocabulary-and-labels", "submit-manifest-and-validator"], "description": "Integrate the two new signals into the explorer's ingest layer so they reach\nthe browser. Two in-repo surfaces change: (1) scripts/generate_corpus_inventory.py,\nwhich today derives trust_label from submission-manifest-sidecar presence\n(_bundle_trust_label, :41-52) and emits per-bundle metadata + a\nby_trust_label summary; (2) docs/development/browser-duckdb-schema.sql, the\nfrozen read-model DDL that already defines trust_label, visibility, and\nis_ranking_eligible on the `results` table and carries trust_label into\nbenchmark_rankings and cohort_metadata.\n\nThis item teaches the inventory to recognize the maintainer-controlled vendor\nsignal (from item 3) and the funding value (from item 2's provenance block /\nitem 3's manifest), adds a `funding` column to the read-model DDL and the\nranking/cohort projections, and sets is_ranking_eligible = true for\nvendor-supplied per decision D2. It records the required read_model_version\nbump as a hand-off obligation for item 5 (the pipeline that writes the DB is\nnot in this checkout).\n", "id": "corpus-inventory-and-read-model", "preserves": ["Bundles with no provenance/funding default to funding='unspecified' and today's trust_label derivation is unchanged for internal/community.", "DuckDB views keep projecting only bare columns (no arithmetic/CASE/window in views); all derivation stays in the pipeline (Python).", "corpus-inventory schema_version is bumped if the entry shape changes (currently '2.2').", "corpus-inventory.json remains deterministic (sorted, timestamp-normalized in --check)."], "prior_art": [{"concept": "_project/scripts/explorer_pipeline/models.py:73-97 RANKING_ELIGIBLE_VISIBILITIES + RANKING_ELIGIBLE_TRUST_LABELS \u2014 supersede (hardcoded sets OMIT public-vendor-reported/vendor-supplied; reconcile with provenance.py so vendor rows rank per D2). Found by item-1 code review.", "decision": "supersede", "path": "_project/scripts/explorer_pipeline/models.py:73-97"}, {"concept": "benchbox/core/results/provenance.py (item 1) \u2014 reference (mapping + funding enum, imported not duplicated)", "decision": "reuse", "path": "benchbox/core/results/provenance.py"}, {"concept": "docs/development/browser-duckdb-schema.sql results table (trust_label/visibility/is_ranking_eligible) + benchmark_rankings + cohort_metadata \u2014 extend (add funding column; wire vendor visibility into ranking eligibility)", "decision": "extend", "path": "docs/development/browser-duckdb-schema.sql"}, {"concept": "scripts/generate_corpus_inventory.py:41-52 _bundle_trust_label + :63-140 extract_metadata/generate_inventory \u2014 extend (add vendor detection, funding extraction, by_funding summary)", "decision": "extend", "path": "scripts/generate_corpus_inventory.py:41-52"}, {"concept": "tests/unit/core/explorer_pipeline/test_pipeline.py:279-465 trust-label sidecar override tests \u2014 extend (add vendor + funding cases)", "decision": "extend", "path": "tests/unit/core/explorer_pipeline/test_pipeline.py:279-465"}, {"concept": "tests/unit/core/explorer_pipeline/test_read_model_contract.py + tests/unit/cli/test_explorer_build_contract.py \u2014 extend (read_model_version + column set)", "decision": "extend", "path": "tests/unit/core/explorer_pipeline/test_read_model_contract.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/scripts/explorer_pipeline/ (not in checkout; item 5)"}, {"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "do_not_modify", "path_glob": "results-explorer/ (not in checkout; item 5)"}, {"kind": "only_modify", "path_glob": "docs/development/browser-duckdb-schema.sql"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json (regenerated)"}, {"kind": "only_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "tests/unit/explorer/"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/ (corpus inventory tests)"}], "state": "active", "title": "Surface vendor-supplied label + funding in the corpus inventory and DuckDB read model", "verifications": [{"command": "uv run -- python scripts/generate_corpus_inventory.py --write && uv run -- python -c \"import json; inv=json.load(open('results-data/corpus-inventory.json')); assert 'by_funding' in inv['summary']; assert all('funding' in b for b in inv['bundles'])\"", "description": "Inventory includes funding + by_funding and vendor labels", "expected": "no assertion error", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/generate_corpus_inventory.py --check", "description": "Inventory --check is stable after --write", "expected": "OK: corpus-inventory.json is up-to-date", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ tests/unit/explorer/ -q", "description": "Read-model + explorer pipeline contract tests pass with the new column", "expected": "all pass (or documented xfails pending item 5 pipeline change)", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"import duckdb; con=duckdb.connect(); con.execute(open('docs/development/browser-duckdb-schema.sql').read())\"", "description": "DDL parses", "expected": "no error", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "corpus-inventory-and-read-model", "needs": [], "notes": "Resolve via item-1 mapping + item-3 vendor signal (not just community-vs-default); extract funding from the bundle provenance block/manifest; add funding to each entry and a by_funding counter to the summary; retire the local COMMUNITY/DEFAULT literals.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In generate_corpus_inventory.py, resolve trust_label via the item-1 mapping and the item-3 vendor signal, and extract funding from the bundle. Import constants from provenance.py.", "wid": "w1"}, {"evidence": null, "item_id": "corpus-inventory-and-read-model", "needs": ["w1"], "notes": "Propagate into benchmark_rankings and cohort_metadata; ensure the vendor visibility state is treated as ranking-eligible in the is_ranking_eligible derivation (computed in Python per the G-11 views-are-bare-columns rule).", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `funding VARCHAR NOT NULL` to the `results` table in browser-duckdb-schema.sql and propagate it where trust_label travels.", "wid": "w2"}, {"evidence": null, "item_id": "corpus-inventory-and-read-model", "needs": ["w2"], "notes": "Record the required read_model_version bump as an explicit obligation passed to item 5.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update the read-model contract tests + corpus-inventory tests for the new column/summary and the vendor/funding cases.", "wid": "w3"}], "worktree": "results-labels-funding"} -{"anti_patterns": [{"INSTEAD": "extend the single primitive.", "dont": "add a second hash or a per-engine digest variant", "why": "calculate_checksum is the one shared definition the TPC-Havoc gates also use"}, {"INSTEAD": "call it that everywhere it is documented.", "dont": "describe the value digest as proving 'correctness'", "why": "it is a regression snapshot vs a DuckDB baseline"}, {"INSTEAD": "make the digest type/engine-stable first.", "dont": "enable cross-engine digest comparison before w2", "why": "the int-vs-Decimal-vs-float and date/NULL rendering differences guarantee spurious mismatches"}, {"INSTEAD": "ship a deterministic writer.", "dont": "leave the reference regenerable only by hand-copy", "why": "an oracle no one can reproduce is the fragility this effort claimed to remove"}, {"INSTEAD": "verify in w1, regenerate in the same change.", "dont": "switch to relative precision without re-verifying cross-build stability at the pinned seed and regenerating the reference", "why": "a naive relative rounding near zero can be LESS stable than absolute"}], "approach": "Treat this as fidelity hardening of an already-shipped, CI-enforced oracle, not a\nrebuild. Sequence: (w1) quantify the precision floor as a test and decide\nabsolute-vs-relative; (w2) make the digest type-stable (prerequisite for\nindependence); (w3) replace the hand-copy with an idempotent regeneration target;\n(w4) document the regression-snapshot nature so green cells are not over-read;\n(w5) decide whether a cross-engine agreement check is worth it now; (w6) cover the\nNone-safe sort path the gate never hits. Any change to normalization MUST\nregenerate the 18 reference digests in the same change and prove two-run\ndeterminism, or the gate will go RED on a correct tree.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-24T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "correctness-gate-value-digest-fidelity-followups", "id": 363, "reason": "Out of scope; breadth is owned by benchmark-correctness-oracle-coverage-map. This item is about the FIDELITY of the existing tpch SF=1 value oracle, not its breadth.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason explicitly says breadth beyond tpch SF=1 is owned by benchmark-correctness-oracle-coverage-map, which is open (active).", "summary": "Backfill value digests for benchmarks beyond tpch, or for scales above SF=1."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "correctness-gate-value-digest-fidelity-followups", "id": 364, "reason": "w5 decides whether to add ONE bounded cross-engine agreement check or defer; a full matrix is a separate, larger effort (cf. tpchavoc-*-engine-equivalence-sampling).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: correctness-gate-value-digest-fidelity-followups and the tpchavoc cross-dialect/third-engine sampling items are DONE (bounded checks shipped). A full cross-engine matrix is the expansive path the project deliberately avoids; per-benchmark differential oracles owned by the coverage-map item. (tracked by benchmark-correctness-oracle-coverage-map)", "summary": "A full cross-engine value-equivalence matrix across all CI engines."}], "deps": [], "description": "An adversarial falsification review of the value-digest oracle (PR #867,\nreviewed on develop @ e4a04484 \u2014 confirmed an ancestor of origin/develop, all\n18 stored digests reproduced by running `make test-correctness-gate` on the\npinned DuckDB 1.3.2) confirmed the headline win is REAL: a wrong-but-same-\ncardinality TPC-H answer at SF=1 now turns the gate RED, the check is CI-wired\nas a hard blocker (.github/workflows/pr.yml correctness-gate job; the aggregator\nrequires CORRECTNESS_RESULT == success for any code PR), and #867's CI ran it\ngreen for ~11.5 min (not skipped). The strict arming\n(`digest_evaluated == digest_reference_count`) was reproduced locally to require\nall 18 digests to evaluate and match, so a missing digest disarms RED not green.\n\nThe same review reproduced four residual fidelity gaps that the AI-implemented,\nAI-reviewed, AI-merged chain did not surface. None is a correctness regression;\neach narrows what \"value-level\" actually proves and should be either fixed or\nhonestly documented:\n\n- PRECISION FLOOR IS ABSOLUTE, NOT RELATIVE (MEDIUM). The digest normalizes real\n numbers with `f\"{value:.4f}\"` (benchbox/core/results/result_digest.py\n _format_real, _DIGEST_FLOAT_PRECISION=4). Reproduced: an error <5e-5 ABSOLUTE\n in any single cell is invisible regardless of column magnitude\n (delta=4.9e-5 -> identical digest; 5.0e-5 -> differs). On large-magnitude\n columns (TPC-H revenue ~1e10) 5e-5 absolute is ~1e-15 relative \u2014 effectively\n perfect. But on small-magnitude averaged ratio columns it is coarse: Q1's\n `avg_disc` is ~0.049985 (a float), so a sub-5e-5-absolute (~0.1% relative)\n aggregate/rounding bug on that column survives the gate\n (compute_result_digest([(0.05,)]) == compute_result_digest([(0.05004,)])).\n\n- DIGEST CONFLATES VALUE WITH TYPE REPRESENTATION (LOW). _normalize_cell renders\n int exactly (\"1478493\") but float/Decimal at fixed precision\n (\"37734107.0000\"). The SAME numeric value with a different Python type hashes\n DIFFERENTLY. Stable today on pinned DuckDB 1.3.2, but it means the digest is a\n value+type-rendering digest, and it is the blocker for any cross-engine reuse.\n\n- EMISSION IS ENGINE-WIDE BUT THE ORACLE IS DUCKDB-ONLY (LOW). compute_result_digest\n is called both in benchbox/platforms/duckdb.py:999 AND in the shared DBAPI\n helper benchbox/platforms/base/sql_execution.py:80 (sqlite/postgres/datafusion/\n ...). It is harmless today \u2014 `_stored_value_digest` only returns a reference for\n tpch SF=1, so a non-DuckDB digest is computed-but-never-compared \u2014 but combined\n with the type-sensitivity above it is a latent trap for the cross-engine\n independence work below, and it sits outside the original TODO's scope_limit\n (benchbox/platforms/) by the implementer's own admission.\n\n- THE VALUE REFERENCE IS A SELF-SNAPSHOT, NOT AN INDEPENDENT ORACLE, AND HAS NO\n AUTOMATED REGENERATION (MEDIUM). The 18 reference digests in\n benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json\n were produced by running benchbox-on-DuckDB-1.3.2 and HAND-COPYING the emitted\n digests (provenance note says exactly this; no script/make target writes the\n file \u2014 grep confirms only loader.py reads it). So the oracle detects CHANGE\n from the frozen baseline (a genuine regression tripwire) but NOT correctness vs\n an authority outside benchbox: a conceptual value bug present at freeze time is\n enshrined, never caught. This is the same hand-curated-answer-key fragility the\n effort claimed to avoid, one level down. There is also no one-button\n regeneration, so a legitimate DuckDB bump means a manual re-copy.\n\n- THE NONE-SAFE SORT PATH IS UNTESTED BY THE GATE (LOW). calculate_checksum sorts\n rows by the None-safe `_row_sort_key`. Reproduced: 0 of the 18 SF=1 gate result\n sets contain a NULL or mixed-type column, so for the gate's inputs the None-safe\n surrogate is order-identical to a plain sort and its NULL/mixed-type branch\n never fires in the gate. (Note: the None-safe sort PREDATES #867 \u2014 it existed at\n parent ce999bd7 \u2014 so #867 only promoted the helper to module level; this is a\n coverage gap, not a #867 regression.)\n\nThis item fixes or honestly documents each gap. It is the fidelity complement to\nthe value axis shipped in bounded-correctness-gate-value-oracle (#867, DONE); it\ndoes NOT re-do the digest emission, the strict arming, or the row-count work.", "id": "correctness-gate-value-digest-fidelity-followups", "preserves": ["The #867 value axis is unchanged: digest emission behind BENCHBOX_EMIT_RESULT_DIGEST, the strict arming (digest_evaluated == digest_reference_count), and the additive row-count check keep working exactly as today.", "The TPC-Havoc gates that share calculate_checksum stay green \u2014 reuse the one digest definition, never fork it.", "`benchbox run` default behavior, payload shape, and cost stay unchanged when the digest flag is unset (digest emission stays gate-only).", "`make test-correctness-gate` stays green on a correct tree and on the required PR path after any normalization change (regenerate the reference in the same change)."], "prior_art": [{"concept": "Makefile:test-correctness-gate (BENCHBOX_EMIT_RESULT_DIGEST=1, pinned seed 17039360, SF=1, 18 query ids) \u2014 the exact run whose stream-0 digests become the reference; the regeneration target must reuse this configuration, not duplicate it.", "decision": "reuse", "path": "Makefile:test-correctness-gate"}, {"concept": "_project/DONE/main/planning/bounded-correctness-gate-value-oracle.yaml \u2014 the effort this hardens; w3 already proved sensitivity to gross mutations, w1 chose absolute rounding as a non-gating open question. This item revisits that choice with the reproduced floor quantified.", "decision": "reuse", "path": "_project/DONE/main/planning/bounded-correctness-gate-value-oracle.yaml"}, {"concept": "_project/analysis/REVIEW-PROTOCOL.md \u2014 where the self-snapshot-vs-independent-oracle distinction should be documented so a future reviewer does not over-read a green value gate.", "decision": "reuse", "path": "_project/analysis/REVIEW-PROTOCOL.md"}, {"concept": "benchbox/core/expected_results/loader.py:load_tpch_value_digests \u2014 the only reader of the JSON today; a regeneration writer should live near it or as a _project/scripts tool.", "decision": "reuse", "path": "benchbox/core/expected_results/loader.py:load_tpch_value_digests"}, {"concept": "benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json \u2014 the hand-copied reference; the regeneration target must WRITE this file deterministically from a gate run, replacing the manual copy described in its provenance note.", "decision": "reuse", "path": "benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json"}, {"concept": "benchbox/core/results/result_digest.py:_format_real/_normalize_cell/_DIGEST_FLOAT_PRECISION \u2014 the absolute-precision normalization to revisit for the precision-floor and type-stability work; extend, do not fork.", "decision": "extend", "path": "benchbox/core/results/result_digest.py:_format_real/_normalize_cell/_DIGEST_FLOAT_PRECISION"}, {"concept": "benchbox/core/tpchavoc/validation.py:calculate_checksum/_row_sort_key \u2014 the promoted single-result digest primitive and its None-safe sort; reuse for the None-coverage test, do not add a second hash.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py:calculate_checksum/_row_sort_key"}, {"concept": "benchbox/platforms/base/sql_execution.py:80 + benchbox/platforms/duckdb.py:999 \u2014 the two emission sites; the engine-scope and cross-engine-independence decisions touch both.", "decision": "reuse", "path": "benchbox/platforms/base/sql_execution.py:80"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/"}, {"kind": "only_modify", "path_glob": "benchbox/core/expected_results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Harden the bounded-gate value digest: precision floor, type-stability, independence, and auditable regeneration", "verifications": [{"command": "make test-correctness-gate", "description": "Bounded gate stays green and the digest assertion still arms for all 18 queries after any normalization change", "expected": "1 passed, ran=1 skipped=0; row-count and value-digest checks evaluated for all 18 queries", "item_id": "correctness-gate-value-digest-fidelity-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/test_correctness_gate_value_oracle.py -q", "description": "The precision-floor test pins the exact sensitivity boundary", "expected": "tests pass; an error just below the floor is uncaught and just above is caught, per the pinned assertion", "item_id": "correctness-gate-value-digest-fidelity-followups", "last_result": null, "last_run": null, "seq": 2}, {"command": "make correctness-gate-digests-regen && git diff --exit-code benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json", "description": "The regeneration target is idempotent on a clean tree", "expected": "no diff (regeneration reproduces the committed reference byte-for-byte)", "item_id": "correctness-gate-value-digest-fidelity-followups", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": [], "notes": "Pin the reproduced floor as a test and make an explicit keep-or-change\ndecision. The floor is 5e-5 ABSOLUTE per cell: invisible below, visible at/above\n(modulo round-half-even at the exact midpoint). Quantify it per gate column\nclass \u2014 large Decimals (revenue) are effectively perfect; small floats\n(Q1 avg_disc ~0.05) are only ~0.1%-relative protected. Decide between:\n(a) keep absolute 4dp and DOCUMENT the small-ratio blind spot with the worst\nexposed column named; or (b) switch to N significant figures / a hybrid\nmax(absolute, relative) so sensitivity is uniform in relative terms, then\nre-verify cross-build stability at the pinned seed and regenerate the reference\n(w4). Record the decision and its rationale next to _format_real.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Quantify the precision floor and decide absolute vs relative/significant-figure normalization", "wid": "w1"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w1"], "notes": "Decide and implement: either canonicalize integer-valued numerics to one\nrepresentation before hashing (so int 37734107, Decimal('37734107.00'), and\nfloat 37734107.0 hash identically) and likewise normalize dates/strings of\nequal logical value, OR explicitly document that the digest is a value+type\ndigest and is therefore DuckDB-pinned. Type-stability is a prerequisite for the\ncross-engine independence option in w5; if w5 is deferred, document instead.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the digest stable across numeric TYPE representation (int vs float vs Decimal of equal value)", "wid": "w2"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w1"], "notes": "Replace the hand-copy described in the JSON provenance note with a script /\nmake target (e.g. `make correctness-gate-digests-regen`) that runs the gate\nconfiguration (BENCHBOX_EMIT_RESULT_DIGEST=1, seed 17039360, SF=1, the 18 query\nids) and writes the digests + provenance block back to the JSON deterministically.\nReuse the Makefile:test-correctness-gate configuration rather than duplicating\nthe seed/scale/query-id list. The target must be idempotent on a clean tree\n(regenerate -> no diff) and must stamp the DuckDB version it ran under.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an auditable, one-command regeneration path that WRITES tpch_value_digests_sf1.json", "wid": "w3"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w3"], "notes": "The Makefile note, tests/README, the JSON provenance, and REVIEW-PROTOCOL.md\nshould state plainly that the value digest detects CHANGE from a benchbox-on-\nDuckDB baseline (a regression tripwire), not correctness against an external\nauthority, and that a conceptual value bug present at freeze time is enshrined,\nnot caught. This closes the over-trust gap a reader of a green \"value+cardinality\"\ncell would otherwise fall into. Coordinate the wording with the\nreference-independence axis in oracle-coverage-map-reference-independence-disclosure\nso the gate docs and the coverage map tell one consistent story.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the oracle as a regression snapshot (not an independent correctness oracle) wherever it is described", "wid": "w4"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w2"], "notes": "The strongest available independence upgrade: once the digest is type-stable\n(w2), assert that a SECOND SQL engine already in CI (postgres or datafusion)\nproduces the SAME stream-0 digests as DuckDB at SF=1/pinned seed for the 18\nqueries. Two independent engines agreeing on full result values is real\nevidence of value correctness, not a self-snapshot. This requires engine-stable\nrendering (dates, decimals, NULL-as-None vs NaN) and confronts the engine-wide\nemission in sql_execution.py (the F finding). Scope it as either a new\nnon-blocking sample gate or a deferred follow-up; do NOT widen this TODO into\nimplementing a full cross-engine matrix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide and scope an INDEPENDENT value cross-check (cross-engine digest agreement at SF=1)", "wid": "w5"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": [], "notes": "Reproduced: 0 of 18 SF=1 gate result sets contain NULL or mixed-type columns,\nso calculate_checksum's None-safe `_row_sort_key` branch is untested by the\ngate. Add (or confirm existing) unit coverage in the tpchavoc validation tests\nthat a NULL-bearing and a mixed-type column hash deterministically and never\nraise, so a latent ordering bug in None handling cannot hide behind the gate's\nNULL-free inputs. Pure test addition \u2014 no production change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover the None-safe / mixed-type sort path that the gate never exercises", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "only correct the correlation binding.", "dont": "restructure variant/query shapes", "why": "identity is the shape"}, {"INSTEAD": "start as an advisory candidate list.", "dont": "ship a noisy static check as a hard CI blocker", "why": "false positives will erode trust"}], "approach": "Reuse the qualify-the-outer-table discipline from the merged self-binding fix.\nFor DataFrame variants, the analogous fix is correlating on the outer frame's\nkey, not the inner aggregate. Confirm every fix against canonical TPC-H/TPC-DS\nat small SF. Build the static detector last, informed by the concrete cases\nfound, so its heuristics match real instances.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "correlated-subquery-self-binding-cross-surface-audit", "id": 127, "reason": "Depends on the provenance finding in w0/w1; only worth it if the pattern proves systematic across surfaces.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Audit w0-w2 (DONE) found self-binding was NOT systematic: 0 real candidates outside 2 TPC-DS false positives. Stated promotion condition failed.", "summary": "Author a correctness contract for the variant generator"}], "deps": [], "description": "PR #756 fixed three TPC-Havoc SQL variants where a correlated subquery's\nUNQUALIFIED correlation column silently bound to the INNER alias instead of\nthe outer table, degenerating the intended correlation into an uncorrelated\nscan. That sweep covered the SQL variant_sets only, by hand. Two reachable\nsurfaces were never checked, and the manual sweep is not a durable guard.\n\nScope of this audit:\n\n1. TPC-Havoc DataFrame variants (benchbox/core/tpchavoc/dataframe_queries/).\n The DataFrame implementations of the correlated queries (Q2 min-cost,\n Q11 threshold, Q17 per-part avg, Q20, Q21, Q22) must encode the SAME outer\n correlation as the canonical query. DataFrame code does not \"self-bind\" the\n same way, but the equivalent mistake - joining/filtering on the inner frame\n instead of the outer row - is possible and equally silent.\n\n Review evidence (PR #770): the DataFrame variants are INDEPENDENT designs,\n not mirrors of the SQL variants - e.g. DataFrame q9_v10 is an algebraic\n reformulation while SQL q9_v10 had nation-bucketing/clamping defects. None\n of the 24 SQL-side divergences PR #770 fixed has a mirrored DataFrame\n analog, but the same variant ID means a DIFFERENT transformation per\n surface, so the SQL equivalence gate proves nothing about this surface.\n\n NOTE: broad DataFrame result-equivalence vs canonical TPC-H is now owned by\n its own work item, tpchavoc-dataframe-variant-equivalence-gate. THIS audit's\n DataFrame scope is the narrower, targeted self-binding / inner-frame-\n correlation check on the correlated queries (Q2, Q11, Q17, Q20, Q21, Q22);\n run it alongside, or fold it into, that gate rather than duplicating the\n harness.\n\n2. Other SQL benchmarks with correlated subqueries: TPC-DS, SSB, join-order,\n read-primitives (subquery category), and write-primitives\n (update_with_subquery). Grep evidence already shows correlated patterns and\n a \"self-join pattern\" comment in tpcds/dataframe_queries. Apply the same\n qualify-the-outer-correlation audit.\n\n3. A DURABLE detector so this class cannot silently reappear: a static check\n (sqlglot-backed) that flags a correlated subquery whose correlation\n predicate has an unqualified column also present in an inner aliased\n relation. A binder-level check is hard to make exact, so a conservative\n lint that surfaces candidates for human confirmation is acceptable.\n\nProvenance question worth answering during the audit: how were these variants\nauthored? If LLM-generated or mechanically transformed, self-binding is likely\na SYSTEMATIC generation artifact, implying more instances and a generator that\nneeds a correctness contract rather than per-instance patches.", "id": "correlated-subquery-self-binding-cross-surface-audit", "preserves": ["Each variant keeps its declared structural identity; only the correlation binding is corrected.", "Existing DataFrame and SQL benchmark results that are already correct are unchanged."], "prior_art": [{"concept": "_project/TODO/main/planning/tpchavoc-correlated-subquery-self-binding.yaml - the SQL-variant fix this audit extends; reuse its qualify-the-outer-table pattern", "decision": "reuse", "path": "_project/TODO/main/planning/tpchavoc-correlated-subquery-self-binding.yaml"}, {"concept": "benchbox/core/tpcds/dataframe_queries/queries.py:2814 - existing 'self-join pattern' comment for correlated subquery implementation; an explicit audit target", "decision": "reuse", "path": "benchbox/core/tpcds/dataframe_queries/queries.py:2814"}, {"concept": "benchbox/core/tpchavoc/dataframe_queries/_delegating_variants.py - notes that Q16-Q22 delegate the core correlated/anti-join shape; verify those delegations are faithful", "decision": "reuse", "path": "benchbox/core/tpchavoc/dataframe_queries/_delegating_variants.py"}, {"concept": "benchbox/sql_compat/rules/ and benchbox/platforms/*_query_transformer.py - existing places SQL is rewritten; a static check could live near these", "decision": "reuse", "path": "benchbox/sql_compat/rules/"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/dataframe_queries/"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Audit DataFrame variants and other benchmarks for correlated-subquery self-binding, and add a durable detector", "verifications": [{"command": "compare each audited query to its canonical benchmark answer on DuckDB small SF", "description": "Audited correlated DataFrame/SQL queries match canonical results at small SF", "expected": "set-equal results (modulo declared exceptions)", "item_id": "correlated-subquery-self-binding-cross-surface-audit", "last_result": null, "last_run": null, "seq": 1}, {"command": "run the detector over variant_sets with a #756 fix reverted", "description": "Static detector flags the known historical self-binding cases when reverted", "expected": "the reverted variant is flagged", "item_id": "correlated-subquery-self-binding-cross-surface-audit", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": [], "notes": "Done via the detector (w3) used as an enumerator. SQL surfaces swept at\nSF=1, DuckDB dialect: TPC-Havoc 220 variants, TPC-DS 99, SSB 13,\nread-primitives 153, write-primitives 109, join-order 113. Only TPC-DS\nflagged anything (2, both false positives - see w2).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm the audit surfaces still exist and enumerate correlated-subquery sites", "wid": "w0"}, {"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": ["w0"], "notes": "Covered by the merged DataFrame equivalence gate (PR #785,\nbenchbox/core/tpchavoc/dataframe_equivalence.py): it compares every\nDataFrame variant to canonical TPC-H at SF=0.1 and is green (0 divergent).\nAn inner-frame-correlation self-bind would change results and therefore\nsurface as a divergence there, so the DataFrame surface is guarded against\nthis class without a separate per-variant check. No DataFrame drift found.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit TPC-Havoc DataFrame variants for outer-correlation faithfulness and fix any drift", "wid": "w1"}, {"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": ["w0"], "notes": "Swept with the detector. SSB, join-order, read-primitives, write-primitives:\n0 candidates. The detector analyzes UPDATE/DELETE subqueries (the DML\ntarget/USING tables are treated as the outer correlation source), so the\nwrite-primitives \"clean\" result is verified, not a skipped-DML blind spot.\nTPC-DS: 2 candidates (q14, q54), both HUMAN-CONFIRMED FALSE\nPOSITIVES - they are derived-table column aliases that reuse a base-table\ncolumn name (q14 `item.i_item_sk AS ss_item_sk`; q54\n`ws_bill_customer_sk AS customer_sk`), so the bare column binds correctly.\nThese are official TPC-DS spec queries and were NOT modified (conformance).\nNo real self-binds found outside the already-fixed #756 TPC-Havoc cases.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit other SQL benchmarks (TPC-DS, SSB, join-order, read/write-primitives) for the self-binding pattern and fix any found", "wid": "w2"}, {"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": ["w0"], "notes": "Implemented: _project/scripts/detect_self_binding.py (sqlglot-backed,\ncatalog-aware via the benchmark's own CREATE TABLE DDL). Flags a bare\ncolumn in a subquery comparison only when (a) the other operand is a column\nqualified by an inner source alias (correlation shape), and (b) the bare\ncolumn is provided by both an inner and an outer source (ambiguous bind).\nRecognizes derived-table/CTE output columns, so CTE correlation targets\n(the #756 Q17 case) are covered. Locked in by\ntests/unit/test_self_binding_detector.py: it flags all three #756 cases\nwhen reverted, stays clean on the fixed variants and all 220 shipped\nvariants, and ignores plain filters / fully-qualified correlations.\nAdvisory only (not a hard CI blocker), per the anti-pattern guidance.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Durable static detector for unqualified self-binding correlations", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "bump read_model_version in the DDL comment without the matching pipeline change", "instead": "(not specified)", "why": "record it as an item-5 obligation to avoid a half-applied version."}, {"dont": "make funding NULLable in the read model", "instead": "(not specified)", "why": "default 'unspecified' keeps the badge/chip logic total."}, {"dont": "put the vendor/funding derivation logic in a DuckDB view (violates the bare-column G-11 rule)", "instead": "(not specified)", "why": "it belongs in the pipeline/Python."}, {"dont": "re-derive label->visibility here; import the mapping from provenance.py.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Keep the inventory's sidecar-presence inference for internal vs community, and\nlayer the vendor signal on top exactly as the item-3 gate resolves it (e.g.\nbundle under results-data/bundles/vendor/ -> vendor-supplied). Read funding\nfrom the bundle's provenance.funding first, then the manifest, defaulting to\n'unspecified'. For the DDL, funding is a plain column; is_ranking_eligible is\nalready a precomputed boolean, so the only change is which visibility states\nmap to true \u2014 encode that from provenance.py's mapping so the DDL, inventory,\nand pipeline agree. Because the pipeline that POPULATES the DB is gitignored,\nthis item changes the SCHEMA + tests + inventory only and hands the populate\nlogic + read_model_version bump to item 5.\n", "blocked_reason": null, "category": "Feature", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": ["bundle-schema-and-run-cli", "provenance-vocabulary-and-labels", "submit-manifest-and-validator"], "description": "Integrate the two new signals into the explorer's ingest layer so they reach\nthe browser. Two in-repo surfaces change: (1) scripts/generate_corpus_inventory.py,\nwhich today derives trust_label from submission-manifest-sidecar presence\n(_bundle_trust_label, :41-52) and emits per-bundle metadata + a\nby_trust_label summary; (2) docs/development/browser-duckdb-schema.sql, the\nfrozen read-model DDL that already defines trust_label, visibility, and\nis_ranking_eligible on the `results` table and carries trust_label into\nbenchmark_rankings and cohort_metadata.\n\nThis item teaches the inventory to recognize the maintainer-controlled vendor\nsignal (from item 3) and the funding value (from item 2's provenance block /\nitem 3's manifest), adds a `funding` column to the read-model DDL and the\nranking/cohort projections, and sets is_ranking_eligible = true for\nvendor-supplied per decision D2. It records the required read_model_version\nbump as a hand-off obligation for item 5 (the pipeline that writes the DB is\nnot in this checkout).\n", "id": "corpus-inventory-and-read-model", "preserves": ["Bundles with no provenance/funding default to funding='unspecified' and today's trust_label derivation is unchanged for internal/community.", "DuckDB views keep projecting only bare columns (no arithmetic/CASE/window in views); all derivation stays in the pipeline (Python).", "corpus-inventory schema_version is bumped if the entry shape changes (currently '2.2').", "corpus-inventory.json remains deterministic (sorted, timestamp-normalized in --check)."], "prior_art": [{"concept": "_project/scripts/explorer_pipeline/models.py:73-97 RANKING_ELIGIBLE_VISIBILITIES + RANKING_ELIGIBLE_TRUST_LABELS \u2014 supersede (hardcoded sets OMIT public-vendor-reported/vendor-supplied; reconcile with provenance.py so vendor rows rank per D2). Found by item-1 code review.", "decision": "supersede", "path": "_project/scripts/explorer_pipeline/models.py:73-97"}, {"concept": "benchbox/core/results/provenance.py (item 1) \u2014 reference (mapping + funding enum, imported not duplicated)", "decision": "reuse", "path": "benchbox/core/results/provenance.py"}, {"concept": "docs/development/browser-duckdb-schema.sql results table (trust_label/visibility/is_ranking_eligible) + benchmark_rankings + cohort_metadata \u2014 extend (add funding column; wire vendor visibility into ranking eligibility)", "decision": "extend", "path": "docs/development/browser-duckdb-schema.sql"}, {"concept": "scripts/generate_corpus_inventory.py:41-52 _bundle_trust_label + :63-140 extract_metadata/generate_inventory \u2014 extend (add vendor detection, funding extraction, by_funding summary)", "decision": "extend", "path": "scripts/generate_corpus_inventory.py:41-52"}, {"concept": "tests/unit/core/explorer_pipeline/test_pipeline.py:279-465 trust-label sidecar override tests \u2014 extend (add vendor + funding cases)", "decision": "extend", "path": "tests/unit/core/explorer_pipeline/test_pipeline.py:279-465"}, {"concept": "tests/unit/core/explorer_pipeline/test_read_model_contract.py + tests/unit/cli/test_explorer_build_contract.py \u2014 extend (read_model_version + column set)", "decision": "extend", "path": "tests/unit/core/explorer_pipeline/test_read_model_contract.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_project/scripts/explorer_pipeline/ (not in checkout; item 5)"}, {"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "do_not_modify", "path_glob": "results-explorer/ (not in checkout; item 5)"}, {"kind": "only_modify", "path_glob": "docs/development/browser-duckdb-schema.sql"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json (regenerated)"}, {"kind": "only_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "tests/unit/explorer/"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/ (corpus inventory tests)"}], "state": "active", "title": "Surface vendor-supplied label + funding in the corpus inventory and DuckDB read model", "verifications": [{"command": "uv run -- python scripts/generate_corpus_inventory.py --write && uv run -- python -c \"import json; inv=json.load(open('results-data/corpus-inventory.json')); assert 'by_funding' in inv['summary']; assert all('funding' in b for b in inv['bundles'])\"", "description": "Inventory includes funding + by_funding and vendor labels", "expected": "no assertion error", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/generate_corpus_inventory.py --check", "description": "Inventory --check is stable after --write", "expected": "OK: corpus-inventory.json is up-to-date", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ tests/unit/explorer/ -q", "description": "Read-model + explorer pipeline contract tests pass with the new column", "expected": "all pass (or documented xfails pending item 5 pipeline change)", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"import duckdb; con=duckdb.connect(); con.execute(open('docs/development/browser-duckdb-schema.sql').read())\"", "description": "DDL parses", "expected": "no error", "item_id": "corpus-inventory-and-read-model", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "corpus-inventory-and-read-model", "needs": [], "notes": "Resolve via item-1 mapping + item-3 vendor signal (not just community-vs-default); extract funding from the bundle provenance block/manifest; add funding to each entry and a by_funding counter to the summary; retire the local COMMUNITY/DEFAULT literals.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In generate_corpus_inventory.py, resolve trust_label via the item-1 mapping and the item-3 vendor signal, and extract funding from the bundle. Import constants from provenance.py.", "wid": "w1"}, {"evidence": null, "item_id": "corpus-inventory-and-read-model", "needs": ["w1"], "notes": "Propagate into benchmark_rankings and cohort_metadata; ensure the vendor visibility state is treated as ranking-eligible in the is_ranking_eligible derivation (computed in Python per the G-11 views-are-bare-columns rule).", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `funding VARCHAR NOT NULL` to the `results` table in browser-duckdb-schema.sql and propagate it where trust_label travels.", "wid": "w2"}, {"evidence": null, "item_id": "corpus-inventory-and-read-model", "needs": ["w2"], "notes": "Record the required read_model_version bump as an explicit obligation passed to item 5.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update the read-model contract tests + corpus-inventory tests for the new column/summary and the vendor/funding cases.", "wid": "w3"}], "worktree": "results-labels-funding"} +{"anti_patterns": [{"dont": "add a second hash or a per-engine digest variant", "instead": "extend the single primitive.", "why": "calculate_checksum is the one shared definition the TPC-Havoc gates also use"}, {"dont": "describe the value digest as proving 'correctness'", "instead": "call it that everywhere it is documented.", "why": "it is a regression snapshot vs a DuckDB baseline"}, {"dont": "enable cross-engine digest comparison before w2", "instead": "make the digest type/engine-stable first.", "why": "the int-vs-Decimal-vs-float and date/NULL rendering differences guarantee spurious mismatches"}, {"dont": "leave the reference regenerable only by hand-copy", "instead": "ship a deterministic writer.", "why": "an oracle no one can reproduce is the fragility this effort claimed to remove"}, {"dont": "switch to relative precision without re-verifying cross-build stability at the pinned seed and regenerating the reference", "instead": "verify in w1, regenerate in the same change.", "why": "a naive relative rounding near zero can be LESS stable than absolute"}], "approach": "Treat this as fidelity hardening of an already-shipped, CI-enforced oracle, not a\nrebuild. Sequence: (w1) quantify the precision floor as a test and decide\nabsolute-vs-relative; (w2) make the digest type-stable (prerequisite for\nindependence); (w3) replace the hand-copy with an idempotent regeneration target;\n(w4) document the regression-snapshot nature so green cells are not over-read;\n(w5) decide whether a cross-engine agreement check is worth it now; (w6) cover the\nNone-safe sort path the gate never hits. Any change to normalization MUST\nregenerate the 18 reference digests in the same change and prove two-run\ndeterminism, or the gate will go RED on a correct tree.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-24T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "correctness-gate-value-digest-fidelity-followups", "id": 363, "reason": "Out of scope; breadth is owned by benchmark-correctness-oracle-coverage-map. This item is about the FIDELITY of the existing tpch SF=1 value oracle, not its breadth.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Reason explicitly says breadth beyond tpch SF=1 is owned by benchmark-correctness-oracle-coverage-map, which is open (active).", "summary": "Backfill value digests for benchmarks beyond tpch, or for scales above SF=1."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "correctness-gate-value-digest-fidelity-followups", "id": 364, "reason": "w5 decides whether to add ONE bounded cross-engine agreement check or defer; a full matrix is a separate, larger effort (cf. tpchavoc-*-engine-equivalence-sampling).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: correctness-gate-value-digest-fidelity-followups and the tpchavoc cross-dialect/third-engine sampling items are DONE (bounded checks shipped). A full cross-engine matrix is the expansive path the project deliberately avoids; per-benchmark differential oracles owned by the coverage-map item. (tracked by benchmark-correctness-oracle-coverage-map)", "summary": "A full cross-engine value-equivalence matrix across all CI engines."}], "deps": [], "description": "An adversarial falsification review of the value-digest oracle (PR #867,\nreviewed on develop @ e4a04484 \u2014 confirmed an ancestor of origin/develop, all\n18 stored digests reproduced by running `make test-correctness-gate` on the\npinned DuckDB 1.3.2) confirmed the headline win is REAL: a wrong-but-same-\ncardinality TPC-H answer at SF=1 now turns the gate RED, the check is CI-wired\nas a hard blocker (.github/workflows/pr.yml correctness-gate job; the aggregator\nrequires CORRECTNESS_RESULT == success for any code PR), and #867's CI ran it\ngreen for ~11.5 min (not skipped). The strict arming\n(`digest_evaluated == digest_reference_count`) was reproduced locally to require\nall 18 digests to evaluate and match, so a missing digest disarms RED not green.\n\nThe same review reproduced four residual fidelity gaps that the AI-implemented,\nAI-reviewed, AI-merged chain did not surface. None is a correctness regression;\neach narrows what \"value-level\" actually proves and should be either fixed or\nhonestly documented:\n\n- PRECISION FLOOR IS ABSOLUTE, NOT RELATIVE (MEDIUM). The digest normalizes real\n numbers with `f\"{value:.4f}\"` (benchbox/core/results/result_digest.py\n _format_real, _DIGEST_FLOAT_PRECISION=4). Reproduced: an error <5e-5 ABSOLUTE\n in any single cell is invisible regardless of column magnitude\n (delta=4.9e-5 -> identical digest; 5.0e-5 -> differs). On large-magnitude\n columns (TPC-H revenue ~1e10) 5e-5 absolute is ~1e-15 relative \u2014 effectively\n perfect. But on small-magnitude averaged ratio columns it is coarse: Q1's\n `avg_disc` is ~0.049985 (a float), so a sub-5e-5-absolute (~0.1% relative)\n aggregate/rounding bug on that column survives the gate\n (compute_result_digest([(0.05,)]) == compute_result_digest([(0.05004,)])).\n\n- DIGEST CONFLATES VALUE WITH TYPE REPRESENTATION (LOW). _normalize_cell renders\n int exactly (\"1478493\") but float/Decimal at fixed precision\n (\"37734107.0000\"). The SAME numeric value with a different Python type hashes\n DIFFERENTLY. Stable today on pinned DuckDB 1.3.2, but it means the digest is a\n value+type-rendering digest, and it is the blocker for any cross-engine reuse.\n\n- EMISSION IS ENGINE-WIDE BUT THE ORACLE IS DUCKDB-ONLY (LOW). compute_result_digest\n is called both in benchbox/platforms/duckdb.py:999 AND in the shared DBAPI\n helper benchbox/platforms/base/sql_execution.py:80 (sqlite/postgres/datafusion/\n ...). It is harmless today \u2014 `_stored_value_digest` only returns a reference for\n tpch SF=1, so a non-DuckDB digest is computed-but-never-compared \u2014 but combined\n with the type-sensitivity above it is a latent trap for the cross-engine\n independence work below, and it sits outside the original TODO's scope_limit\n (benchbox/platforms/) by the implementer's own admission.\n\n- THE VALUE REFERENCE IS A SELF-SNAPSHOT, NOT AN INDEPENDENT ORACLE, AND HAS NO\n AUTOMATED REGENERATION (MEDIUM). The 18 reference digests in\n benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json\n were produced by running benchbox-on-DuckDB-1.3.2 and HAND-COPYING the emitted\n digests (provenance note says exactly this; no script/make target writes the\n file \u2014 grep confirms only loader.py reads it). So the oracle detects CHANGE\n from the frozen baseline (a genuine regression tripwire) but NOT correctness vs\n an authority outside benchbox: a conceptual value bug present at freeze time is\n enshrined, never caught. This is the same hand-curated-answer-key fragility the\n effort claimed to avoid, one level down. There is also no one-button\n regeneration, so a legitimate DuckDB bump means a manual re-copy.\n\n- THE NONE-SAFE SORT PATH IS UNTESTED BY THE GATE (LOW). calculate_checksum sorts\n rows by the None-safe `_row_sort_key`. Reproduced: 0 of the 18 SF=1 gate result\n sets contain a NULL or mixed-type column, so for the gate's inputs the None-safe\n surrogate is order-identical to a plain sort and its NULL/mixed-type branch\n never fires in the gate. (Note: the None-safe sort PREDATES #867 \u2014 it existed at\n parent ce999bd7 \u2014 so #867 only promoted the helper to module level; this is a\n coverage gap, not a #867 regression.)\n\nThis item fixes or honestly documents each gap. It is the fidelity complement to\nthe value axis shipped in bounded-correctness-gate-value-oracle (#867, DONE); it\ndoes NOT re-do the digest emission, the strict arming, or the row-count work.", "id": "correctness-gate-value-digest-fidelity-followups", "preserves": ["The #867 value axis is unchanged: digest emission behind BENCHBOX_EMIT_RESULT_DIGEST, the strict arming (digest_evaluated == digest_reference_count), and the additive row-count check keep working exactly as today.", "The TPC-Havoc gates that share calculate_checksum stay green \u2014 reuse the one digest definition, never fork it.", "`benchbox run` default behavior, payload shape, and cost stay unchanged when the digest flag is unset (digest emission stays gate-only).", "`make test-correctness-gate` stays green on a correct tree and on the required PR path after any normalization change (regenerate the reference in the same change)."], "prior_art": [{"concept": "Makefile:test-correctness-gate (BENCHBOX_EMIT_RESULT_DIGEST=1, pinned seed 17039360, SF=1, 18 query ids) \u2014 the exact run whose stream-0 digests become the reference; the regeneration target must reuse this configuration, not duplicate it.", "decision": "reuse", "path": "Makefile:test-correctness-gate"}, {"concept": "_project/DONE/main/planning/bounded-correctness-gate-value-oracle.yaml \u2014 the effort this hardens; w3 already proved sensitivity to gross mutations, w1 chose absolute rounding as a non-gating open question. This item revisits that choice with the reproduced floor quantified.", "decision": "reuse", "path": "_project/DONE/main/planning/bounded-correctness-gate-value-oracle.yaml"}, {"concept": "_project/analysis/REVIEW-PROTOCOL.md \u2014 where the self-snapshot-vs-independent-oracle distinction should be documented so a future reviewer does not over-read a green value gate.", "decision": "reuse", "path": "_project/analysis/REVIEW-PROTOCOL.md"}, {"concept": "benchbox/core/expected_results/loader.py:load_tpch_value_digests \u2014 the only reader of the JSON today; a regeneration writer should live near it or as a _project/scripts tool.", "decision": "reuse", "path": "benchbox/core/expected_results/loader.py:load_tpch_value_digests"}, {"concept": "benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json \u2014 the hand-copied reference; the regeneration target must WRITE this file deterministically from a gate run, replacing the manual copy described in its provenance note.", "decision": "reuse", "path": "benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json"}, {"concept": "benchbox/core/results/result_digest.py:_format_real/_normalize_cell/_DIGEST_FLOAT_PRECISION \u2014 the absolute-precision normalization to revisit for the precision-floor and type-stability work; extend, do not fork.", "decision": "extend", "path": "benchbox/core/results/result_digest.py:_format_real/_normalize_cell/_DIGEST_FLOAT_PRECISION"}, {"concept": "benchbox/core/tpchavoc/validation.py:calculate_checksum/_row_sort_key \u2014 the promoted single-result digest primitive and its None-safe sort; reuse for the None-coverage test, do not add a second hash.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py:calculate_checksum/_row_sort_key"}, {"concept": "benchbox/platforms/base/sql_execution.py:80 + benchbox/platforms/duckdb.py:999 \u2014 the two emission sites; the engine-scope and cross-engine-independence decisions touch both.", "decision": "reuse", "path": "benchbox/platforms/base/sql_execution.py:80"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/"}, {"kind": "only_modify", "path_glob": "benchbox/core/expected_results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Harden the bounded-gate value digest: precision floor, type-stability, independence, and auditable regeneration", "verifications": [{"command": "make test-correctness-gate", "description": "Bounded gate stays green and the digest assertion still arms for all 18 queries after any normalization change", "expected": "1 passed, ran=1 skipped=0; row-count and value-digest checks evaluated for all 18 queries", "item_id": "correctness-gate-value-digest-fidelity-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/test_correctness_gate_value_oracle.py -q", "description": "The precision-floor test pins the exact sensitivity boundary", "expected": "tests pass; an error just below the floor is uncaught and just above is caught, per the pinned assertion", "item_id": "correctness-gate-value-digest-fidelity-followups", "last_result": null, "last_run": null, "seq": 2}, {"command": "make correctness-gate-digests-regen && git diff --exit-code benchbox/core/expected_results/reference_digests/tpch_value_digests_sf1.json", "description": "The regeneration target is idempotent on a clean tree", "expected": "no diff (regeneration reproduces the committed reference byte-for-byte)", "item_id": "correctness-gate-value-digest-fidelity-followups", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": [], "notes": "Pin the reproduced floor as a test and make an explicit keep-or-change\ndecision. The floor is 5e-5 ABSOLUTE per cell: invisible below, visible at/above\n(modulo round-half-even at the exact midpoint). Quantify it per gate column\nclass \u2014 large Decimals (revenue) are effectively perfect; small floats\n(Q1 avg_disc ~0.05) are only ~0.1%-relative protected. Decide between:\n(a) keep absolute 4dp and DOCUMENT the small-ratio blind spot with the worst\nexposed column named; or (b) switch to N significant figures / a hybrid\nmax(absolute, relative) so sensitivity is uniform in relative terms, then\nre-verify cross-build stability at the pinned seed and regenerate the reference\n(w4). Record the decision and its rationale next to _format_real.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Quantify the precision floor and decide absolute vs relative/significant-figure normalization", "wid": "w1"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w1"], "notes": "Decide and implement: either canonicalize integer-valued numerics to one\nrepresentation before hashing (so int 37734107, Decimal('37734107.00'), and\nfloat 37734107.0 hash identically) and likewise normalize dates/strings of\nequal logical value, OR explicitly document that the digest is a value+type\ndigest and is therefore DuckDB-pinned. Type-stability is a prerequisite for the\ncross-engine independence option in w5; if w5 is deferred, document instead.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the digest stable across numeric TYPE representation (int vs float vs Decimal of equal value)", "wid": "w2"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w1"], "notes": "Replace the hand-copy described in the JSON provenance note with a script /\nmake target (e.g. `make correctness-gate-digests-regen`) that runs the gate\nconfiguration (BENCHBOX_EMIT_RESULT_DIGEST=1, seed 17039360, SF=1, the 18 query\nids) and writes the digests + provenance block back to the JSON deterministically.\nReuse the Makefile:test-correctness-gate configuration rather than duplicating\nthe seed/scale/query-id list. The target must be idempotent on a clean tree\n(regenerate -> no diff) and must stamp the DuckDB version it ran under.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an auditable, one-command regeneration path that WRITES tpch_value_digests_sf1.json", "wid": "w3"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w3"], "notes": "The Makefile note, tests/README, the JSON provenance, and REVIEW-PROTOCOL.md\nshould state plainly that the value digest detects CHANGE from a benchbox-on-\nDuckDB baseline (a regression tripwire), not correctness against an external\nauthority, and that a conceptual value bug present at freeze time is enshrined,\nnot caught. This closes the over-trust gap a reader of a green \"value+cardinality\"\ncell would otherwise fall into. Coordinate the wording with the\nreference-independence axis in oracle-coverage-map-reference-independence-disclosure\nso the gate docs and the coverage map tell one consistent story.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the oracle as a regression snapshot (not an independent correctness oracle) wherever it is described", "wid": "w4"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": ["w2"], "notes": "The strongest available independence upgrade: once the digest is type-stable\n(w2), assert that a SECOND SQL engine already in CI (postgres or datafusion)\nproduces the SAME stream-0 digests as DuckDB at SF=1/pinned seed for the 18\nqueries. Two independent engines agreeing on full result values is real\nevidence of value correctness, not a self-snapshot. This requires engine-stable\nrendering (dates, decimals, NULL-as-None vs NaN) and confronts the engine-wide\nemission in sql_execution.py (the F finding). Scope it as either a new\nnon-blocking sample gate or a deferred follow-up; do NOT widen this TODO into\nimplementing a full cross-engine matrix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide and scope an INDEPENDENT value cross-check (cross-engine digest agreement at SF=1)", "wid": "w5"}, {"evidence": null, "item_id": "correctness-gate-value-digest-fidelity-followups", "needs": [], "notes": "Reproduced: 0 of 18 SF=1 gate result sets contain NULL or mixed-type columns,\nso calculate_checksum's None-safe `_row_sort_key` branch is untested by the\ngate. Add (or confirm existing) unit coverage in the tpchavoc validation tests\nthat a NULL-bearing and a mixed-type column hash deterministically and never\nraise, so a latent ordering bug in None handling cannot hide behind the gate's\nNULL-free inputs. Pure test addition \u2014 no production change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover the None-safe / mixed-type sort path that the gate never exercises", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "restructure variant/query shapes", "instead": "only correct the correlation binding.", "why": "identity is the shape"}, {"dont": "ship a noisy static check as a hard CI blocker", "instead": "start as an advisory candidate list.", "why": "false positives will erode trust"}], "approach": "Reuse the qualify-the-outer-table discipline from the merged self-binding fix.\nFor DataFrame variants, the analogous fix is correlating on the outer frame's\nkey, not the inner aggregate. Confirm every fix against canonical TPC-H/TPC-DS\nat small SF. Build the static detector last, informed by the concrete cases\nfound, so its heuristics match real instances.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-08T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "correlated-subquery-self-binding-cross-surface-audit", "id": 127, "reason": "Depends on the provenance finding in w0/w1; only worth it if the pattern proves systematic across surfaces.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Audit w0-w2 (DONE) found self-binding was NOT systematic: 0 real candidates outside 2 TPC-DS false positives. Stated promotion condition failed.", "summary": "Author a correctness contract for the variant generator"}], "deps": [], "description": "PR #756 fixed three TPC-Havoc SQL variants where a correlated subquery's\nUNQUALIFIED correlation column silently bound to the INNER alias instead of\nthe outer table, degenerating the intended correlation into an uncorrelated\nscan. That sweep covered the SQL variant_sets only, by hand. Two reachable\nsurfaces were never checked, and the manual sweep is not a durable guard.\n\nScope of this audit:\n\n1. TPC-Havoc DataFrame variants (benchbox/core/tpchavoc/dataframe_queries/).\n The DataFrame implementations of the correlated queries (Q2 min-cost,\n Q11 threshold, Q17 per-part avg, Q20, Q21, Q22) must encode the SAME outer\n correlation as the canonical query. DataFrame code does not \"self-bind\" the\n same way, but the equivalent mistake - joining/filtering on the inner frame\n instead of the outer row - is possible and equally silent.\n\n Review evidence (PR #770): the DataFrame variants are INDEPENDENT designs,\n not mirrors of the SQL variants - e.g. DataFrame q9_v10 is an algebraic\n reformulation while SQL q9_v10 had nation-bucketing/clamping defects. None\n of the 24 SQL-side divergences PR #770 fixed has a mirrored DataFrame\n analog, but the same variant ID means a DIFFERENT transformation per\n surface, so the SQL equivalence gate proves nothing about this surface.\n\n NOTE: broad DataFrame result-equivalence vs canonical TPC-H is now owned by\n its own work item, tpchavoc-dataframe-variant-equivalence-gate. THIS audit's\n DataFrame scope is the narrower, targeted self-binding / inner-frame-\n correlation check on the correlated queries (Q2, Q11, Q17, Q20, Q21, Q22);\n run it alongside, or fold it into, that gate rather than duplicating the\n harness.\n\n2. Other SQL benchmarks with correlated subqueries: TPC-DS, SSB, join-order,\n read-primitives (subquery category), and write-primitives\n (update_with_subquery). Grep evidence already shows correlated patterns and\n a \"self-join pattern\" comment in tpcds/dataframe_queries. Apply the same\n qualify-the-outer-correlation audit.\n\n3. A DURABLE detector so this class cannot silently reappear: a static check\n (sqlglot-backed) that flags a correlated subquery whose correlation\n predicate has an unqualified column also present in an inner aliased\n relation. A binder-level check is hard to make exact, so a conservative\n lint that surfaces candidates for human confirmation is acceptable.\n\nProvenance question worth answering during the audit: how were these variants\nauthored? If LLM-generated or mechanically transformed, self-binding is likely\na SYSTEMATIC generation artifact, implying more instances and a generator that\nneeds a correctness contract rather than per-instance patches.", "id": "correlated-subquery-self-binding-cross-surface-audit", "preserves": ["Each variant keeps its declared structural identity; only the correlation binding is corrected.", "Existing DataFrame and SQL benchmark results that are already correct are unchanged."], "prior_art": [{"concept": "_project/TODO/main/planning/tpchavoc-correlated-subquery-self-binding.yaml - the SQL-variant fix this audit extends; reuse its qualify-the-outer-table pattern", "decision": "reuse", "path": "_project/TODO/main/planning/tpchavoc-correlated-subquery-self-binding.yaml"}, {"concept": "benchbox/core/tpcds/dataframe_queries/queries.py:2814 - existing 'self-join pattern' comment for correlated subquery implementation; an explicit audit target", "decision": "reuse", "path": "benchbox/core/tpcds/dataframe_queries/queries.py:2814"}, {"concept": "benchbox/core/tpchavoc/dataframe_queries/_delegating_variants.py - notes that Q16-Q22 delegate the core correlated/anti-join shape; verify those delegations are faithful", "decision": "reuse", "path": "benchbox/core/tpchavoc/dataframe_queries/_delegating_variants.py"}, {"concept": "benchbox/sql_compat/rules/ and benchbox/platforms/*_query_transformer.py - existing places SQL is rewritten; a static check could live near these", "decision": "reuse", "path": "benchbox/sql_compat/rules/"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/dataframe_queries/"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Audit DataFrame variants and other benchmarks for correlated-subquery self-binding, and add a durable detector", "verifications": [{"command": "compare each audited query to its canonical benchmark answer on DuckDB small SF", "description": "Audited correlated DataFrame/SQL queries match canonical results at small SF", "expected": "set-equal results (modulo declared exceptions)", "item_id": "correlated-subquery-self-binding-cross-surface-audit", "last_result": null, "last_run": null, "seq": 1}, {"command": "run the detector over variant_sets with a #756 fix reverted", "description": "Static detector flags the known historical self-binding cases when reverted", "expected": "the reverted variant is flagged", "item_id": "correlated-subquery-self-binding-cross-surface-audit", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": [], "notes": "Done via the detector (w3) used as an enumerator. SQL surfaces swept at\nSF=1, DuckDB dialect: TPC-Havoc 220 variants, TPC-DS 99, SSB 13,\nread-primitives 153, write-primitives 109, join-order 113. Only TPC-DS\nflagged anything (2, both false positives - see w2).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm the audit surfaces still exist and enumerate correlated-subquery sites", "wid": "w0"}, {"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": ["w0"], "notes": "Covered by the merged DataFrame equivalence gate (PR #785,\nbenchbox/core/tpchavoc/dataframe_equivalence.py): it compares every\nDataFrame variant to canonical TPC-H at SF=0.1 and is green (0 divergent).\nAn inner-frame-correlation self-bind would change results and therefore\nsurface as a divergence there, so the DataFrame surface is guarded against\nthis class without a separate per-variant check. No DataFrame drift found.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit TPC-Havoc DataFrame variants for outer-correlation faithfulness and fix any drift", "wid": "w1"}, {"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": ["w0"], "notes": "Swept with the detector. SSB, join-order, read-primitives, write-primitives:\n0 candidates. The detector analyzes UPDATE/DELETE subqueries (the DML\ntarget/USING tables are treated as the outer correlation source), so the\nwrite-primitives \"clean\" result is verified, not a skipped-DML blind spot.\nTPC-DS: 2 candidates (q14, q54), both HUMAN-CONFIRMED FALSE\nPOSITIVES - they are derived-table column aliases that reuse a base-table\ncolumn name (q14 `item.i_item_sk AS ss_item_sk`; q54\n`ws_bill_customer_sk AS customer_sk`), so the bare column binds correctly.\nThese are official TPC-DS spec queries and were NOT modified (conformance).\nNo real self-binds found outside the already-fixed #756 TPC-Havoc cases.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit other SQL benchmarks (TPC-DS, SSB, join-order, read/write-primitives) for the self-binding pattern and fix any found", "wid": "w2"}, {"evidence": null, "item_id": "correlated-subquery-self-binding-cross-surface-audit", "needs": ["w0"], "notes": "Implemented: _project/scripts/detect_self_binding.py (sqlglot-backed,\ncatalog-aware via the benchmark's own CREATE TABLE DDL). Flags a bare\ncolumn in a subquery comparison only when (a) the other operand is a column\nqualified by an inner source alias (correlation shape), and (b) the bare\ncolumn is provided by both an inner and an outer source (ambiguous bind).\nRecognizes derived-table/CTE output columns, so CTE correlation targets\n(the #756 Q17 case) are covered. Locked in by\ntests/unit/test_self_binding_detector.py: it flags all three #756 cases\nwhen reverted, stays clean on the fixed variants and all 220 shipped\nvariants, and ignores plain filters / fully-qualified correlations.\nAdvisory only (not a hard CI blocker), per the anti-pattern guidance.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Durable static detector for unqualified self-binding correlations", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality - Cost Estimation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Conducted comprehensive adversarial review of cost estimation framework implementation identifying 18 defects. Completed 4-phase remediation plan with production-quality implementations (no placeholders). Enhanced cost calculation accuracy, improved reliability with validation and error handling, added granular regional pricing for \u00b15% accuracy target, and completed type safety verification. Framework now production-ready with comprehensive test coverage.", "id": "cost-estimation-framework-adversarial-review-correction", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Cost Estimation Framework - Adversarial Review & Correction", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Cost Intelligence", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Build rule-based cost optimizer that generates actionable recommendations\nwith savings estimates and implementation guides.", "id": "cost-optimization-engine-automated-recommendations", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Cost Optimization Engine - Automated Recommendations", "verifications": [], "work": [], "worktree": "cost-intelligence-optimizer"} -{"anti_patterns": [{"INSTEAD": "either test or refactor for testability", "dont": "add broad deferred exclusions", "why": "this repeats temporary-split debt"}, {"INSTEAD": "use deterministic unit tests", "dont": "add slow/live integration tests for coverage points", "why": "CI must stay fast/offline"}, {"INSTEAD": "assert structure and behavior", "dont": "assert exact SQL formatting when intent-level checks suffice", "why": "formatting churn creates flaky tests"}, {"INSTEAD": "close missing branches with tests", "dont": "lower thresholds or weaken the checker", "why": "that hides quality gaps"}], "approach": "Use wave-by-wave, file-targeted unit tests first, refactoring production code\nonly when required for testability. For each wave:\n 1) identify exact uncovered branches from term-missing output,\n 2) add focused tests in tests/unit matching existing patterns,\n 3) run targeted pytest selection,\n 4) run coverage-per-file and record deltas.\nFavor mocks/fakes over integration setup. Keep each wave independently\ncommittable with explicit module-level coverage deltas.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-10T00:00:00Z", "completed_pr": null, "created_at": "2026-02-09T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-below-50-modules-wave-plan", "id": 128, "reason": "Goal is full closure of currently below-50 modules, not further scope deferral", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: This is a negative scope note ('do not add new deferred entries in this campaign'), not an actionable deferred task.", "summary": "Do not add new permanent deferred coverage list entries in this campaign"}], "deps": [], "description": "Coverage snapshot from coverage.json (generated 2026-02-09) still shows 75\nbenchbox/*.py modules below 50% line coverage. This TODO defines a detailed,\nwave-based implementation plan to bring each of those modules to >=50% while\npreserving fast, deterministic unit-test execution.\n\nPlanning basis:\n - Source report: coverage.json + scripts/check_per_file_coverage.py\n - Current low-water marks include 0%-20% in reporting/query helpers,\n TPC-DS/TPC-H dataframe query modules, legacy ETL modules, and selected\n CLI/platform adapters.\n\nCompletion target:\n - No benchbox/*.py module below 50% in the standard coverage-per-file run\n - keep per-module gate reproducible in CI/local\n - preserve existing CLI/platform behavior", "id": "coverage-below-50-modules-wave-plan", "preserves": ["Current CLI command contracts, option names, and help output semantics remain unchanged", "Existing fast/unit suite remains deterministic and does not require cloud credentials", "No destructive changes to benchmark query semantics or result schemas", "TPC-DI SQL/DataFrame ETL parity and adapter behavior remain intact"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_binaries/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Raise remaining benchbox modules below 50% to >=50% coverage", "verifications": [{"command": "uv run -- python -m pytest tests/unit -m fast --tb=short", "description": "Targeted module wave tests pass", "expected": "0 failures", "item_id": "coverage-below-50-modules-wave-plan", "last_result": null, "last_run": null, "seq": 1}, {"command": "make coverage-per-file", "description": "Per-file coverage gate passes with no module below 50%", "expected": "No active-scope files below threshold", "item_id": "coverage-below-50-modules-wave-plan", "last_result": null, "last_run": null, "seq": 2}, {"command": "make ci-test", "description": "CI-oriented fast checks still green", "expected": "all configured checks pass", "item_id": "coverage-below-50-modules-wave-plan", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generate canonical backlog artifact of all <50% modules from coverage.json", "wid": "w1"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w5", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave H: TPC-DS/TPC-H large dataframe query modules and reporting modules to >=50%", "wid": "w10"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w6", "w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave I: remaining analytics/concurrency/cloud-storage low modules to >=50%", "wid": "w11"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w10", "w11", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full make coverage-per-file and capture zero-failure report artifact", "wid": "w12"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Address stragglers from final run, commit by wave, and keep per-wave before/after module snapshots", "wid": "w13"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w13"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Close TODO when all modules reach >=50%, validate TODO metadata, and move to DONE", "wid": "w14"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify backlog into effort tiers: tiny (<120 stmts), medium (120-400), large (>400)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave A: lift tiny 0%-20% helper/entry files to >=50% with focused import+error-path tests", "wid": "w3"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave B: lift tiny 20%-49% files (results/runner/query parser helpers) with branch-targeted tests", "wid": "w4"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave C: core benchmark wrappers (amplab/ssb/h2odb/clickbench/nyctaxi) to >=50%", "wid": "w5"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave D: CLI low-coverage commands (results/run/plot/exceptions) to >=50%", "wid": "w6"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave E: platform adapters below 50% (fabric_warehouse, influxdb client/setup, redshift, databricks, bigquery, clickhouse workload)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave F: metadata/read/write primitives modules below 50%", "wid": "w8"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w5", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave G: TPC-DI low modules (etl pipeline/batch/transformations/source_generators/scd + remaining generators) to >=50%", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep the fast tests to pure/deterministic logic and leave timing behavior to the existing slow integration test.", "dont": "attempt to characterize real multi-threaded timing/concurrency behavior of core/throughput/runner.py in fast tests", "why": "that inherently requires wall-clock-sensitive execution unsuitable for a fast deterministic lane"}, {"INSTEAD": "verify with --durations=0.", "dont": "mark test_etl_scd_processor.py or new throughput tests 'fast' without measuring actual runtime first", "why": "mis-marking a slow test as fast defeats the purpose of the lane split and can make the required PR gate flaky/slow"}, {"INSTEAD": "if they come back, that's a new decision for a new commit, not a reason to keep stale references now.", "dont": "re-add omit entries for transformations.py / batch.py 'just in case'", "why": "they were deliberately deleted as dead code by #637/#640"}], "approach": "Start with w1 (pure config cleanup, zero risk, unblocks nothing but is\ntrivial to land first). w2 and w3 are independent and can proceed in\nparallel - both are \"add or re-mark fast tests for an under-tested\nmodule\" with the same shape. Measure actual runtime with\n`pytest --durations=0` before deciding fast vs. keep-medium for w2's\nexisting test file. w4 depends on w2+w3 landing since it removes the\nomit entry only once real fast-lane coverage exists to replace it.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-04T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [], "deps": [], "description": "Test-strategy review of coverage configuration found real coverage\nblind spots, but the originally-cited facts needed correction after\nreading the current tree - two of the three \"omitted\" TPC-DI ETL files\nno longer exist:\n\n`.coveragerc_core:16-18` currently reads:\n\n benchbox/core/tpcdi/etl/transformations.py\n benchbox/core/tpcdi/etl/batch.py\n benchbox/core/tpcdi/etl/scd_processor.py\n\n- `transformations.py` was deleted in commit `0fbbd431` (\"chore: shrink\n dead TPC-DI transformations (#640)\").\n- `batch.py` was deleted in commit `5b15277b` (\"chore: shrink dead TPC-DI\n ETL batch scaffold (#637)\").\n- Both omit entries are now stale/dead config referring to files that\n don't exist - harmless (coverage's `omit` glob just matches nothing)\n but should be cleaned up so the file doesn't misrepresent what's\n actually being excluded from coverage.\n- `scd_processor.py` DOES still exist (951 lines,\n `benchbox/core/tpcdi/etl/scd_processor.py`) and IS still omitted from\n coverage entirely. It has a dedicated unit test file\n (`tests/unit/core/tpcdi/test_etl_scd_processor.py`, marked\n `pytest.mark.unit, pytest.mark.medium` - not `fast`) plus exercise via\n `tests/integration/test_tpcdi_phase3_benchmark.py` (`integration,\n medium`) and `tests/integration/test_tpcdi_full_benchmark.py`\n (`integration, slow`). None of these run in the fast lane\n (`-m \"fast and not (slow or ...)\"`, `.github/workflows/pr.yml:450`,\n `--cov-fail-under=70`), so even without the coverage omit,\n `scd_processor.py` contributes zero coverage to the number CI actually\n gates on - the omit just also hides it from the full/nightly coverage\n report.\n\nSeparately, `benchbox/core/throughput/` (`runner.py` 163 lines,\n`result.py` 72 lines - imported by\n`benchbox/core/tpch/throughput_test.py` and\n`benchbox/core/tpcds/throughput_test.py`) is NOT in the `.coveragerc_core`\nomit list, so it's nominally measured - but its only real exerciser,\n`tests/integration/test_tpch_throughput_test.py`, is marked\n`pytest.mark.integration, pytest.mark.slow`. There is no\n`fast`-marked test that imports or exercises `core/throughput` at all,\nso its real fast-lane coverage is effectively zero even though the\n90-line-ish module isn't formally excluded.\n\nNet effect: the 70% `--cov-fail-under` floor\n(`.github/workflows/pr.yml:450`) is measured only on the fast lane, and\ntwo real modules (`scd_processor.py`, `core/throughput/`) have no\nfast-lane characterization tests protecting them from regressions\nbetween releases, even though one of them isn't even coverage-omitted.", "id": "coverage-blindspot-burndown", "preserves": ["Existing medium/slow/integration TPC-DI and throughput tests (test_etl_scd_processor.py if left as medium, test_tpcdi_phase3_benchmark.py, test_tpcdi_full_benchmark.py, test_tpch_throughput_test.py) continue to pass unchanged.", "No behavior change to benchbox/core/tpcdi/etl/scd_processor.py or benchbox/core/throughput/ - this is a test-and-config-only TODO.", "The 70% --cov-fail-under floor in .github/workflows/pr.yml:450 continues to pass after adding new fast tests (new tests should raise coverage, not just add runtime)."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/pr.yml"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/throughput/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcdi/etl/scd_processor.py"}, {"kind": "only_modify", "path_glob": ".coveragerc_core"}, {"kind": "only_modify", "path_glob": "tests/unit/core/throughput/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_etl_scd_processor.py"}], "state": "done", "title": "Coverage config hygiene: drop stale .coveragerc_core omits, add fast-lane tests for TPC-DI SCD processor and core/throughput", "verifications": [{"command": "! grep -n 'transformations.py\\|batch.py' .coveragerc_core", "description": ".coveragerc_core no longer references deleted files", "expected": "Exit 0 (grep found no match; bare grep exits 1 on no-match, so it must be inverted for this to be a pass/fail gate)", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_etl_scd_processor.py -m fast -v", "description": "New/re-marked fast tests for scd_processor.py pass and are collected in the fast lane", "expected": "Tests collected and pass under the fast marker", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/throughput/ -m fast -v", "description": "New fast tests for core/throughput pass and are collected in the fast lane", "expected": "Tests collected and pass under the fast marker", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests -m \"fast and not (slow or stress or resource_heavy or live_integration)\" --cov=benchbox --cov-report=term-missing --cov-fail-under=70", "description": "Fast-lane coverage floor still passes with the coveragerc omissions removed", "expected": "Coverage >= 70%, all fast tests pass", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": [], "notes": "Delete the `benchbox/core/tpcdi/etl/transformations.py` and\n`benchbox/core/tpcdi/etl/batch.py` lines from `.coveragerc_core`'s\n`omit` list (lines 16-17) - both files were removed by #640 and #637\nrespectively and no longer exist. Confirm no other file resurrects\nthose paths before deleting (`git log --all -- ` should show\nonly the deletion commits as the most recent history).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the two stale .coveragerc_core omit entries for deleted files", "wid": "w1"}, {"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": [], "notes": "`tests/unit/core/tpcdi/test_etl_scd_processor.py` already exists and\nis well-scoped (104 lines) but marked `medium`, not `fast`. Two\noptions, pick based on actual runtime once measured:\n 1. If the existing medium tests are individually fast (likely,\n given SCD logic is pure in-memory transformation, not I/O), just\n re-mark the file's `pytestmark` to `fast` after confirming\n runtime with `--durations=0`.\n 2. If the existing tests are genuinely slower (e.g. they spin up a\n DataFrame/DB fixture), add a small new `fast`-marked test module\n covering the core SCD Type-2 branching logic\n (insert-new-version / expire-old-version / no-op-unchanged)\n against minimal in-memory fixtures, leaving the existing\n `medium` file for broader integration-shaped cases.\nWhichever path, the fast lane must end up covering scd_processor.py.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add fast-marked characterization tests for benchbox/core/tpcdi/etl/scd_processor.py", "wid": "w2"}, {"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": [], "notes": "`benchbox/core/throughput/runner.py` (163 lines) and `result.py`\n(72 lines) currently have no test coverage outside the slow-marked\n`tests/integration/test_tpch_throughput_test.py`. Add a small\n`fast`-marked unit test module (e.g.\n`tests/unit/core/throughput/test_runner.py`) covering the runner's\npure logic paths with a mocked/fake benchmark object (mirroring the\npattern already used in test_tpch_throughput_test.py's\n`benchmark.get_query.return_value = \"SELECT 1\"` style mocking, but\nwithout the `slow` marker) - concurrency/throughput timing behavior,\nresult aggregation in `result.py`, and error/edge-case handling\n(zero queries, single-thread fallback, etc.). Do not attempt to\ncharacterize actual multi-threaded timing behavior in the fast lane -\nthat's what the existing slow integration test is for; focus fast\ntests on deterministic logic (aggregation math, config validation,\nresult-shape correctness).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add fast-marked characterization tests for benchbox/core/throughput/", "wid": "w3"}, {"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": ["w2", "w3"], "notes": "Once w2/w3 land fast-lane coverage for scd_processor.py (and\ncore/throughput if it turns out to need adding to the omit list\npending w3 - it currently is NOT omitted, so no removal needed\nthere), remove the now-covered `scd_processor.py` line from\n`.coveragerc_core`'s omit list. For every omit entry that remains\n(the dataframe_queries/queries.py-style generated-query files,\n`benchbox/experimental/concurrency/*`), add a one-line rationale\ncomment above it in `.coveragerc_core` explaining WHY it's\ndeliberately excluded (e.g. \"# generated/templated SQL construction,\nexercised via golden-output tests, not line coverage\" or\n\"# experimental, not on any release-gate path\") so future readers\ndon't have to archaeology their way to an answer the way this TODO\nhad to.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the (now-clean) coveragerc omissions and document any remaining deliberate ones", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use only @pytest.mark.fast", "dont": "add @pytest.mark.slow, @pytest.mark.live_integration, or @pytest.mark.resource_heavy markers", "why": "those markers cause tests to be excluded from the fast lane and won't count toward the 80% coverage gate"}, {"INSTEAD": "keep per-module or per-group test files aligned with existing test directory structure", "dont": "batch all 25 modules into a single test file", "why": "monolithic test files make coverage diffs unreadable and slow down targeted reruns"}, {"INSTEAD": "(not specified)", "dont": "call list() on color_cycle() return values", "why": "color_cycle() returns an infinite itertools.cycle; use list(theme.palette or DEFAULT_PALETTE) instead"}, {"INSTEAD": "mock all external SDK calls with unittest.mock.patch", "dont": "import or instantiate real cloud SDK clients in tests", "why": "CI has no cloud credentials and live calls cause flakiness and violate the fast-lane contract"}, {"INSTEAD": "only add/extend test files", "dont": "modify production source files to make them more testable", "why": "changes to production code require their own review cycle and could introduce regressions that dwarf the coverage gain"}, {"INSTEAD": "(not specified)", "dont": "use -o 'addopts=' to override pytest.ini", "why": "this strips -n auto and makes tests 10x slower"}], "approach": "Original work units (w1-w14) - completed except w3:\n w1-w4 and w6-w9 are fully independent. w5 before w10 (mock patterns).\n w8 \u2192 w13, w12 \u2192 w14 (file extension chains).\n\nNew work units (w15-w24) - dependency chains:\n w15 depends on w4, w2, w5 (extends their test files)\n w16 depends on w2, w9 (extends comparison and cloud adapter tests)\n w17 depends on w6, w10 (extends staging and fabric tests)\n w18 depends on w1, w6 (extends core primitive tests)\n w19 depends on w7, w5 (extends spark and credential tests)\n w20 depends on w10 (extends snowflake test file from w10)\n w21 depends on w11 (extends bigquery test file from w11)\n w22 depends on w13 (extends databricks test file from w8/w13)\n w23 depends on w14 (extends redshift test file from w12/w14)\n w24 depends on w23 (extends redshift further)\n\nParallelism: w15-w19 are independent of each other (different files).\nw20-w22 are independent of each other.\nw23 must complete before w24.\n\nCritical: verify per-module coverage after each work unit, not just overall.\nUse: make coverage-fast 2>&1 | grep \"module_path\" to check individual modules.\n\nFor each work unit:\n1. Run coverage on the target module first to identify the exact uncovered lines:\n uv run -- python -m pytest --cov=benchbox/ --cov-report=term-missing -m fast -q\n2. Add tests targeting the uncovered lines\n3. Verify module coverage \u226580% before committing\n4. Run the full fast smoke suite to confirm no regressions\n\nRecommended mock setup for cloud SDKs:\n- boto3: patch at \"benchbox.platforms.redshift.boto3\" etc. (not at boto3 root)\n- google.cloud: use MagicMock() for both bigquery.Client and storage.Client\n- snowflake.connector: patch \"benchbox.platforms.snowflake.snowflake.connector.connect\"\n- databricks.sql: patch \"benchbox.platforms.databricks.adapter.databricks\"\n- pyspark: patch \"benchbox.platforms.dataframe.pyspark_df.SparkSession\"\n- pyodbc: patch \"benchbox.platforms.fabric_warehouse.pyodbc.connect\"\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-02T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Current fast-test coverage sits at 80.04% (85,262 / 106,518 lines) - just 47\ncovered lines above the 80% threshold. The buffer is dangerously thin: a single\nbad merge could drop us below the gate.\n\nThis item systematically brings the 25 lowest-coverage modules up to \u226580% each,\nadding robustness to the overall coverage floor and reducing the risk of accidental\nregression below the gate.\n\nCoverage gaps (lines needed to reach 80% per module):\n databricks/dataframe_adapter.py 23.57% \u2192 +89 lines\n dataframe/pyspark_df.py 28.06% \u2192 +145 lines\n azure/spark_execution_utils.py 34.78% \u2192 +21 lines\n databricks/adapter.py 41.68% \u2192 +396 lines\n bigquery.py 42.15% \u2192 +321 lines\n redshift.py 42.98% \u2192 +435 lines\n spark.py 46.04% \u2192 +133 lines\n snowflake.py 46.43% \u2192 +245 lines\n dataframe/lakesail_df.py 48.94% \u2192 +88 lines\n credentials/redshift.py 50.96% \u2192 +121 lines\n tpcds/generator/streaming.py 52.20% \u2192 +57 lines\n motherduck.py 52.60% \u2192 +42 lines\n base/cloud_spark/staging.py 53.24% \u2192 +99 lines\n tpchavoc/benchmark.py 53.51% \u2192 +30 lines\n credentials/snowflake.py 54.26% \u2192 +48 lines\n fabric_warehouse.py 55.68% \u2192 +109 lines\n clickhouse/diagnostics.py 56.18% \u2192 +21 lines\n transaction_primitives/dataframe_operations.py 56.18% \u2192 +89 lines\n tpcdi/financial_data.py 56.33% \u2192 +38 lines\n starburst.py 56.73% \u2192 +24 lines\n comparison/suite.py 56.97% \u2192 +56 lines\n results/anonymization.py 57.20% \u2192 +59 lines\n write_primitives/benchmark.py 57.35% \u2192 +109 lines\n gcp/dataproc_adapter.py 58.14% \u2192 +56 lines\n aws/athena_spark_adapter.py 58.66% \u2192 +60 lines\n\nTotal: ~2,891 new covered lines across 12 work units (original estimate).\n\n--- Progress Review (2026-04-02) ---\n\nAfter 13 of 14 original work units (w1-w2, w4-w14), overall coverage rose from\n80.04% to 81%. However, only 3 of 25 target modules actually reached 80%:\n\n Module Before \u2192 After Status\n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n results/anonymization.py 57.20% \u2192 89% \u2705 HIT\n tpchavoc/benchmark.py 53.51% \u2192 85% \u2705 HIT\n starburst.py 56.73% \u2192 82% \u2705 HIT\n motherduck.py 52.60% \u2192 77% Near (needs +4)\n tpcds/generator/streaming.py 52.20% \u2192 73% Near (needs +15)\n comparison/suite.py 56.97% \u2192 65% Gap (needs +37)\n gcp/dataproc_adapter.py 58.14% \u2192 66% Gap (needs +37)\n aws/athena_spark_adapter.py 58.66% \u2192 63% Gap (needs +47)\n credentials/snowflake.py 54.26% \u2192 55% Gap (needs +46)\n transaction_primitives/dataframe_ops 56.18% \u2192 56% Gap (needs +88)\n clickhouse/diagnostics.py 56.18% \u2192 57% Gap (needs +20)\n write_primitives/benchmark.py 57.35% \u2192 58% Gap (needs +104)\n fabric_warehouse.py 55.68% \u2192 59% Gap (needs +96)\n base/cloud_spark/staging.py 53.24% \u2192 54% Gap (needs +97)\n spark.py 46.04% \u2192 50% Gap (needs +116)\n credentials/redshift.py 50.96% \u2192 52% Gap (needs +117)\n snowflake.py 46.43% \u2192 54% Gap (needs +186)\n databricks/adapter.py 41.68% \u2192 53% Gap (needs +274)\n bigquery.py 42.15% \u2192 48% Gap (needs +273)\n redshift.py 42.98% \u2192 48% Gap (needs +372)\n dataframe/pyspark_df.py 28.06% \u2192 28% w3 pending\n dataframe/lakesail_df.py 48.94% \u2192 49% w3 pending\n databricks/dataframe_adapter.py 23.57% \u2192 24% w3 pending\n\nRoot cause: the original work units under-estimated the number of new covered\nlines required. Large adapter modules (bigquery, redshift, databricks, snowflake)\nhave deep call trees with many branches that need dedicated test scenarios for\neach path through connection setup, data loading, query execution, SQL generation,\nand tuning application.\n\nAdditional work units w15-w24 added below to close the remaining ~1,744 lines\nof coverage gap across the 19 modules that haven't yet reached 80%.\nCombined with w3 (~322 lines), total remaining work is ~2,066 new covered lines.\n\nAll tests MUST carry the @pytest.mark.fast marker so they run in the fast lane.\nAll external SDKs (databricks, pyspark, snowflake-connector, boto3, google-cloud,\npyodbc, duckdb, requests) are mocked - no live connections.", "id": "coverage-bottom-25-modules-to-80pct", "preserves": ["@pytest.mark.fast marker present on every new test function", "All existing passing tests continue to pass - zero regressions", "No live network calls - all external SDK clients mocked with unittest.mock.patch", "No production source files modified - tests only", "Overall fast-lane coverage stays \u226580.04% after every individual work unit commit"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "tests/unit/core/results/test_anonymization.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_comparison.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_financial_data_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcds/generator/test_streaming_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/transaction_primitives/test_dataframe_ops_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/write_primitives/test_benchmark_execution.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/aws/test_athena_spark_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/azure/test_spark_execution_utils.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/cloud_spark/test_staging.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/credentials/test_redshift_defaults.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/credentials/test_snowflake_defaults.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/databricks/test_databricks_df_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_lakesail_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_pyspark_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/gcp/test_dataproc_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_bigquery_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_clickhouse_cloud_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databricks_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_microsoft_fabric_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_motherduck.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_redshift_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_snowflake_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_spark_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_starburst.py"}], "state": "done", "title": "Bring the 25 lowest-coverage modules to 80% fast-test coverage", "verifications": [{"command": "uv run -- python -m pytest -m fast -q --cov=benchbox/platforms/databricks --cov=benchbox/platforms/dataframe --cov=benchbox/platforms/azure --cov=benchbox/platforms/bigquery --cov=benchbox/platforms/redshift --cov=benchbox/platforms/spark --cov=benchbox/platforms/snowflake --cov=benchbox/platforms/credentials --cov=benchbox/platforms/motherduck --cov=benchbox/platforms/base/cloud_spark --cov=benchbox/platforms/fabric_warehouse --cov=benchbox/platforms/clickhouse --cov=benchbox/platforms/starburst --cov=benchbox/platforms/gcp --cov=benchbox/platforms/aws --cov=benchbox/core/tpcds/generator --cov=benchbox/core/tpchavoc --cov=benchbox/core/comparison --cov=benchbox/core/results/anonymization --cov=benchbox/core/write_primitives --cov=benchbox/core/transaction_primitives --cov=benchbox/core/tpcdi --cov-report=term-missing 2>&1 | grep -E '(FAILED|ERROR|PASSED|[0-9]+%)'", "description": "All 25 target modules individually reach \u226580% coverage in the fast lane", "expected": "All target modules show \u226580% coverage; no FAILED or ERROR lines", "item_id": "coverage-bottom-25-modules-to-80pct", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q --cov=benchbox --cov-report=term 2>&1 | tail -5", "description": "Overall fast-lane coverage stays at or above 80.04%", "expected": "Total coverage \u2265 80%", "item_id": "coverage-bottom-25-modules-to-80pct", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q 2>&1 | tail -5", "description": "No regressions - full fast test suite passes", "expected": "passed, 0 failed", "item_id": "coverage-bottom-25-modules-to-80pct", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pure-logic core utilities: financial_data, dataframe_operations, anonymization, spark_execution_utils", "wid": "w1"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "fabric_warehouse.py and snowflake.py: auth logic and connection config", "wid": "w10"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "bigquery.py: GCS integration, auth, and query execution branches", "wid": "w11"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 1: connector selection, connection setup, S3 URI parsing", "wid": "w12"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "databricks/adapter.py part 2: data loading, DBFS upload, error handling", "wid": "w13"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 2: data loading flow, SQL generation, WLM query group", "wid": "w14"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w2", "w4", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Small top-ups: motherduck, streaming, clickhouse/diagnostics, credentials/snowflake", "wid": "w15"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w2", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Mid-tier adapters part 2: comparison/suite, dataproc, athena_spark", "wid": "w16"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w10", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud infrastructure: staging.py providers + fabric_warehouse data loading", "wid": "w17"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w1", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Core primitives part 2: transaction dataframe_ops + write_primitives benchmark", "wid": "w18"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w5", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "spark.py part 2 + credentials/redshift part 2", "wid": "w19"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "TPC generators and comparison suite: streaming, tpchavoc, comparison/suite", "wid": "w2"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "snowflake.py part 2: connection, external tables, query execution, tuning", "wid": "w20"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "bigquery.py part 2: metadata, validation, loading, execution, tuning", "wid": "w21"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w13"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "databricks/adapter.py part 3: from_config, UC volumes, data loading, tuning", "wid": "w22"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w14"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 3: from_config, platform info, drop_database, S3 loading", "wid": "w23"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 4: external tables, query execution, statistics, tuning", "wid": "w24"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DataFrame expression builders: pyspark_df, lakesail_df, databricks DF adapter", "wid": "w3"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Small cloud adapters: motherduck, starburst, clickhouse diagnostics", "wid": "w4"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Credential modules: credentials/snowflake and credentials/redshift", "wid": "w5"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud staging URI parsing and write-primitives validation logic", "wid": "w6"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "spark.py: deployment modes, config validation, resource allocation", "wid": "w7"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "databricks/adapter.py part 1: warehouse selection, clustering, SQL generation", "wid": "w8"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "AWS Athena Spark + GCP Dataproc: state polling and job lifecycle", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use only stdlib (xml.etree, pathlib, argparse)", "dont": "add external dependencies to the coverage script", "why": "it runs in CI where install time matters"}, {"INSTEAD": "exit 0 with a warning if the file is absent", "dont": "make the script fail on missing coverage.xml", "why": "not all CI runs generate it"}], "approach": "Write a standalone Python script (no external dependencies beyond stdlib)\nthat uses xml.etree.ElementTree to parse coverage.xml. Group files by\ntheir first two path components (e.g., benchbox/cli, benchbox/core/tpch).\nPrint a table of packages with coverage, highlight failures, exit 1 if\nany package is below threshold. Make threshold configurable via CLI arg.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": ["coverage-cli-modules", "coverage-core-benchmarks", "coverage-core-tpcdi", "coverage-core-utilities", "coverage-platforms"], "description": "Once all modules reach \u226550% coverage (via the companion TODO items),\nadd CI enforcement to prevent regressions below the floor.\n\nCurrent CI uses --cov-fail-under=50 which checks OVERALL coverage only.\nThis allows individual modules to drop to 0% as long as the aggregate\nstays above 50%. A per-module check would catch this.\n\nOptions:\n 1. Use coverage.py's [report] fail_under per-package (limited granularity)\n 2. Add a custom script that parses coverage.xml and fails on any module <50%\n 3. Use diff-cover to enforce coverage on changed files only\n\nRecommend option 2: a lightweight Python script in scripts/ that parses\ncoverage.xml, groups by top-level package, and fails if any package is\nbelow threshold.", "id": "coverage-ci-enforcement", "preserves": ["Existing passing CI jobs remain passing with no increase in runtime >20%", "ci-test Makefile target continues to run pytest with existing flags and pass"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}], "state": "done", "title": "Enforce per-module 50% coverage floor in CI", "verifications": [{"command": "uv run -- python scripts/check_module_coverage.py --coverage-xml coverage.xml --threshold 50 --exclude-module benchbox/data --exclude-module benchbox/examples", "description": "Script correctly identifies modules below threshold", "expected": "table of per-module coverage with pass/fail indicators", "item_id": "coverage-ci-enforcement", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_module_coverage.py --coverage-xml coverage.xml --threshold 99 --exclude-module benchbox/data --exclude-module benchbox/examples; echo $?", "description": "Script exits non-zero when modules fail", "expected": "exit code 1 (since no module is at 99%)", "item_id": "coverage-ci-enforcement", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write scripts/check_module_coverage.py that parses coverage.xml", "wid": "w1"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Parse coverage.xml, group by benchbox/* top-level package", "wid": "w2"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Report per-package coverage with pass/fail against 50% threshold", "wid": "w3"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Exit non-zero if any package fails", "wid": "w4"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for the coverage check script itself", "wid": "w5"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add check_module_coverage.py to Makefile as coverage-check target", "wid": "w6"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add to ci-test target in Makefile after pytest coverage run", "wid": "w7"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update GitHub Actions workflow to run the check", "wid": "w8"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "mock the runner with unittest.mock.patch", "dont": "launch actual benchmark runs in CLI tests", "why": "benchmark runs take 30+ seconds and require platform credentials"}, {"INSTEAD": "test return values, exit codes, and side effects instead", "dont": "test by parsing CLI stdout strings", "why": "stdout format changes would cause false failures"}], "approach": "Follow existing patterns in tests/unit/cli/. Most CLI modules can be tested\nby invoking Click commands with CliRunner or by testing individual functions\ndirectly. For compare commands, create small mock result files. For tuning.py,\ntest YAML config parsing and parameter resolution. For onboarding.py, mock\nuser input and filesystem operations.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Several CLI modules are below 50% test coverage. These handle user-facing\ncommands, configuration, output formatting, and error handling.\n\nCurrent coverage breakdown:\n - cli/commands/compare.py: 30.2% (623 missing)\n - cli/commands/run.py: 49.8% (516 missing) - nearly there\n - cli/commands/shell.py: 30.0% (301 missing)\n - cli/commands/compare_plans.py: 11.6% (229 missing)\n - cli/commands/compare_dataframes.py: 12.7% (200 missing)\n - cli/tuning.py: 0.0% (397 missing)\n - cli/config.py: 47.6% (267 missing) - nearly there\n - cli/onboarding.py: 0.0% (192 missing)\n - cli/output.py: 49.0% (177 missing) - nearly there\n - cli/exceptions.py: 36.9% (178 missing)\n - cli/preferences.py: 19.2% (126 missing)\n - cli/composite_params.py: 46.4% (120 missing)\n - cli/presentation/system.py: 16.7% (20 missing)\n - cli/system.py: 43.3% (17 missing)\n\nThree modules (run.py, config.py, output.py) are very close to 50% and\nneed minimal effort. The zero-coverage modules (tuning.py, onboarding.py)\nneed foundational test suites.", "id": "coverage-cli-modules", "preserves": ["All existing CLI tests continue to pass", "CLI --help output remains unchanged", "No changes to production code"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}], "state": "done", "title": "Improve test coverage for CLI modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/ -v --tb=short", "description": "All CLI tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-cli-modules", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/run.py (49.8% \u2192 50%)", "wid": "w1"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/compare_plans.py (11.6% \u2192 50%)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/compare_dataframes.py (12.7% \u2192 50%)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/exceptions.py (36.9% \u2192 50%)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/preferences.py (19.2% \u2192 50%)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/presentation/system.py (16.7% \u2192 50%)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/config.py (47.6% \u2192 50%)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/output.py (49.0% \u2192 50%)", "wid": "w3"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/composite_params.py (46.4% \u2192 50%)", "wid": "w4"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/system.py (43.3% \u2192 50%)", "wid": "w5"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/tuning.py (0% \u2192 50%)", "wid": "w6"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/onboarding.py (0% \u2192 50%)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/compare.py (30.2% \u2192 50%)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/shell.py (30.0% \u2192 50%)", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use mocks and in-memory fixtures", "dont": "create integration tests requiring live databases", "why": "CI has no database servers and tests must run in <1s"}, {"INSTEAD": "write tests for code as-is", "dont": "modify production code to improve coverage", "why": "the goal is to verify existing behavior, not refactor"}, {"INSTEAD": "test query structure, table references, and parameters instead", "dont": "test auto-generated SQL strings character-by-character", "why": "SQL formatting changes would cause false failures"}], "approach": "Follow existing test patterns in tests/unit/benchmarks/ (e.g., test_tpch_core.py,\ntest_tpcds_core.py). Use @pytest.mark.fast for tests <1s, @pytest.mark.unit for\nall. Mock external dependencies (file I/O, database connections, TPC binaries).\nFor dataframe_queries files, test a representative subset of queries (5-10 per\nbenchmark) rather than exhaustive coverage.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-11T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Core benchmark module coverage has been refreshed from coverage.json generated\non 2026-02-10. Most originally-targeted files are now >=50% from prior waves.\n\nRemaining files below 50% that still require targeted unit tests:\n - benchbox/core/ssb/benchmark.py: 48%\n - benchbox/core/nyctaxi/benchmark.py: 49%\n - benchbox/core/metadata_primitives/benchmark.py: 47%\n\nThis item now focuses on closing only these remaining gaps to >=50% while\npreserving existing benchmark behavior and avoiding production code changes.", "id": "coverage-core-benchmarks", "preserves": ["All existing tests continue to pass", "No changes to production code - tests only"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/integration/"}, {"kind": "only_modify", "path_glob": "tests/unit/benchmarks/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Improve test coverage for core benchmark modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/ -v --tb=short -m fast", "description": "All new tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-core-benchmarks", "last_result": null, "last_run": null, "seq": 1}, {"command": "make coverage-per-file && for f in benchbox/core/ssb/benchmark.py benchbox/core/nyctaxi/benchmark.py benchbox/core/metadata_primitives/benchmark.py; do jq -r --arg f \"$f\" '.files[$f].summary.percent_covered_display' coverage.json; done", "description": "Coverage for each targeted module reaches \u226550%", "expected": "each printed value is >=50", "item_id": "coverage-core-benchmarks", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpch/reporting.py (0% \u2192 50%)", "wid": "w1"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/clickbench/benchmark.py (25.8% \u2192 50%)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/amplab/benchmark.py (18.4% \u2192 50%)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/amplab/generator.py (29.1% \u2192 50%)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/nyctaxi/benchmark.py (49% \u2192 50%)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/nyctaxi/downloader.py (20.5% \u2192 50%)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/h2odb/benchmark.py (28.8% \u2192 50%)", "wid": "w15"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/read_primitives/dataframe_queries.py (16.5% \u2192 50%)", "wid": "w16"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/transaction_primitives/generator.py (12.9% \u2192 50%)", "wid": "w17"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/transaction_primitives/benchmark.py (24.5% \u2192 50%)", "wid": "w18"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/metadata_primitives/generator.py (8.0% \u2192 50%)", "wid": "w19"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpch/maintenance_test.py (12.8% \u2192 50%)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/metadata_primitives/benchmark.py (47% \u2192 50%)", "wid": "w20"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/datavault/dataframe_queries/queries.py (6.7% \u2192 50%)", "wid": "w21"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/joinorder/generator.py (20.1% \u2192 50%)", "wid": "w22"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpch/parameter_extractor.py (23.9% \u2192 50%)", "wid": "w3"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/reporting.py (0% \u2192 50%)", "wid": "w4"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/maintenance_test.py (20.6% \u2192 50%)", "wid": "w5"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/throughput_test.py (23.8% \u2192 50%)", "wid": "w6"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/benchmark/runner.py (32.7% \u2192 50%)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/ssb/benchmark.py (48% \u2192 50%)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/ssb/generator.py (45.0% \u2192 50%)", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "mock data generation calls", "dont": "require dsqgen binary", "why": "it is platform-specific, not available in CI, and segfaults on fractional scale factors"}, {"INSTEAD": "create minimal synthetic fixtures (5-10 rows)", "dont": "test with real TPC-DI data files", "why": "they require the dsqgen binary and are 100MB+ at SF=1, making tests slow and environment-dependent"}], "approach": "The ETL transformations are mostly pure functions that transform DataFrames\nor dictionaries - ideal for unit testing. Create test fixtures with small\nsample data (5-10 rows) that exercise the transform logic. For SCD processor,\ntest Type 1 and Type 2 SCD operations with before/after state assertions.\nFor batch.py, mock the database connection and test orchestration logic.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "This item was completed by validating that existing TPC-DI test coverage\nalready satisfies the >=50% targets for the scoped files and package-level\nthreshold, without requiring production or test-code changes in this item.\n\nVerified coverage results at completion:\n - core/tpcdi/etl/transformations.py: 61%\n - core/tpcdi/etl/scd_processor.py: 76%\n - core/tpcdi/source_generators.py: 99%\n - core/tpcdi/etl/batch.py: 68%\n - core/tpcdi/benchmark.py: 54%\n\nAll TPC-DI unit tests in scope passed, and package-level coverage exceeded\nthe 50% minimum.", "id": "coverage-core-tpcdi", "preserves": ["All existing TPC-DI tests continue to pass", "No changes to production code"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "do_not_modify", "path_glob": "tests/integration/"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/coverage-core-tpcdi.yaml"}], "state": "done", "title": "Improve test coverage for core/tpcdi module to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcdi -v --tb=short", "description": "TPC-DI unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-core-tpcdi", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi --cov=benchbox/core/tpcdi --cov-report=term --cov-fail-under=50 -q", "description": "core/tpcdi package coverage reaches >=50%", "expected": "coverage check passes with total >=50%", "item_id": "coverage-core-tpcdi", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi -p pytest_cov --cov=benchbox/core/tpcdi --cov-report=json:coverage_tpcdi.json --cov-report=term --cov-fail-under=0 -q && for f in benchbox/core/tpcdi/etl/transformations.py benchbox/core/tpcdi/etl/scd_processor.py benchbox/core/tpcdi/source_generators.py benchbox/core/tpcdi/etl/batch.py benchbox/core/tpcdi/benchmark.py; do v=$(jq -r --arg f \"$f\" '.files[$f].summary.percent_covered_display' coverage_tpcdi.json); echo \"$f: $v\"; awk -v n=\"$v\" 'BEGIN { exit !(n+0 >= 50) }' || exit 1; done", "description": "Each targeted TPC-DI file reaches >=50% coverage", "expected": "each listed file prints a value >=50 and command exits 0", "item_id": "coverage-core-tpcdi", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "coverage-core-tpcdi", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Establish scoped verification baseline and fixture expectations for TPC-DI coverage review", "wid": "w1"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/etl/transformations.py meets >=50% coverage target", "wid": "w2"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/etl/scd_processor.py meets >=50% coverage target", "wid": "w3"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/source_generators.py meets >=50% coverage target", "wid": "w4"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/etl/batch.py meets >=50% coverage target", "wid": "w5"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/benchmark.py meets >=50% coverage target", "wid": "w6"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run final package + per-file coverage verification and capture completion evidence", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "mock I/O using unittest.mock or pytest tmp_path fixture", "dont": "add tests that depend on filesystem state or network", "why": "CI environments vary and network tests are flaky"}, {"INSTEAD": "test via public methods and documented interfaces only", "dont": "test private implementation details", "why": "private APIs can change without notice, breaking tests"}], "approach": "Start with zero-coverage modules since any test at all is progress.\ncore/tpc_metrics.py likely has pure functions for metric calculation - test\nwith known inputs/outputs. core/comparison/ is a self-contained comparison\nengine - test the comparison logic with mock result objects. For release/sync.py,\nmock filesystem and git operations. utils/lazy_loader.py is a module-level\nlazy import utility - test import mechanics.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Several standalone core modules and utility files have zero or low test\ncoverage. These are smaller, self-contained modules that should be\nstraightforward to test.\n\nZero-coverage modules (quick wins):\n - core/tpc_metrics.py: 0% (402 missing)\n - core/comparison/ (suite, types, plotter): 0% (346 missing)\n - core/platform_config_utils.py: 0% (102 missing)\n - core/reports.py: 0% (26 missing)\n - release/sync.py: 0% (259 missing)\n - utils/lazy_loader.py: 0% (70 missing)\n\nLow-coverage modules:\n - core/dryrun.py: 49.3% (265 missing) - nearly there\n - utils/cloud_storage.py: 37.5% (185 missing)\n - utils/config_helpers.py: 37.8% (69 missing)\n - examples/dryrun_utils.py: 15.9% (53 missing)\n - mcp/resources/registry.py: 27.0% (65 missing)\n - mcp/prompts/registry.py: 40.4% (34 missing)\n - mcp/__init__.py: 39.1% (14 missing)\n\nPriority: zero-coverage modules first (biggest bang-for-buck), then\nmodules near 50%.", "id": "coverage-core-utilities", "preserves": ["All existing tests continue to pass", "No changes to production code"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Improve test coverage for core utility and standalone modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/ -k 'tpc_metrics or comparison or platform_config or reports or sync or lazy_loader' -v", "description": "New tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-core-utilities", "last_result": null, "last_run": null, "seq": 1}], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "assert concrete outputs, SQL fragments, state transitions, and error text instead.", "dont": "add blanket import/smoke tests", "why": "they create coverage theater"}, {"INSTEAD": "close the gap with executable tests.", "dont": "change coverage policy to make the denominator smaller", "why": "that hides risk rather than reducing it"}, {"INSTEAD": "land one work unit at a time with module-local verification.", "dont": "mix multiple heavyweight adapters into one implementation commit", "why": "diagnosis and rollback get harder"}, {"INSTEAD": "keep those changes in the publishing stream.", "dont": "touch `benchbox/core/publishing/**`, `benchbox/cli/commands/submit.py`, or `results-explorer/` in this item", "why": "that surface is still moving"}, {"INSTEAD": "patch SDK clients, Spark builders, subprocesses, file I/O, and network probes.", "dont": "use live cloud accounts, real Spark clusters, or real sockets to raise unit coverage", "why": "the fast lane must remain hermetic"}], "approach": "Use the April 2, 2026 filtered ranking as the fixed baseline for this wave.\nSequence the work so shared fixture layers are built once and reused:\nDataFrame foundation (w2) before DataFrame transaction/query work (w3-w4),\ncloud-Spark mixins (w9) before Spark/LakeSail (w10-w11), and transaction\nsemantics (w3) before the write-primitives benchmark paths (w18).\n\nHeavyweight adapters are intentionally isolated one module per work unit\n(w7-w16, w18-w19) so each implementation step can land as a single-purpose\ncommit with module-local rollback.\n\nExtend existing coverage-focused test files wherever the repo already has them.\nOnly add new dedicated test files when current coverage is scattered across generic\ntest modules and a focused file will make the missing branches easier to own.\nPlanned-new files are named explicitly with `Add ...`; existing files to be\nextended are named explicitly with `Extend ...`.\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-03T00:00:00Z", "completed_pr": null, "created_at": "2026-04-02T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "id": 129, "reason": "Explicitly excluded from this item: `benchbox/core/publishing/**/*.py` and `benchbox/cli/commands/submit.py` are still in flight and should be planned against the publishing productization TODOs instead of this coverage wave.", "resolution": "promoted", "resolved_item": "results-publishing-coverage-work", "resolved_reason": null, "summary": "Results-publishing coverage work"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "id": 130, "reason": "Anything already excluded by `[tool.coverage.run].omit`, `[tool.coverage.report].exclude_lines`, or `pragma: no cover` is intentionally out of scope; this item targets only measured statements.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason states coverage-ignored/omit-listed lines are intentionally out of scope by design, not deferred work needing action.", "summary": "Coverage-ignored lines and omit-listed modules"}], "deps": [], "description": "Fresh baseline from `.coverage` rendered via `uv run -- python -m coverage json`\nat 2026-04-02 19:03:25 local time shows overall fast-lane coverage at 82.87%\n(88,387 / 106,663 measured statements).\n\nThis ranking already excludes coverage-ignore lines from the denominator:\n - `[tool.coverage.run].omit` entries are not in the measured file set\n - `[tool.coverage.report].exclude_lines` and `pragma: no cover` are not in\n `num_statements`\n\nThis plan also explicitly excludes the in-flight results-publishing surface:\n - `benchbox/core/publishing/**/*.py`\n - `benchbox/cli/commands/submit.py`\n\nFiltered bottom-25 modules and current gap to 80%:\n platforms/databricks/dataframe_adapter.py 23.57% -> +89 lines\n platforms/spark.py 46.04% -> +133 lines\n platforms/credentials/redshift.py 51.91% -> +118 lines\n core/transaction_primitives/dataframe_operations.py 56.45% -> +88 lines\n platforms/clickhouse/diagnostics.py 57.30% -> +21 lines\n core/write_primitives/benchmark.py 58.39% -> +105 lines\n core/tuning/generators/pg_mooncake.py 59.09% -> +5 lines\n metadata_primitives.py 59.38% -> +7 lines\n core/tpcds/generator/manager.py 59.63% -> +45 lines\n core/ssb/generator.py 59.78% -> +55 lines\n platforms/databricks/credentials.py 60.22% -> +37 lines\n core/tpcdi/etl/error_recovery.py 60.34% -> +58 lines\n core/datavault/benchmark.py 60.49% -> +32 lines\n experimental/gpu/benchmark.py 60.87% -> +40 lines\n platforms/base/cloud_spark/mixins.py 61.01% -> +61 lines\n platforms/aws/glue_adapter.py 61.15% -> +49 lines\n platforms/dataframe/iceberg_maintenance.py 61.46% -> +36 lines\n platforms/questdb.py 61.61% -> +94 lines\n experimental/aiml_functions/benchmark.py 61.66% -> +36 lines\n platforms/lakesail.py 61.68% -> +62 lines\n core/coffeeshop/dataframe_queries/queries.py 61.81% -> +47 lines\n platforms/firebolt.py 62.17% -> +121 lines\n platforms/dataframe/expression_family.py 62.18% -> +69 lines\n platforms/clickhouse_cloud.py 62.41% -> +75 lines\n core/amplab/dataframe_queries/queries.py 62.50% -> +31 lines\n\nTotal remaining gap across the filtered set: about 1,514 newly covered lines.\n\nSequencing rules for this item:\n1. Harvest low-cost pure/helper wins first.\n2. Build reusable DataFrame and cloud-Spark fixtures before the adapters that\n depend on them.\n3. Keep heavyweight adapters isolated one per work unit when the branch\n surface is wide enough to justify dedicated review and rollback.\n4. Do not spend time on results-publishing code until its productization TODO\n stabilizes the public surface.", "id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "preserves": ["All new tests stay in the fast lane: mark them `@pytest.mark.fast` when appropriate and keep every scenario hermetic.", "Assertions must verify actual SQL strings, config values, returned dataclasses, or error messages; import-only tests do not count.", "Existing adapter behavior and public CLI/config contracts remain unchanged.", "No new coverage ignore rules, omit entries, or `pragma: no cover` annotations are added to hit the target.", "Results-publishing code stays untouched in this item."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/publishing/**"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml coverage omit/exclude settings"}, {"kind": "do_not_modify", "path_glob": "results-explorer/**"}, {"kind": "do_not_modify", "path_glob": "unrelated production modules outside the 25-file target set"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/coverage-current-bottom-25-modules-to-80pct-apr-2026.yaml"}, {"kind": "only_modify", "path_glob": "tests/integration/** only if a named unit explicitly calls for an existing helper adjustment"}, {"kind": "only_modify", "path_glob": "tests/unit/** for the named target modules"}, {"kind": "only_modify", "path_glob": "the named production modules only when a tiny test seam is unavoidable"}], "state": "done", "title": "Bring the current 25 lowest-coverage non-publishing modules to 80% fast-test coverage", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tuning/generators/test_pg_mooncake_ddl.py tests/unit/platforms/test_clickhouse_diagnostics.py -q -p pytest_cov --cov=benchbox/core/tuning/generators/pg_mooncake.py --cov=benchbox/platforms/clickhouse/diagnostics.py --cov-report=term-missing", "description": "w1 targeted coverage: pg_mooncake generator and ClickHouse diagnostics", "expected": "Both named modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/test_expression_family.py tests/unit/platforms/dataframe/test_expression_family_behavioral.py tests/unit/platforms/dataframe/test_expression_family_coverage.py tests/unit/platforms/dataframe/test_iceberg_maintenance.py tests/unit/platforms/dataframe/test_iceberg_maintenance_coverage.py -q -p pytest_cov --cov=benchbox/platforms/dataframe/expression_family.py --cov=benchbox/platforms/dataframe/iceberg_maintenance.py --cov-report=term-missing", "description": "w2 targeted coverage: DataFrame foundation modules", "expected": "Both foundation modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/transaction_primitives/test_dataframe_operations.py tests/unit/core/transaction_primitives/test_dataframe_ops_coverage.py -q -p pytest_cov --cov=benchbox/core/transaction_primitives/dataframe_operations.py --cov-report=term-missing", "description": "w3 targeted coverage: transaction DataFrame operations", "expected": "The transaction DataFrame operations module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core/coffeeshop/test_coffeeshop_dataframe_queries.py tests/unit/core/amplab/test_amplab_dataframe_queries.py -q -p pytest_cov --cov=benchbox/core/coffeeshop/dataframe_queries/queries.py --cov=benchbox/core/amplab/dataframe_queries/queries.py --cov-report=term-missing", "description": "w4 targeted coverage: CoffeeShop and AMPLab DataFrame query registries", "expected": "Both registry modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/tpcds/generator/test_manager.py tests/unit/generators/test_ssb_manifest.py tests/unit/core/ssb/test_ssb_generator.py tests/unit/core/datavault/test_datavault_benchmark.py -q -p pytest_cov --cov=benchbox/core/tpcds/generator/manager.py --cov=benchbox/core/ssb/generator.py --cov=benchbox/core/datavault/benchmark.py --cov-report=term-missing", "description": "w5 targeted coverage: TPC-DS manager, SSB generator, and DataVault benchmark", "expected": "All three generator or wrapper modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/gpu/test_gpu_benchmark.py tests/unit/core/aiml_functions/test_aiml_functions_benchmark.py -q -p pytest_cov --cov=benchbox/experimental/gpu/benchmark.py --cov=benchbox/experimental/aiml_functions/benchmark.py --cov-report=term-missing", "description": "w6 targeted coverage: experimental benchmark wrappers", "expected": "Both experimental benchmark wrappers report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest tests/unit/platforms/databricks/test_databricks_df_adapter.py tests/unit/platforms/databricks/test_databricks_df_adapter_coverage.py -q -p pytest_cov --cov=benchbox/platforms/databricks/dataframe_adapter.py --cov-report=term-missing", "description": "w7 targeted coverage: Databricks DataFrame adapter", "expected": "The Databricks DataFrame adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 7}, {"command": "uv run -- python -m pytest tests/unit/platforms/credentials/test_databricks_defaults.py -q -p pytest_cov --cov=benchbox/platforms/databricks/credentials.py --cov-report=term-missing", "description": "w8 targeted coverage: Databricks credentials", "expected": "The Databricks credentials module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 8}, {"command": "uv run -- python -m pytest tests/unit/platforms/cloud_spark/test_spark_ddl_mixin.py tests/unit/platforms/base/cloud_spark/test_config.py -q -p pytest_cov --cov=benchbox/platforms/base/cloud_spark/mixins.py --cov-report=term-missing", "description": "w9 targeted coverage: cloud-Spark mixins", "expected": "The cloud-Spark mixins module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 9}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_spark_adapter.py tests/unit/platforms/test_spark_adapter_coverage.py -q -p pytest_cov --cov=benchbox/platforms/spark.py --cov-report=term-missing", "description": "w10 targeted coverage: core Spark adapter", "expected": "The core Spark adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 10}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_lakesail_adapter.py -q -p pytest_cov --cov=benchbox/platforms/lakesail.py --cov-report=term-missing", "description": "w11 targeted coverage: LakeSail adapter", "expected": "The LakeSail adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 11}, {"command": "uv run -- python -m pytest tests/unit/platforms/aws/test_glue_adapter.py -q -p pytest_cov --cov=benchbox/platforms/aws/glue_adapter.py --cov-report=term-missing", "description": "w12 targeted coverage: Glue adapter", "expected": "The Glue adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 12}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_firebolt_adapter.py tests/unit/platforms/test_firebolt_cloud.py -q -p pytest_cov --cov=benchbox/platforms/firebolt.py --cov-report=term-missing", "description": "w13 targeted coverage: Firebolt adapter", "expected": "The Firebolt adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 13}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_cloud_coverage.py -q -p pytest_cov --cov=benchbox/platforms/clickhouse_cloud.py --cov-report=term-missing", "description": "w14 targeted coverage: ClickHouse Cloud adapter", "expected": "The ClickHouse Cloud adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 14}, {"command": "uv run -- python -m pytest tests/unit/platforms/credentials/test_redshift_defaults.py tests/unit/cli/test_cli_setup.py -q -p pytest_cov --cov=benchbox/platforms/credentials/redshift.py --cov-report=term-missing", "description": "w15 targeted coverage: Redshift credential and diagnostics branches", "expected": "The Redshift credentials module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 15}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_questdb_adapter.py tests/unit/core/tuning/test_questdb_ddl_generator.py -q -p pytest_cov --cov=benchbox/platforms/questdb.py --cov-report=term-missing", "description": "w16 targeted coverage: QuestDB adapter", "expected": "The QuestDB adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 16}, {"command": "uv run -- python -m pytest tests/unit/core/metadata_primitives/test_metadata_primitives_core.py -q -p pytest_cov --cov=benchbox/metadata_primitives.py --cov-report=term-missing", "description": "w17 targeted coverage: metadata_primitives facade", "expected": "The metadata_primitives facade reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 17}, {"command": "uv run -- python -m pytest tests/unit/core/write_primitives/test_benchmark_execution.py tests/unit/core/write_primitives/test_benchmark_operations.py tests/unit/benchmarks/test_write_primitives_validation.py -q -p pytest_cov --cov=benchbox/core/write_primitives/benchmark.py --cov-report=term-missing", "description": "w18 targeted coverage: write_primitives benchmark", "expected": "The write_primitives benchmark module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 18}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_error_recovery.py tests/unit/tpcdi/test_phase3_etl_enhanced.py -q -p pytest_cov --cov=benchbox/core/tpcdi/etl/error_recovery.py --cov-report=term-missing", "description": "w19 targeted coverage: TPC-DI error recovery", "expected": "The TPC-DI error recovery module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 19}, {"command": "make coverage-fast", "description": "The project fast lane still passes with coverage enabled.", "expected": "Fast tests pass and total measured coverage remains >= 80%.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 20}, {"command": "uv run -- python -m coverage json -o /tmp/coverage_bottom25_verify.json && uv run -- python -c \"import json; targets = [ 'benchbox/platforms/databricks/dataframe_adapter.py', 'benchbox/platforms/spark.py', 'benchbox/platforms/credentials/redshift.py', 'benchbox/core/transaction_primitives/dataframe_operations.py', 'benchbox/platforms/clickhouse/diagnostics.py', 'benchbox/core/write_primitives/benchmark.py', 'benchbox/core/tuning/generators/pg_mooncake.py', 'benchbox/metadata_primitives.py', 'benchbox/core/tpcds/generator/manager.py', 'benchbox/core/ssb/generator.py', 'benchbox/platforms/databricks/credentials.py', 'benchbox/core/tpcdi/etl/error_recovery.py', 'benchbox/core/datavault/benchmark.py', 'benchbox/experimental/gpu/benchmark.py', 'benchbox/platforms/base/cloud_spark/mixins.py', 'benchbox/platforms/aws/glue_adapter.py', 'benchbox/platforms/dataframe/iceberg_maintenance.py', 'benchbox/platforms/questdb.py', 'benchbox/experimental/aiml_functions/benchmark.py', 'benchbox/platforms/lakesail.py', 'benchbox/core/coffeeshop/dataframe_queries/queries.py', 'benchbox/platforms/firebolt.py', 'benchbox/platforms/dataframe/expression_family.py', 'benchbox/platforms/clickhouse_cloud.py', 'benchbox/core/amplab/dataframe_queries/queries.py']; data = json.load(open('/tmp/coverage_bottom25_verify.json')); missing = [t for t in targets if t not in data['files']]; below = [(t, round(data['files'][t]['summary']['percent_covered'], 2)) for t in targets if t in data['files'] and data['files'][t]['summary']['percent_covered'] < 80]; assert not missing, missing; assert not below, below; print('all 25 target modules >= 80%')\"", "description": "Recompute the filtered module ranking from the coverage JSON after the final fast-lane run.", "expected": "The command exits cleanly and prints `all 25 target modules >= 80%`.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 21}], "work": [{"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Low-cost helper top-ups: pg_mooncake DDL generator and ClickHouse diagnostics", "wid": "w1"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Core Spark adapter branches", "wid": "w10"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "LakeSail adapter branches", "wid": "w11"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Glue adapter orchestration branches", "wid": "w12"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Firebolt adapter loading and fallback branches", "wid": "w13"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "ClickHouse Cloud adapter external-table branches", "wid": "w14"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Redshift credential and diagnostics branches", "wid": "w15"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "QuestDB adapter branches and ILP helpers", "wid": "w16"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Metadata primitives facade coverage", "wid": "w17"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write-primitives benchmark execution paths", "wid": "w18"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "TPC-DI error recovery engine", "wid": "w19"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DataFrame foundation: expression_family and iceberg_maintenance", "wid": "w2"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Transaction DataFrame operations: capabilities, format validation, atomic paths", "wid": "w3"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DataFrame query registries: CoffeeShop and AMPLab expression/pandas parity", "wid": "w4"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generator and wrapper flow: TPC-DS manager, SSB generator, DataVault benchmark", "wid": "w5"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Experimental benchmark wrappers: GPU and AIML functions", "wid": "w6"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Databricks DataFrame adapter branches", "wid": "w7"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Databricks credential defaults and validation branches", "wid": "w8"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud-Spark DDL and config mixins", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add broad deferred exclusions", "instead": "either test or refactor for testability", "why": "this repeats temporary-split debt"}, {"dont": "add slow/live integration tests for coverage points", "instead": "use deterministic unit tests", "why": "CI must stay fast/offline"}, {"dont": "assert exact SQL formatting when intent-level checks suffice", "instead": "assert structure and behavior", "why": "formatting churn creates flaky tests"}, {"dont": "lower thresholds or weaken the checker", "instead": "close missing branches with tests", "why": "that hides quality gaps"}], "approach": "Use wave-by-wave, file-targeted unit tests first, refactoring production code\nonly when required for testability. For each wave:\n 1) identify exact uncovered branches from term-missing output,\n 2) add focused tests in tests/unit matching existing patterns,\n 3) run targeted pytest selection,\n 4) run coverage-per-file and record deltas.\nFavor mocks/fakes over integration setup. Keep each wave independently\ncommittable with explicit module-level coverage deltas.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-10T00:00:00Z", "completed_pr": null, "created_at": "2026-02-09T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-below-50-modules-wave-plan", "id": 128, "reason": "Goal is full closure of currently below-50 modules, not further scope deferral", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: This is a negative scope note ('do not add new deferred entries in this campaign'), not an actionable deferred task.", "summary": "Do not add new permanent deferred coverage list entries in this campaign"}], "deps": [], "description": "Coverage snapshot from coverage.json (generated 2026-02-09) still shows 75\nbenchbox/*.py modules below 50% line coverage. This TODO defines a detailed,\nwave-based implementation plan to bring each of those modules to >=50% while\npreserving fast, deterministic unit-test execution.\n\nPlanning basis:\n - Source report: coverage.json + scripts/check_per_file_coverage.py\n - Current low-water marks include 0%-20% in reporting/query helpers,\n TPC-DS/TPC-H dataframe query modules, legacy ETL modules, and selected\n CLI/platform adapters.\n\nCompletion target:\n - No benchbox/*.py module below 50% in the standard coverage-per-file run\n - keep per-module gate reproducible in CI/local\n - preserve existing CLI/platform behavior", "id": "coverage-below-50-modules-wave-plan", "preserves": ["Current CLI command contracts, option names, and help output semantics remain unchanged", "Existing fast/unit suite remains deterministic and does not require cloud credentials", "No destructive changes to benchmark query semantics or result schemas", "TPC-DI SQL/DataFrame ETL parity and adapter behavior remain intact"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "_binaries/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Raise remaining benchbox modules below 50% to >=50% coverage", "verifications": [{"command": "uv run -- python -m pytest tests/unit -m fast --tb=short", "description": "Targeted module wave tests pass", "expected": "0 failures", "item_id": "coverage-below-50-modules-wave-plan", "last_result": null, "last_run": null, "seq": 1}, {"command": "make coverage-per-file", "description": "Per-file coverage gate passes with no module below 50%", "expected": "No active-scope files below threshold", "item_id": "coverage-below-50-modules-wave-plan", "last_result": null, "last_run": null, "seq": 2}, {"command": "make ci-test", "description": "CI-oriented fast checks still green", "expected": "all configured checks pass", "item_id": "coverage-below-50-modules-wave-plan", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generate canonical backlog artifact of all <50% modules from coverage.json", "wid": "w1"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w5", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave H: TPC-DS/TPC-H large dataframe query modules and reporting modules to >=50%", "wid": "w10"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w6", "w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave I: remaining analytics/concurrency/cloud-storage low modules to >=50%", "wid": "w11"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w10", "w11", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full make coverage-per-file and capture zero-failure report artifact", "wid": "w12"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Address stragglers from final run, commit by wave, and keep per-wave before/after module snapshots", "wid": "w13"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w13"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Close TODO when all modules reach >=50%, validate TODO metadata, and move to DONE", "wid": "w14"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify backlog into effort tiers: tiny (<120 stmts), medium (120-400), large (>400)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave A: lift tiny 0%-20% helper/entry files to >=50% with focused import+error-path tests", "wid": "w3"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave B: lift tiny 20%-49% files (results/runner/query parser helpers) with branch-targeted tests", "wid": "w4"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave C: core benchmark wrappers (amplab/ssb/h2odb/clickbench/nyctaxi) to >=50%", "wid": "w5"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave D: CLI low-coverage commands (results/run/plot/exceptions) to >=50%", "wid": "w6"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave E: platform adapters below 50% (fabric_warehouse, influxdb client/setup, redshift, databricks, bigquery, clickhouse workload)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave F: metadata/read/write primitives modules below 50%", "wid": "w8"}, {"evidence": null, "item_id": "coverage-below-50-modules-wave-plan", "needs": ["w5", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wave G: TPC-DI low modules (etl pipeline/batch/transformations/source_generators/scd + remaining generators) to >=50%", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "attempt to characterize real multi-threaded timing/concurrency behavior of core/throughput/runner.py in fast tests", "instead": "keep the fast tests to pure/deterministic logic and leave timing behavior to the existing slow integration test.", "why": "that inherently requires wall-clock-sensitive execution unsuitable for a fast deterministic lane"}, {"dont": "mark test_etl_scd_processor.py or new throughput tests 'fast' without measuring actual runtime first", "instead": "verify with --durations=0.", "why": "mis-marking a slow test as fast defeats the purpose of the lane split and can make the required PR gate flaky/slow"}, {"dont": "re-add omit entries for transformations.py / batch.py 'just in case'", "instead": "if they come back, that's a new decision for a new commit, not a reason to keep stale references now.", "why": "they were deliberately deleted as dead code by #637/#640"}], "approach": "Start with w1 (pure config cleanup, zero risk, unblocks nothing but is\ntrivial to land first). w2 and w3 are independent and can proceed in\nparallel - both are \"add or re-mark fast tests for an under-tested\nmodule\" with the same shape. Measure actual runtime with\n`pytest --durations=0` before deciding fast vs. keep-medium for w2's\nexisting test file. w4 depends on w2+w3 landing since it removes the\nomit entry only once real fast-lane coverage exists to replace it.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-04T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [], "deps": [], "description": "Test-strategy review of coverage configuration found real coverage\nblind spots, but the originally-cited facts needed correction after\nreading the current tree - two of the three \"omitted\" TPC-DI ETL files\nno longer exist:\n\n`.coveragerc_core:16-18` currently reads:\n\n benchbox/core/tpcdi/etl/transformations.py\n benchbox/core/tpcdi/etl/batch.py\n benchbox/core/tpcdi/etl/scd_processor.py\n\n- `transformations.py` was deleted in commit `0fbbd431` (\"chore: shrink\n dead TPC-DI transformations (#640)\").\n- `batch.py` was deleted in commit `5b15277b` (\"chore: shrink dead TPC-DI\n ETL batch scaffold (#637)\").\n- Both omit entries are now stale/dead config referring to files that\n don't exist - harmless (coverage's `omit` glob just matches nothing)\n but should be cleaned up so the file doesn't misrepresent what's\n actually being excluded from coverage.\n- `scd_processor.py` DOES still exist (951 lines,\n `benchbox/core/tpcdi/etl/scd_processor.py`) and IS still omitted from\n coverage entirely. It has a dedicated unit test file\n (`tests/unit/core/tpcdi/test_etl_scd_processor.py`, marked\n `pytest.mark.unit, pytest.mark.medium` - not `fast`) plus exercise via\n `tests/integration/test_tpcdi_phase3_benchmark.py` (`integration,\n medium`) and `tests/integration/test_tpcdi_full_benchmark.py`\n (`integration, slow`). None of these run in the fast lane\n (`-m \"fast and not (slow or ...)\"`, `.github/workflows/pr.yml:450`,\n `--cov-fail-under=70`), so even without the coverage omit,\n `scd_processor.py` contributes zero coverage to the number CI actually\n gates on - the omit just also hides it from the full/nightly coverage\n report.\n\nSeparately, `benchbox/core/throughput/` (`runner.py` 163 lines,\n`result.py` 72 lines - imported by\n`benchbox/core/tpch/throughput_test.py` and\n`benchbox/core/tpcds/throughput_test.py`) is NOT in the `.coveragerc_core`\nomit list, so it's nominally measured - but its only real exerciser,\n`tests/integration/test_tpch_throughput_test.py`, is marked\n`pytest.mark.integration, pytest.mark.slow`. There is no\n`fast`-marked test that imports or exercises `core/throughput` at all,\nso its real fast-lane coverage is effectively zero even though the\n90-line-ish module isn't formally excluded.\n\nNet effect: the 70% `--cov-fail-under` floor\n(`.github/workflows/pr.yml:450`) is measured only on the fast lane, and\ntwo real modules (`scd_processor.py`, `core/throughput/`) have no\nfast-lane characterization tests protecting them from regressions\nbetween releases, even though one of them isn't even coverage-omitted.", "id": "coverage-blindspot-burndown", "preserves": ["Existing medium/slow/integration TPC-DI and throughput tests (test_etl_scd_processor.py if left as medium, test_tpcdi_phase3_benchmark.py, test_tpcdi_full_benchmark.py, test_tpch_throughput_test.py) continue to pass unchanged.", "No behavior change to benchbox/core/tpcdi/etl/scd_processor.py or benchbox/core/throughput/ - this is a test-and-config-only TODO.", "The 70% --cov-fail-under floor in .github/workflows/pr.yml:450 continues to pass after adding new fast tests (new tests should raise coverage, not just add runtime)."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/pr.yml"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/throughput/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcdi/etl/scd_processor.py"}, {"kind": "only_modify", "path_glob": ".coveragerc_core"}, {"kind": "only_modify", "path_glob": "tests/unit/core/throughput/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_etl_scd_processor.py"}], "state": "done", "title": "Coverage config hygiene: drop stale .coveragerc_core omits, add fast-lane tests for TPC-DI SCD processor and core/throughput", "verifications": [{"command": "! grep -n 'transformations.py\\|batch.py' .coveragerc_core", "description": ".coveragerc_core no longer references deleted files", "expected": "Exit 0 (grep found no match; bare grep exits 1 on no-match, so it must be inverted for this to be a pass/fail gate)", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_etl_scd_processor.py -m fast -v", "description": "New/re-marked fast tests for scd_processor.py pass and are collected in the fast lane", "expected": "Tests collected and pass under the fast marker", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/throughput/ -m fast -v", "description": "New fast tests for core/throughput pass and are collected in the fast lane", "expected": "Tests collected and pass under the fast marker", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests -m \"fast and not (slow or stress or resource_heavy or live_integration)\" --cov=benchbox --cov-report=term-missing --cov-fail-under=70", "description": "Fast-lane coverage floor still passes with the coveragerc omissions removed", "expected": "Coverage >= 70%, all fast tests pass", "item_id": "coverage-blindspot-burndown", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": [], "notes": "Delete the `benchbox/core/tpcdi/etl/transformations.py` and\n`benchbox/core/tpcdi/etl/batch.py` lines from `.coveragerc_core`'s\n`omit` list (lines 16-17) - both files were removed by #640 and #637\nrespectively and no longer exist. Confirm no other file resurrects\nthose paths before deleting (`git log --all -- ` should show\nonly the deletion commits as the most recent history).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the two stale .coveragerc_core omit entries for deleted files", "wid": "w1"}, {"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": [], "notes": "`tests/unit/core/tpcdi/test_etl_scd_processor.py` already exists and\nis well-scoped (104 lines) but marked `medium`, not `fast`. Two\noptions, pick based on actual runtime once measured:\n 1. If the existing medium tests are individually fast (likely,\n given SCD logic is pure in-memory transformation, not I/O), just\n re-mark the file's `pytestmark` to `fast` after confirming\n runtime with `--durations=0`.\n 2. If the existing tests are genuinely slower (e.g. they spin up a\n DataFrame/DB fixture), add a small new `fast`-marked test module\n covering the core SCD Type-2 branching logic\n (insert-new-version / expire-old-version / no-op-unchanged)\n against minimal in-memory fixtures, leaving the existing\n `medium` file for broader integration-shaped cases.\nWhichever path, the fast lane must end up covering scd_processor.py.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add fast-marked characterization tests for benchbox/core/tpcdi/etl/scd_processor.py", "wid": "w2"}, {"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": [], "notes": "`benchbox/core/throughput/runner.py` (163 lines) and `result.py`\n(72 lines) currently have no test coverage outside the slow-marked\n`tests/integration/test_tpch_throughput_test.py`. Add a small\n`fast`-marked unit test module (e.g.\n`tests/unit/core/throughput/test_runner.py`) covering the runner's\npure logic paths with a mocked/fake benchmark object (mirroring the\npattern already used in test_tpch_throughput_test.py's\n`benchmark.get_query.return_value = \"SELECT 1\"` style mocking, but\nwithout the `slow` marker) - concurrency/throughput timing behavior,\nresult aggregation in `result.py`, and error/edge-case handling\n(zero queries, single-thread fallback, etc.). Do not attempt to\ncharacterize actual multi-threaded timing behavior in the fast lane -\nthat's what the existing slow integration test is for; focus fast\ntests on deterministic logic (aggregation math, config validation,\nresult-shape correctness).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add fast-marked characterization tests for benchbox/core/throughput/", "wid": "w3"}, {"evidence": null, "item_id": "coverage-blindspot-burndown", "needs": ["w2", "w3"], "notes": "Once w2/w3 land fast-lane coverage for scd_processor.py (and\ncore/throughput if it turns out to need adding to the omit list\npending w3 - it currently is NOT omitted, so no removal needed\nthere), remove the now-covered `scd_processor.py` line from\n`.coveragerc_core`'s omit list. For every omit entry that remains\n(the dataframe_queries/queries.py-style generated-query files,\n`benchbox/experimental/concurrency/*`), add a one-line rationale\ncomment above it in `.coveragerc_core` explaining WHY it's\ndeliberately excluded (e.g. \"# generated/templated SQL construction,\nexercised via golden-output tests, not line coverage\" or\n\"# experimental, not on any release-gate path\") so future readers\ndon't have to archaeology their way to an answer the way this TODO\nhad to.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove the (now-clean) coveragerc omissions and document any remaining deliberate ones", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add @pytest.mark.slow, @pytest.mark.live_integration, or @pytest.mark.resource_heavy markers", "instead": "use only @pytest.mark.fast", "why": "those markers cause tests to be excluded from the fast lane and won't count toward the 80% coverage gate"}, {"dont": "batch all 25 modules into a single test file", "instead": "keep per-module or per-group test files aligned with existing test directory structure", "why": "monolithic test files make coverage diffs unreadable and slow down targeted reruns"}, {"dont": "call list() on color_cycle() return values", "instead": "(not specified)", "why": "color_cycle() returns an infinite itertools.cycle; use list(theme.palette or DEFAULT_PALETTE) instead"}, {"dont": "import or instantiate real cloud SDK clients in tests", "instead": "mock all external SDK calls with unittest.mock.patch", "why": "CI has no cloud credentials and live calls cause flakiness and violate the fast-lane contract"}, {"dont": "modify production source files to make them more testable", "instead": "only add/extend test files", "why": "changes to production code require their own review cycle and could introduce regressions that dwarf the coverage gain"}, {"dont": "use -o 'addopts=' to override pytest.ini", "instead": "(not specified)", "why": "this strips -n auto and makes tests 10x slower"}], "approach": "Original work units (w1-w14) - completed except w3:\n w1-w4 and w6-w9 are fully independent. w5 before w10 (mock patterns).\n w8 \u2192 w13, w12 \u2192 w14 (file extension chains).\n\nNew work units (w15-w24) - dependency chains:\n w15 depends on w4, w2, w5 (extends their test files)\n w16 depends on w2, w9 (extends comparison and cloud adapter tests)\n w17 depends on w6, w10 (extends staging and fabric tests)\n w18 depends on w1, w6 (extends core primitive tests)\n w19 depends on w7, w5 (extends spark and credential tests)\n w20 depends on w10 (extends snowflake test file from w10)\n w21 depends on w11 (extends bigquery test file from w11)\n w22 depends on w13 (extends databricks test file from w8/w13)\n w23 depends on w14 (extends redshift test file from w12/w14)\n w24 depends on w23 (extends redshift further)\n\nParallelism: w15-w19 are independent of each other (different files).\nw20-w22 are independent of each other.\nw23 must complete before w24.\n\nCritical: verify per-module coverage after each work unit, not just overall.\nUse: make coverage-fast 2>&1 | grep \"module_path\" to check individual modules.\n\nFor each work unit:\n1. Run coverage on the target module first to identify the exact uncovered lines:\n uv run -- python -m pytest --cov=benchbox/ --cov-report=term-missing -m fast -q\n2. Add tests targeting the uncovered lines\n3. Verify module coverage \u226580% before committing\n4. Run the full fast smoke suite to confirm no regressions\n\nRecommended mock setup for cloud SDKs:\n- boto3: patch at \"benchbox.platforms.redshift.boto3\" etc. (not at boto3 root)\n- google.cloud: use MagicMock() for both bigquery.Client and storage.Client\n- snowflake.connector: patch \"benchbox.platforms.snowflake.snowflake.connector.connect\"\n- databricks.sql: patch \"benchbox.platforms.databricks.adapter.databricks\"\n- pyspark: patch \"benchbox.platforms.dataframe.pyspark_df.SparkSession\"\n- pyodbc: patch \"benchbox.platforms.fabric_warehouse.pyodbc.connect\"\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-02T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Current fast-test coverage sits at 80.04% (85,262 / 106,518 lines) - just 47\ncovered lines above the 80% threshold. The buffer is dangerously thin: a single\nbad merge could drop us below the gate.\n\nThis item systematically brings the 25 lowest-coverage modules up to \u226580% each,\nadding robustness to the overall coverage floor and reducing the risk of accidental\nregression below the gate.\n\nCoverage gaps (lines needed to reach 80% per module):\n databricks/dataframe_adapter.py 23.57% \u2192 +89 lines\n dataframe/pyspark_df.py 28.06% \u2192 +145 lines\n azure/spark_execution_utils.py 34.78% \u2192 +21 lines\n databricks/adapter.py 41.68% \u2192 +396 lines\n bigquery.py 42.15% \u2192 +321 lines\n redshift.py 42.98% \u2192 +435 lines\n spark.py 46.04% \u2192 +133 lines\n snowflake.py 46.43% \u2192 +245 lines\n dataframe/lakesail_df.py 48.94% \u2192 +88 lines\n credentials/redshift.py 50.96% \u2192 +121 lines\n tpcds/generator/streaming.py 52.20% \u2192 +57 lines\n motherduck.py 52.60% \u2192 +42 lines\n base/cloud_spark/staging.py 53.24% \u2192 +99 lines\n tpchavoc/benchmark.py 53.51% \u2192 +30 lines\n credentials/snowflake.py 54.26% \u2192 +48 lines\n fabric_warehouse.py 55.68% \u2192 +109 lines\n clickhouse/diagnostics.py 56.18% \u2192 +21 lines\n transaction_primitives/dataframe_operations.py 56.18% \u2192 +89 lines\n tpcdi/financial_data.py 56.33% \u2192 +38 lines\n starburst.py 56.73% \u2192 +24 lines\n comparison/suite.py 56.97% \u2192 +56 lines\n results/anonymization.py 57.20% \u2192 +59 lines\n write_primitives/benchmark.py 57.35% \u2192 +109 lines\n gcp/dataproc_adapter.py 58.14% \u2192 +56 lines\n aws/athena_spark_adapter.py 58.66% \u2192 +60 lines\n\nTotal: ~2,891 new covered lines across 12 work units (original estimate).\n\n--- Progress Review (2026-04-02) ---\n\nAfter 13 of 14 original work units (w1-w2, w4-w14), overall coverage rose from\n80.04% to 81%. However, only 3 of 25 target modules actually reached 80%:\n\n Module Before \u2192 After Status\n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n results/anonymization.py 57.20% \u2192 89% \u2705 HIT\n tpchavoc/benchmark.py 53.51% \u2192 85% \u2705 HIT\n starburst.py 56.73% \u2192 82% \u2705 HIT\n motherduck.py 52.60% \u2192 77% Near (needs +4)\n tpcds/generator/streaming.py 52.20% \u2192 73% Near (needs +15)\n comparison/suite.py 56.97% \u2192 65% Gap (needs +37)\n gcp/dataproc_adapter.py 58.14% \u2192 66% Gap (needs +37)\n aws/athena_spark_adapter.py 58.66% \u2192 63% Gap (needs +47)\n credentials/snowflake.py 54.26% \u2192 55% Gap (needs +46)\n transaction_primitives/dataframe_ops 56.18% \u2192 56% Gap (needs +88)\n clickhouse/diagnostics.py 56.18% \u2192 57% Gap (needs +20)\n write_primitives/benchmark.py 57.35% \u2192 58% Gap (needs +104)\n fabric_warehouse.py 55.68% \u2192 59% Gap (needs +96)\n base/cloud_spark/staging.py 53.24% \u2192 54% Gap (needs +97)\n spark.py 46.04% \u2192 50% Gap (needs +116)\n credentials/redshift.py 50.96% \u2192 52% Gap (needs +117)\n snowflake.py 46.43% \u2192 54% Gap (needs +186)\n databricks/adapter.py 41.68% \u2192 53% Gap (needs +274)\n bigquery.py 42.15% \u2192 48% Gap (needs +273)\n redshift.py 42.98% \u2192 48% Gap (needs +372)\n dataframe/pyspark_df.py 28.06% \u2192 28% w3 pending\n dataframe/lakesail_df.py 48.94% \u2192 49% w3 pending\n databricks/dataframe_adapter.py 23.57% \u2192 24% w3 pending\n\nRoot cause: the original work units under-estimated the number of new covered\nlines required. Large adapter modules (bigquery, redshift, databricks, snowflake)\nhave deep call trees with many branches that need dedicated test scenarios for\neach path through connection setup, data loading, query execution, SQL generation,\nand tuning application.\n\nAdditional work units w15-w24 added below to close the remaining ~1,744 lines\nof coverage gap across the 19 modules that haven't yet reached 80%.\nCombined with w3 (~322 lines), total remaining work is ~2,066 new covered lines.\n\nAll tests MUST carry the @pytest.mark.fast marker so they run in the fast lane.\nAll external SDKs (databricks, pyspark, snowflake-connector, boto3, google-cloud,\npyodbc, duckdb, requests) are mocked - no live connections.", "id": "coverage-bottom-25-modules-to-80pct", "preserves": ["@pytest.mark.fast marker present on every new test function", "All existing passing tests continue to pass - zero regressions", "No live network calls - all external SDK clients mocked with unittest.mock.patch", "No production source files modified - tests only", "Overall fast-lane coverage stays \u226580.04% after every individual work unit commit"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "tests/unit/core/results/test_anonymization.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_comparison.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_financial_data_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcds/generator/test_streaming_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/transaction_primitives/test_dataframe_ops_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/write_primitives/test_benchmark_execution.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/aws/test_athena_spark_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/azure/test_spark_execution_utils.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/cloud_spark/test_staging.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/credentials/test_redshift_defaults.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/credentials/test_snowflake_defaults.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/databricks/test_databricks_df_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_lakesail_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_pyspark_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/gcp/test_dataproc_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_bigquery_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_clickhouse_cloud_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databricks_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_microsoft_fabric_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_motherduck.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_redshift_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_snowflake_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_spark_adapter_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_starburst.py"}], "state": "done", "title": "Bring the 25 lowest-coverage modules to 80% fast-test coverage", "verifications": [{"command": "uv run -- python -m pytest -m fast -q --cov=benchbox/platforms/databricks --cov=benchbox/platforms/dataframe --cov=benchbox/platforms/azure --cov=benchbox/platforms/bigquery --cov=benchbox/platforms/redshift --cov=benchbox/platforms/spark --cov=benchbox/platforms/snowflake --cov=benchbox/platforms/credentials --cov=benchbox/platforms/motherduck --cov=benchbox/platforms/base/cloud_spark --cov=benchbox/platforms/fabric_warehouse --cov=benchbox/platforms/clickhouse --cov=benchbox/platforms/starburst --cov=benchbox/platforms/gcp --cov=benchbox/platforms/aws --cov=benchbox/core/tpcds/generator --cov=benchbox/core/tpchavoc --cov=benchbox/core/comparison --cov=benchbox/core/results/anonymization --cov=benchbox/core/write_primitives --cov=benchbox/core/transaction_primitives --cov=benchbox/core/tpcdi --cov-report=term-missing 2>&1 | grep -E '(FAILED|ERROR|PASSED|[0-9]+%)'", "description": "All 25 target modules individually reach \u226580% coverage in the fast lane", "expected": "All target modules show \u226580% coverage; no FAILED or ERROR lines", "item_id": "coverage-bottom-25-modules-to-80pct", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q --cov=benchbox --cov-report=term 2>&1 | tail -5", "description": "Overall fast-lane coverage stays at or above 80.04%", "expected": "Total coverage \u2265 80%", "item_id": "coverage-bottom-25-modules-to-80pct", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q 2>&1 | tail -5", "description": "No regressions - full fast test suite passes", "expected": "passed, 0 failed", "item_id": "coverage-bottom-25-modules-to-80pct", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pure-logic core utilities: financial_data, dataframe_operations, anonymization, spark_execution_utils", "wid": "w1"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "fabric_warehouse.py and snowflake.py: auth logic and connection config", "wid": "w10"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "bigquery.py: GCS integration, auth, and query execution branches", "wid": "w11"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 1: connector selection, connection setup, S3 URI parsing", "wid": "w12"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "databricks/adapter.py part 2: data loading, DBFS upload, error handling", "wid": "w13"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w12"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 2: data loading flow, SQL generation, WLM query group", "wid": "w14"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w2", "w4", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Small top-ups: motherduck, streaming, clickhouse/diagnostics, credentials/snowflake", "wid": "w15"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w2", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Mid-tier adapters part 2: comparison/suite, dataproc, athena_spark", "wid": "w16"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w10", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud infrastructure: staging.py providers + fabric_warehouse data loading", "wid": "w17"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w1", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Core primitives part 2: transaction dataframe_ops + write_primitives benchmark", "wid": "w18"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w5", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "spark.py part 2 + credentials/redshift part 2", "wid": "w19"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "TPC generators and comparison suite: streaming, tpchavoc, comparison/suite", "wid": "w2"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w10"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "snowflake.py part 2: connection, external tables, query execution, tuning", "wid": "w20"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "bigquery.py part 2: metadata, validation, loading, execution, tuning", "wid": "w21"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w13"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "databricks/adapter.py part 3: from_config, UC volumes, data loading, tuning", "wid": "w22"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w14"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 3: from_config, platform info, drop_database, S3 loading", "wid": "w23"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": ["w23"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "redshift.py part 4: external tables, query execution, statistics, tuning", "wid": "w24"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DataFrame expression builders: pyspark_df, lakesail_df, databricks DF adapter", "wid": "w3"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Small cloud adapters: motherduck, starburst, clickhouse diagnostics", "wid": "w4"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Credential modules: credentials/snowflake and credentials/redshift", "wid": "w5"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud staging URI parsing and write-primitives validation logic", "wid": "w6"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "spark.py: deployment modes, config validation, resource allocation", "wid": "w7"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "databricks/adapter.py part 1: warehouse selection, clustering, SQL generation", "wid": "w8"}, {"evidence": null, "item_id": "coverage-bottom-25-modules-to-80pct", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "AWS Athena Spark + GCP Dataproc: state polling and job lifecycle", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add external dependencies to the coverage script", "instead": "use only stdlib (xml.etree, pathlib, argparse)", "why": "it runs in CI where install time matters"}, {"dont": "make the script fail on missing coverage.xml", "instead": "exit 0 with a warning if the file is absent", "why": "not all CI runs generate it"}], "approach": "Write a standalone Python script (no external dependencies beyond stdlib)\nthat uses xml.etree.ElementTree to parse coverage.xml. Group files by\ntheir first two path components (e.g., benchbox/cli, benchbox/core/tpch).\nPrint a table of packages with coverage, highlight failures, exit 1 if\nany package is below threshold. Make threshold configurable via CLI arg.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": ["coverage-cli-modules", "coverage-core-benchmarks", "coverage-core-tpcdi", "coverage-core-utilities", "coverage-platforms"], "description": "Once all modules reach \u226550% coverage (via the companion TODO items),\nadd CI enforcement to prevent regressions below the floor.\n\nCurrent CI uses --cov-fail-under=50 which checks OVERALL coverage only.\nThis allows individual modules to drop to 0% as long as the aggregate\nstays above 50%. A per-module check would catch this.\n\nOptions:\n 1. Use coverage.py's [report] fail_under per-package (limited granularity)\n 2. Add a custom script that parses coverage.xml and fails on any module <50%\n 3. Use diff-cover to enforce coverage on changed files only\n\nRecommend option 2: a lightweight Python script in scripts/ that parses\ncoverage.xml, groups by top-level package, and fails if any package is\nbelow threshold.", "id": "coverage-ci-enforcement", "preserves": ["Existing passing CI jobs remain passing with no increase in runtime >20%", "ci-test Makefile target continues to run pytest with existing flags and pass"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}], "state": "done", "title": "Enforce per-module 50% coverage floor in CI", "verifications": [{"command": "uv run -- python scripts/check_module_coverage.py --coverage-xml coverage.xml --threshold 50 --exclude-module benchbox/data --exclude-module benchbox/examples", "description": "Script correctly identifies modules below threshold", "expected": "table of per-module coverage with pass/fail indicators", "item_id": "coverage-ci-enforcement", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/check_module_coverage.py --coverage-xml coverage.xml --threshold 99 --exclude-module benchbox/data --exclude-module benchbox/examples; echo $?", "description": "Script exits non-zero when modules fail", "expected": "exit code 1 (since no module is at 99%)", "item_id": "coverage-ci-enforcement", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write scripts/check_module_coverage.py that parses coverage.xml", "wid": "w1"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Parse coverage.xml, group by benchbox/* top-level package", "wid": "w2"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Report per-package coverage with pass/fail against 50% threshold", "wid": "w3"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Exit non-zero if any package fails", "wid": "w4"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for the coverage check script itself", "wid": "w5"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add check_module_coverage.py to Makefile as coverage-check target", "wid": "w6"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add to ci-test target in Makefile after pytest coverage run", "wid": "w7"}, {"evidence": null, "item_id": "coverage-ci-enforcement", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update GitHub Actions workflow to run the check", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "launch actual benchmark runs in CLI tests", "instead": "mock the runner with unittest.mock.patch", "why": "benchmark runs take 30+ seconds and require platform credentials"}, {"dont": "test by parsing CLI stdout strings", "instead": "test return values, exit codes, and side effects instead", "why": "stdout format changes would cause false failures"}], "approach": "Follow existing patterns in tests/unit/cli/. Most CLI modules can be tested\nby invoking Click commands with CliRunner or by testing individual functions\ndirectly. For compare commands, create small mock result files. For tuning.py,\ntest YAML config parsing and parameter resolution. For onboarding.py, mock\nuser input and filesystem operations.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Several CLI modules are below 50% test coverage. These handle user-facing\ncommands, configuration, output formatting, and error handling.\n\nCurrent coverage breakdown:\n - cli/commands/compare.py: 30.2% (623 missing)\n - cli/commands/run.py: 49.8% (516 missing) - nearly there\n - cli/commands/shell.py: 30.0% (301 missing)\n - cli/commands/compare_plans.py: 11.6% (229 missing)\n - cli/commands/compare_dataframes.py: 12.7% (200 missing)\n - cli/tuning.py: 0.0% (397 missing)\n - cli/config.py: 47.6% (267 missing) - nearly there\n - cli/onboarding.py: 0.0% (192 missing)\n - cli/output.py: 49.0% (177 missing) - nearly there\n - cli/exceptions.py: 36.9% (178 missing)\n - cli/preferences.py: 19.2% (126 missing)\n - cli/composite_params.py: 46.4% (120 missing)\n - cli/presentation/system.py: 16.7% (20 missing)\n - cli/system.py: 43.3% (17 missing)\n\nThree modules (run.py, config.py, output.py) are very close to 50% and\nneed minimal effort. The zero-coverage modules (tuning.py, onboarding.py)\nneed foundational test suites.", "id": "coverage-cli-modules", "preserves": ["All existing CLI tests continue to pass", "CLI --help output remains unchanged", "No changes to production code"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}], "state": "done", "title": "Improve test coverage for CLI modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/ -v --tb=short", "description": "All CLI tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-cli-modules", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/run.py (49.8% \u2192 50%)", "wid": "w1"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/compare_plans.py (11.6% \u2192 50%)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/compare_dataframes.py (12.7% \u2192 50%)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/exceptions.py (36.9% \u2192 50%)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/preferences.py (19.2% \u2192 50%)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/presentation/system.py (16.7% \u2192 50%)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/config.py (47.6% \u2192 50%)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/output.py (49.0% \u2192 50%)", "wid": "w3"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/composite_params.py (46.4% \u2192 50%)", "wid": "w4"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/system.py (43.3% \u2192 50%)", "wid": "w5"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/tuning.py (0% \u2192 50%)", "wid": "w6"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/onboarding.py (0% \u2192 50%)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/compare.py (30.2% \u2192 50%)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-cli-modules", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for cli/commands/shell.py (30.0% \u2192 50%)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "create integration tests requiring live databases", "instead": "use mocks and in-memory fixtures", "why": "CI has no database servers and tests must run in <1s"}, {"dont": "modify production code to improve coverage", "instead": "write tests for code as-is", "why": "the goal is to verify existing behavior, not refactor"}, {"dont": "test auto-generated SQL strings character-by-character", "instead": "test query structure, table references, and parameters instead", "why": "SQL formatting changes would cause false failures"}], "approach": "Follow existing test patterns in tests/unit/benchmarks/ (e.g., test_tpch_core.py,\ntest_tpcds_core.py). Use @pytest.mark.fast for tests <1s, @pytest.mark.unit for\nall. Mock external dependencies (file I/O, database connections, TPC binaries).\nFor dataframe_queries files, test a representative subset of queries (5-10 per\nbenchmark) rather than exhaustive coverage.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-11T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Core benchmark module coverage has been refreshed from coverage.json generated\non 2026-02-10. Most originally-targeted files are now >=50% from prior waves.\n\nRemaining files below 50% that still require targeted unit tests:\n - benchbox/core/ssb/benchmark.py: 48%\n - benchbox/core/nyctaxi/benchmark.py: 49%\n - benchbox/core/metadata_primitives/benchmark.py: 47%\n\nThis item now focuses on closing only these remaining gaps to >=50% while\npreserving existing benchmark behavior and avoiding production code changes.", "id": "coverage-core-benchmarks", "preserves": ["All existing tests continue to pass", "No changes to production code - tests only"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/integration/"}, {"kind": "only_modify", "path_glob": "tests/unit/benchmarks/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Improve test coverage for core benchmark modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/ -v --tb=short -m fast", "description": "All new tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-core-benchmarks", "last_result": null, "last_run": null, "seq": 1}, {"command": "make coverage-per-file && for f in benchbox/core/ssb/benchmark.py benchbox/core/nyctaxi/benchmark.py benchbox/core/metadata_primitives/benchmark.py; do jq -r --arg f \"$f\" '.files[$f].summary.percent_covered_display' coverage.json; done", "description": "Coverage for each targeted module reaches \u226550%", "expected": "each printed value is >=50", "item_id": "coverage-core-benchmarks", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpch/reporting.py (0% \u2192 50%)", "wid": "w1"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/clickbench/benchmark.py (25.8% \u2192 50%)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/amplab/benchmark.py (18.4% \u2192 50%)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/amplab/generator.py (29.1% \u2192 50%)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/nyctaxi/benchmark.py (49% \u2192 50%)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/nyctaxi/downloader.py (20.5% \u2192 50%)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/h2odb/benchmark.py (28.8% \u2192 50%)", "wid": "w15"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/read_primitives/dataframe_queries.py (16.5% \u2192 50%)", "wid": "w16"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/transaction_primitives/generator.py (12.9% \u2192 50%)", "wid": "w17"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/transaction_primitives/benchmark.py (24.5% \u2192 50%)", "wid": "w18"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/metadata_primitives/generator.py (8.0% \u2192 50%)", "wid": "w19"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpch/maintenance_test.py (12.8% \u2192 50%)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/metadata_primitives/benchmark.py (47% \u2192 50%)", "wid": "w20"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/datavault/dataframe_queries/queries.py (6.7% \u2192 50%)", "wid": "w21"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/joinorder/generator.py (20.1% \u2192 50%)", "wid": "w22"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpch/parameter_extractor.py (23.9% \u2192 50%)", "wid": "w3"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/reporting.py (0% \u2192 50%)", "wid": "w4"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/maintenance_test.py (20.6% \u2192 50%)", "wid": "w5"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/throughput_test.py (23.8% \u2192 50%)", "wid": "w6"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/tpcds/benchmark/runner.py (32.7% \u2192 50%)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/ssb/benchmark.py (48% \u2192 50%)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-core-benchmarks", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for core/ssb/generator.py (45.0% \u2192 50%)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "require dsqgen binary", "instead": "mock data generation calls", "why": "it is platform-specific, not available in CI, and segfaults on fractional scale factors"}, {"dont": "test with real TPC-DI data files", "instead": "create minimal synthetic fixtures (5-10 rows)", "why": "they require the dsqgen binary and are 100MB+ at SF=1, making tests slow and environment-dependent"}], "approach": "The ETL transformations are mostly pure functions that transform DataFrames\nor dictionaries - ideal for unit testing. Create test fixtures with small\nsample data (5-10 rows) that exercise the transform logic. For SCD processor,\ntest Type 1 and Type 2 SCD operations with before/after state assertions.\nFor batch.py, mock the database connection and test orchestration logic.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "This item was completed by validating that existing TPC-DI test coverage\nalready satisfies the >=50% targets for the scoped files and package-level\nthreshold, without requiring production or test-code changes in this item.\n\nVerified coverage results at completion:\n - core/tpcdi/etl/transformations.py: 61%\n - core/tpcdi/etl/scd_processor.py: 76%\n - core/tpcdi/source_generators.py: 99%\n - core/tpcdi/etl/batch.py: 68%\n - core/tpcdi/benchmark.py: 54%\n\nAll TPC-DI unit tests in scope passed, and package-level coverage exceeded\nthe 50% minimum.", "id": "coverage-core-tpcdi", "preserves": ["All existing TPC-DI tests continue to pass", "No changes to production code"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "do_not_modify", "path_glob": "tests/integration/"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/coverage-core-tpcdi.yaml"}], "state": "done", "title": "Improve test coverage for core/tpcdi module to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcdi -v --tb=short", "description": "TPC-DI unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-core-tpcdi", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi --cov=benchbox/core/tpcdi --cov-report=term --cov-fail-under=50 -q", "description": "core/tpcdi package coverage reaches >=50%", "expected": "coverage check passes with total >=50%", "item_id": "coverage-core-tpcdi", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi -p pytest_cov --cov=benchbox/core/tpcdi --cov-report=json:coverage_tpcdi.json --cov-report=term --cov-fail-under=0 -q && for f in benchbox/core/tpcdi/etl/transformations.py benchbox/core/tpcdi/etl/scd_processor.py benchbox/core/tpcdi/source_generators.py benchbox/core/tpcdi/etl/batch.py benchbox/core/tpcdi/benchmark.py; do v=$(jq -r --arg f \"$f\" '.files[$f].summary.percent_covered_display' coverage_tpcdi.json); echo \"$f: $v\"; awk -v n=\"$v\" 'BEGIN { exit !(n+0 >= 50) }' || exit 1; done", "description": "Each targeted TPC-DI file reaches >=50% coverage", "expected": "each listed file prints a value >=50 and command exits 0", "item_id": "coverage-core-tpcdi", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "coverage-core-tpcdi", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Establish scoped verification baseline and fixture expectations for TPC-DI coverage review", "wid": "w1"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/etl/transformations.py meets >=50% coverage target", "wid": "w2"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/etl/scd_processor.py meets >=50% coverage target", "wid": "w3"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/source_generators.py meets >=50% coverage target", "wid": "w4"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/etl/batch.py meets >=50% coverage target", "wid": "w5"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify core/tpcdi/benchmark.py meets >=50% coverage target", "wid": "w6"}, {"evidence": null, "item_id": "coverage-core-tpcdi", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run final package + per-file coverage verification and capture completion evidence", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add tests that depend on filesystem state or network", "instead": "mock I/O using unittest.mock or pytest tmp_path fixture", "why": "CI environments vary and network tests are flaky"}, {"dont": "test private implementation details", "instead": "test via public methods and documented interfaces only", "why": "private APIs can change without notice, breaking tests"}], "approach": "Start with zero-coverage modules since any test at all is progress.\ncore/tpc_metrics.py likely has pure functions for metric calculation - test\nwith known inputs/outputs. core/comparison/ is a self-contained comparison\nengine - test the comparison logic with mock result objects. For release/sync.py,\nmock filesystem and git operations. utils/lazy_loader.py is a module-level\nlazy import utility - test import mechanics.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Several standalone core modules and utility files have zero or low test\ncoverage. These are smaller, self-contained modules that should be\nstraightforward to test.\n\nZero-coverage modules (quick wins):\n - core/tpc_metrics.py: 0% (402 missing)\n - core/comparison/ (suite, types, plotter): 0% (346 missing)\n - core/platform_config_utils.py: 0% (102 missing)\n - core/reports.py: 0% (26 missing)\n - release/sync.py: 0% (259 missing)\n - utils/lazy_loader.py: 0% (70 missing)\n\nLow-coverage modules:\n - core/dryrun.py: 49.3% (265 missing) - nearly there\n - utils/cloud_storage.py: 37.5% (185 missing)\n - utils/config_helpers.py: 37.8% (69 missing)\n - examples/dryrun_utils.py: 15.9% (53 missing)\n - mcp/resources/registry.py: 27.0% (65 missing)\n - mcp/prompts/registry.py: 40.4% (34 missing)\n - mcp/__init__.py: 39.1% (14 missing)\n\nPriority: zero-coverage modules first (biggest bang-for-buck), then\nmodules near 50%.", "id": "coverage-core-utilities", "preserves": ["All existing tests continue to pass", "No changes to production code"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Improve test coverage for core utility and standalone modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/ -k 'tpc_metrics or comparison or platform_config or reports or sync or lazy_loader' -v", "description": "New tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-core-utilities", "last_result": null, "last_run": null, "seq": 1}], "work": [], "worktree": "main"} +{"anti_patterns": [{"dont": "add blanket import/smoke tests", "instead": "assert concrete outputs, SQL fragments, state transitions, and error text instead.", "why": "they create coverage theater"}, {"dont": "change coverage policy to make the denominator smaller", "instead": "close the gap with executable tests.", "why": "that hides risk rather than reducing it"}, {"dont": "mix multiple heavyweight adapters into one implementation commit", "instead": "land one work unit at a time with module-local verification.", "why": "diagnosis and rollback get harder"}, {"dont": "touch `benchbox/core/publishing/**`, `benchbox/cli/commands/submit.py`, or `results-explorer/` in this item", "instead": "keep those changes in the publishing stream.", "why": "that surface is still moving"}, {"dont": "use live cloud accounts, real Spark clusters, or real sockets to raise unit coverage", "instead": "patch SDK clients, Spark builders, subprocesses, file I/O, and network probes.", "why": "the fast lane must remain hermetic"}], "approach": "Use the April 2, 2026 filtered ranking as the fixed baseline for this wave.\nSequence the work so shared fixture layers are built once and reused:\nDataFrame foundation (w2) before DataFrame transaction/query work (w3-w4),\ncloud-Spark mixins (w9) before Spark/LakeSail (w10-w11), and transaction\nsemantics (w3) before the write-primitives benchmark paths (w18).\n\nHeavyweight adapters are intentionally isolated one module per work unit\n(w7-w16, w18-w19) so each implementation step can land as a single-purpose\ncommit with module-local rollback.\n\nExtend existing coverage-focused test files wherever the repo already has them.\nOnly add new dedicated test files when current coverage is scattered across generic\ntest modules and a focused file will make the missing branches easier to own.\nPlanned-new files are named explicitly with `Add ...`; existing files to be\nextended are named explicitly with `Extend ...`.\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-03T00:00:00Z", "completed_pr": null, "created_at": "2026-04-02T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "id": 129, "reason": "Explicitly excluded from this item: `benchbox/core/publishing/**/*.py` and `benchbox/cli/commands/submit.py` are still in flight and should be planned against the publishing productization TODOs instead of this coverage wave.", "resolution": "promoted", "resolved_item": "results-publishing-coverage-work", "resolved_reason": null, "summary": "Results-publishing coverage work"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "id": 130, "reason": "Anything already excluded by `[tool.coverage.run].omit`, `[tool.coverage.report].exclude_lines`, or `pragma: no cover` is intentionally out of scope; this item targets only measured statements.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason states coverage-ignored/omit-listed lines are intentionally out of scope by design, not deferred work needing action.", "summary": "Coverage-ignored lines and omit-listed modules"}], "deps": [], "description": "Fresh baseline from `.coverage` rendered via `uv run -- python -m coverage json`\nat 2026-04-02 19:03:25 local time shows overall fast-lane coverage at 82.87%\n(88,387 / 106,663 measured statements).\n\nThis ranking already excludes coverage-ignore lines from the denominator:\n - `[tool.coverage.run].omit` entries are not in the measured file set\n - `[tool.coverage.report].exclude_lines` and `pragma: no cover` are not in\n `num_statements`\n\nThis plan also explicitly excludes the in-flight results-publishing surface:\n - `benchbox/core/publishing/**/*.py`\n - `benchbox/cli/commands/submit.py`\n\nFiltered bottom-25 modules and current gap to 80%:\n platforms/databricks/dataframe_adapter.py 23.57% -> +89 lines\n platforms/spark.py 46.04% -> +133 lines\n platforms/credentials/redshift.py 51.91% -> +118 lines\n core/transaction_primitives/dataframe_operations.py 56.45% -> +88 lines\n platforms/clickhouse/diagnostics.py 57.30% -> +21 lines\n core/write_primitives/benchmark.py 58.39% -> +105 lines\n core/tuning/generators/pg_mooncake.py 59.09% -> +5 lines\n metadata_primitives.py 59.38% -> +7 lines\n core/tpcds/generator/manager.py 59.63% -> +45 lines\n core/ssb/generator.py 59.78% -> +55 lines\n platforms/databricks/credentials.py 60.22% -> +37 lines\n core/tpcdi/etl/error_recovery.py 60.34% -> +58 lines\n core/datavault/benchmark.py 60.49% -> +32 lines\n experimental/gpu/benchmark.py 60.87% -> +40 lines\n platforms/base/cloud_spark/mixins.py 61.01% -> +61 lines\n platforms/aws/glue_adapter.py 61.15% -> +49 lines\n platforms/dataframe/iceberg_maintenance.py 61.46% -> +36 lines\n platforms/questdb.py 61.61% -> +94 lines\n experimental/aiml_functions/benchmark.py 61.66% -> +36 lines\n platforms/lakesail.py 61.68% -> +62 lines\n core/coffeeshop/dataframe_queries/queries.py 61.81% -> +47 lines\n platforms/firebolt.py 62.17% -> +121 lines\n platforms/dataframe/expression_family.py 62.18% -> +69 lines\n platforms/clickhouse_cloud.py 62.41% -> +75 lines\n core/amplab/dataframe_queries/queries.py 62.50% -> +31 lines\n\nTotal remaining gap across the filtered set: about 1,514 newly covered lines.\n\nSequencing rules for this item:\n1. Harvest low-cost pure/helper wins first.\n2. Build reusable DataFrame and cloud-Spark fixtures before the adapters that\n depend on them.\n3. Keep heavyweight adapters isolated one per work unit when the branch\n surface is wide enough to justify dedicated review and rollback.\n4. Do not spend time on results-publishing code until its productization TODO\n stabilizes the public surface.", "id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "preserves": ["All new tests stay in the fast lane: mark them `@pytest.mark.fast` when appropriate and keep every scenario hermetic.", "Assertions must verify actual SQL strings, config values, returned dataclasses, or error messages; import-only tests do not count.", "Existing adapter behavior and public CLI/config contracts remain unchanged.", "No new coverage ignore rules, omit entries, or `pragma: no cover` annotations are added to hit the target.", "Results-publishing code stays untouched in this item."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/publishing/**"}, {"kind": "do_not_modify", "path_glob": "pyproject.toml coverage omit/exclude settings"}, {"kind": "do_not_modify", "path_glob": "results-explorer/**"}, {"kind": "do_not_modify", "path_glob": "unrelated production modules outside the 25-file target set"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/coverage-current-bottom-25-modules-to-80pct-apr-2026.yaml"}, {"kind": "only_modify", "path_glob": "tests/integration/** only if a named unit explicitly calls for an existing helper adjustment"}, {"kind": "only_modify", "path_glob": "tests/unit/** for the named target modules"}, {"kind": "only_modify", "path_glob": "the named production modules only when a tiny test seam is unavoidable"}], "state": "done", "title": "Bring the current 25 lowest-coverage non-publishing modules to 80% fast-test coverage", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tuning/generators/test_pg_mooncake_ddl.py tests/unit/platforms/test_clickhouse_diagnostics.py -q -p pytest_cov --cov=benchbox/core/tuning/generators/pg_mooncake.py --cov=benchbox/platforms/clickhouse/diagnostics.py --cov-report=term-missing", "description": "w1 targeted coverage: pg_mooncake generator and ClickHouse diagnostics", "expected": "Both named modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/test_expression_family.py tests/unit/platforms/dataframe/test_expression_family_behavioral.py tests/unit/platforms/dataframe/test_expression_family_coverage.py tests/unit/platforms/dataframe/test_iceberg_maintenance.py tests/unit/platforms/dataframe/test_iceberg_maintenance_coverage.py -q -p pytest_cov --cov=benchbox/platforms/dataframe/expression_family.py --cov=benchbox/platforms/dataframe/iceberg_maintenance.py --cov-report=term-missing", "description": "w2 targeted coverage: DataFrame foundation modules", "expected": "Both foundation modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/transaction_primitives/test_dataframe_operations.py tests/unit/core/transaction_primitives/test_dataframe_ops_coverage.py -q -p pytest_cov --cov=benchbox/core/transaction_primitives/dataframe_operations.py --cov-report=term-missing", "description": "w3 targeted coverage: transaction DataFrame operations", "expected": "The transaction DataFrame operations module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core/coffeeshop/test_coffeeshop_dataframe_queries.py tests/unit/core/amplab/test_amplab_dataframe_queries.py -q -p pytest_cov --cov=benchbox/core/coffeeshop/dataframe_queries/queries.py --cov=benchbox/core/amplab/dataframe_queries/queries.py --cov-report=term-missing", "description": "w4 targeted coverage: CoffeeShop and AMPLab DataFrame query registries", "expected": "Both registry modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/tpcds/generator/test_manager.py tests/unit/generators/test_ssb_manifest.py tests/unit/core/ssb/test_ssb_generator.py tests/unit/core/datavault/test_datavault_benchmark.py -q -p pytest_cov --cov=benchbox/core/tpcds/generator/manager.py --cov=benchbox/core/ssb/generator.py --cov=benchbox/core/datavault/benchmark.py --cov-report=term-missing", "description": "w5 targeted coverage: TPC-DS manager, SSB generator, and DataVault benchmark", "expected": "All three generator or wrapper modules report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/gpu/test_gpu_benchmark.py tests/unit/core/aiml_functions/test_aiml_functions_benchmark.py -q -p pytest_cov --cov=benchbox/experimental/gpu/benchmark.py --cov=benchbox/experimental/aiml_functions/benchmark.py --cov-report=term-missing", "description": "w6 targeted coverage: experimental benchmark wrappers", "expected": "Both experimental benchmark wrappers report at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest tests/unit/platforms/databricks/test_databricks_df_adapter.py tests/unit/platforms/databricks/test_databricks_df_adapter_coverage.py -q -p pytest_cov --cov=benchbox/platforms/databricks/dataframe_adapter.py --cov-report=term-missing", "description": "w7 targeted coverage: Databricks DataFrame adapter", "expected": "The Databricks DataFrame adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 7}, {"command": "uv run -- python -m pytest tests/unit/platforms/credentials/test_databricks_defaults.py -q -p pytest_cov --cov=benchbox/platforms/databricks/credentials.py --cov-report=term-missing", "description": "w8 targeted coverage: Databricks credentials", "expected": "The Databricks credentials module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 8}, {"command": "uv run -- python -m pytest tests/unit/platforms/cloud_spark/test_spark_ddl_mixin.py tests/unit/platforms/base/cloud_spark/test_config.py -q -p pytest_cov --cov=benchbox/platforms/base/cloud_spark/mixins.py --cov-report=term-missing", "description": "w9 targeted coverage: cloud-Spark mixins", "expected": "The cloud-Spark mixins module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 9}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_spark_adapter.py tests/unit/platforms/test_spark_adapter_coverage.py -q -p pytest_cov --cov=benchbox/platforms/spark.py --cov-report=term-missing", "description": "w10 targeted coverage: core Spark adapter", "expected": "The core Spark adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 10}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_lakesail_adapter.py -q -p pytest_cov --cov=benchbox/platforms/lakesail.py --cov-report=term-missing", "description": "w11 targeted coverage: LakeSail adapter", "expected": "The LakeSail adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 11}, {"command": "uv run -- python -m pytest tests/unit/platforms/aws/test_glue_adapter.py -q -p pytest_cov --cov=benchbox/platforms/aws/glue_adapter.py --cov-report=term-missing", "description": "w12 targeted coverage: Glue adapter", "expected": "The Glue adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 12}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_firebolt_adapter.py tests/unit/platforms/test_firebolt_cloud.py -q -p pytest_cov --cov=benchbox/platforms/firebolt.py --cov-report=term-missing", "description": "w13 targeted coverage: Firebolt adapter", "expected": "The Firebolt adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 13}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_clickhouse_cloud_coverage.py -q -p pytest_cov --cov=benchbox/platforms/clickhouse_cloud.py --cov-report=term-missing", "description": "w14 targeted coverage: ClickHouse Cloud adapter", "expected": "The ClickHouse Cloud adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 14}, {"command": "uv run -- python -m pytest tests/unit/platforms/credentials/test_redshift_defaults.py tests/unit/cli/test_cli_setup.py -q -p pytest_cov --cov=benchbox/platforms/credentials/redshift.py --cov-report=term-missing", "description": "w15 targeted coverage: Redshift credential and diagnostics branches", "expected": "The Redshift credentials module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 15}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_questdb_adapter.py tests/unit/core/tuning/test_questdb_ddl_generator.py -q -p pytest_cov --cov=benchbox/platforms/questdb.py --cov-report=term-missing", "description": "w16 targeted coverage: QuestDB adapter", "expected": "The QuestDB adapter reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 16}, {"command": "uv run -- python -m pytest tests/unit/core/metadata_primitives/test_metadata_primitives_core.py -q -p pytest_cov --cov=benchbox/metadata_primitives.py --cov-report=term-missing", "description": "w17 targeted coverage: metadata_primitives facade", "expected": "The metadata_primitives facade reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 17}, {"command": "uv run -- python -m pytest tests/unit/core/write_primitives/test_benchmark_execution.py tests/unit/core/write_primitives/test_benchmark_operations.py tests/unit/benchmarks/test_write_primitives_validation.py -q -p pytest_cov --cov=benchbox/core/write_primitives/benchmark.py --cov-report=term-missing", "description": "w18 targeted coverage: write_primitives benchmark", "expected": "The write_primitives benchmark module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 18}, {"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_error_recovery.py tests/unit/tpcdi/test_phase3_etl_enhanced.py -q -p pytest_cov --cov=benchbox/core/tpcdi/etl/error_recovery.py --cov-report=term-missing", "description": "w19 targeted coverage: TPC-DI error recovery", "expected": "The TPC-DI error recovery module reports at least 80% coverage.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 19}, {"command": "make coverage-fast", "description": "The project fast lane still passes with coverage enabled.", "expected": "Fast tests pass and total measured coverage remains >= 80%.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 20}, {"command": "uv run -- python -m coverage json -o /tmp/coverage_bottom25_verify.json && uv run -- python -c \"import json; targets = [ 'benchbox/platforms/databricks/dataframe_adapter.py', 'benchbox/platforms/spark.py', 'benchbox/platforms/credentials/redshift.py', 'benchbox/core/transaction_primitives/dataframe_operations.py', 'benchbox/platforms/clickhouse/diagnostics.py', 'benchbox/core/write_primitives/benchmark.py', 'benchbox/core/tuning/generators/pg_mooncake.py', 'benchbox/metadata_primitives.py', 'benchbox/core/tpcds/generator/manager.py', 'benchbox/core/ssb/generator.py', 'benchbox/platforms/databricks/credentials.py', 'benchbox/core/tpcdi/etl/error_recovery.py', 'benchbox/core/datavault/benchmark.py', 'benchbox/experimental/gpu/benchmark.py', 'benchbox/platforms/base/cloud_spark/mixins.py', 'benchbox/platforms/aws/glue_adapter.py', 'benchbox/platforms/dataframe/iceberg_maintenance.py', 'benchbox/platforms/questdb.py', 'benchbox/experimental/aiml_functions/benchmark.py', 'benchbox/platforms/lakesail.py', 'benchbox/core/coffeeshop/dataframe_queries/queries.py', 'benchbox/platforms/firebolt.py', 'benchbox/platforms/dataframe/expression_family.py', 'benchbox/platforms/clickhouse_cloud.py', 'benchbox/core/amplab/dataframe_queries/queries.py']; data = json.load(open('/tmp/coverage_bottom25_verify.json')); missing = [t for t in targets if t not in data['files']]; below = [(t, round(data['files'][t]['summary']['percent_covered'], 2)) for t in targets if t in data['files'] and data['files'][t]['summary']['percent_covered'] < 80]; assert not missing, missing; assert not below, below; print('all 25 target modules >= 80%')\"", "description": "Recompute the filtered module ranking from the coverage JSON after the final fast-lane run.", "expected": "The command exits cleanly and prints `all 25 target modules >= 80%`.", "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "last_result": null, "last_run": null, "seq": 21}], "work": [{"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Low-cost helper top-ups: pg_mooncake DDL generator and ClickHouse diagnostics", "wid": "w1"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Core Spark adapter branches", "wid": "w10"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "LakeSail adapter branches", "wid": "w11"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Glue adapter orchestration branches", "wid": "w12"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Firebolt adapter loading and fallback branches", "wid": "w13"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "ClickHouse Cloud adapter external-table branches", "wid": "w14"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Redshift credential and diagnostics branches", "wid": "w15"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "QuestDB adapter branches and ILP helpers", "wid": "w16"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Metadata primitives facade coverage", "wid": "w17"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write-primitives benchmark execution paths", "wid": "w18"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "TPC-DI error recovery engine", "wid": "w19"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DataFrame foundation: expression_family and iceberg_maintenance", "wid": "w2"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Transaction DataFrame operations: capabilities, format validation, atomic paths", "wid": "w3"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "DataFrame query registries: CoffeeShop and AMPLab expression/pandas parity", "wid": "w4"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generator and wrapper flow: TPC-DS manager, SSB generator, DataVault benchmark", "wid": "w5"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Experimental benchmark wrappers: GPU and AIML functions", "wid": "w6"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Databricks DataFrame adapter branches", "wid": "w7"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Databricks credential defaults and validation branches", "wid": "w8"}, {"evidence": null, "item_id": "coverage-current-bottom-25-modules-to-80pct-apr-2026", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cloud-Spark DDL and config mixins", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Address test execution gaps and coverage enforcement. Analysis (Dec 2025) found:\n- 7,876 total tests (7,318 unit + 558 integration)\n- Platform adapter test coverage: 94% (17/18 adapters have tests)\n- CRITICAL: 90.5% of unit test files lack @pytest.mark markers\n- CI only runs tests marked \"fast\", causing unmarked tests to be skipped\n- This gap caused the PYSPARK_AVAILABLE NameError bug (commit 9d524094)", "id": "coverage-improvement", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Test Infrastructure & Coverage Improvement", "verifications": [], "work": [], "worktree": "observability-quality"} -{"anti_patterns": [{"INSTEAD": "assert key clauses, table refs, and parameter handling", "dont": "assert entire generated SQL strings verbatim", "why": "formatting churn creates brittle tests"}, {"INSTEAD": "add focused tests for existing behavior", "dont": "modify production code only to satisfy coverage targets", "why": "this risks behavioral drift"}, {"INSTEAD": "mock network/client layers", "dont": "write tests that require live databases or cloud credentials", "why": "CI is offline for these integrations"}], "approach": "Implement tests in focused batches by domain. Reuse existing fixture and\nmocking patterns in tests/unit/benchmarks/, tests/unit/core/, and\ntests/unit/platforms/. For dataframe_queries modules, validate representative\nquery builder behavior (structure, required references, deterministic outputs)\ninstead of asserting full SQL text equality. For cloud/platform modules,\nisolate credential parsing, request construction, and error mapping with\nmocks only.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-10T00:00:00Z", "completed_pr": null, "created_at": "2026-02-10T00:00:00Z", "deferrals": [], "deps": [], "description": "The following files are still below the 50% per-file coverage floor and need\ntargeted unit tests. This item defines a detailed execution plan that groups\nfiles by domain so coverage can be raised systematically without production\ncode churn.\n\nTarget files:\n- benchbox/core/tpcdi/worker_pool_examples.py\n- benchbox/core/tpcds/reporting.py\n- benchbox/core/tpch/reporting.py\n- benchbox/platforms/dataframe/ducklake_maintenance.py\n- benchbox/core/results/display.py\n- benchbox/core/metadata_primitives/generator.py\n- benchbox/core/tpcds/dataframe_queries/queries.py\n- benchbox/core/tpcdi/etl/transformations.py\n- benchbox/core/tpcdi/source_generators.py\n- benchbox/core/tpcdi/etl/pipeline.py\n- benchbox/core/tpch/maintenance_test.py\n- benchbox/core/analysis/insights.py\n- benchbox/core/datavault/dataframe_queries/queries.py\n- benchbox/core/amplab/benchmark.py\n- benchbox/core/tpcdi/etl/batch.py\n- benchbox/core/analysis/ranking.py\n- benchbox/core/nyctaxi/benchmark.py\n- benchbox/core/ssb/benchmark.py\n- benchbox/core/read_primitives/dataframe_queries.py\n- benchbox/core/ssb/dataframe_queries/queries.py\n- benchbox/core/amplab/dataframe_queries/queries.py\n- benchbox/core/joinorder/generator.py\n- benchbox/core/nyctaxi/dataframe_queries/queries.py\n- benchbox/core/nyctaxi/downloader.py\n- benchbox/core/clickbench/benchmark.py\n- benchbox/core/tpcdi/etl/scd_processor.py\n- benchbox/core/h2odb/benchmark.py\n- benchbox/core/transaction_primitives/benchmark.py\n- benchbox/core/results/timing.py\n- benchbox/core/tpcdi/generator/dimensions.py\n- benchbox/platforms/base/data_loading.py\n- benchbox/core/amplab/generator.py\n- benchbox/core/concurrency/pool_tester.py\n- benchbox/core/tpcdi/generator/facts.py\n- benchbox/core/concurrency/executor.py\n- benchbox/core/clickbench/dataframe_queries/queries.py\n- benchbox/core/tpcds/official_benchmark.py\n- benchbox/core/coffeeshop/dataframe_queries/queries.py\n- benchbox/cli/exceptions.py\n- benchbox/utils/cloud_storage.py\n- benchbox/core/metadata_primitives/benchmark.py\n- benchbox/platforms/athena.py\n- benchbox/platforms/fabric_warehouse.py\n- benchbox/core/ssb/generator.py\n- benchbox/core/tpch/throughput_test.py\n- benchbox/core/write_primitives/benchmark.py\n- benchbox/platforms/influxdb/client.py\n- benchbox/core/ai_primitives/dataframe_operations.py\n- benchbox/core/tpcds/generator/runner.py\n- benchbox/core/metadata_primitives/dataframe_operations.py\n- benchbox/core/tpcds/c_tools.py\n- benchbox/platforms/dataframe/__init__.py\n- benchbox/cli/commands/run.py\n- benchbox/core/results/anonymization.py\n- benchbox/platforms/redshift.py\n- benchbox/core/tpcds/power_test.py\n- benchbox/platforms/bigquery.py\n\nCoverage goal: each listed file reports >=50% line coverage in the per-file\nreport generated by the CI-aligned coverage workflow.", "id": "coverage-per-module-50-remaining-wave-2026-02-10", "preserves": ["Benchmark query generation semantics remain intact for TPC-H, TPC-DS, SSB, Amplab, NYC Taxi, and ClickBench", "No live cloud access is required for unit tests (all cloud SDK behavior is mocked)", "benchbox CLI behavior and option parsing in run commands remains unchanged"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/integration/"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/coverage-per-module-50-remaining-wave-2026-02-10.yaml"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Raise remaining sub-50% files to >=50% coverage (wave 2026-02-10)", "verifications": [{"command": "make coverage-per-file", "description": "Coverage report runs in CI profile without regressions", "expected": "Updated per-file report shows each listed file at >=50%", "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit -m 'fast or unit' -q", "description": "Fast/unit tests remain green for changed coverage areas", "expected": "all tests pass, 0 failures", "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": [], "notes": "Run per-file coverage and record exact missing ranges for each file to\nprioritize high-yield branches and pure helper functions first.\nBaseline captured from `make coverage-per-file` on 2026-02-10.\nSummary:\n- Target files: 57\n- Below 50%: 49\n- Already >=50%: 8\nHighest-priority quick wins by smallest line delta to 50%:\n- benchbox/platforms/dataframe/__init__.py: need +2 lines\n- benchbox/core/ai_primitives/dataframe_operations.py: need +4 lines\n- benchbox/core/tpcds/generator/runner.py: need +7 lines\n- benchbox/platforms/influxdb/client.py: need +9 lines\n- benchbox/core/ssb/generator.py: need +14 lines\nLargest effort files by line delta to 50%:\n- benchbox/core/tpcds/dataframe_queries/queries.py: need +1930 lines\n- benchbox/core/read_primitives/dataframe_queries.py: need +635 lines\n- benchbox/core/tpcdi/etl/transformations.py: need +461 lines\n- benchbox/core/tpcdi/etl/batch.py: need +350 lines\n- benchbox/core/datavault/dataframe_queries/queries.py: need +314 lines\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline missing lines for all listed files and rank by easiest gains", "wid": "w1"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "95 tests added. Results:\n- ducklake_maintenance.py: 0% -> 86% (DONE)\n- data_loading.py: 45% -> 50% (DONE)\n- influxdb/client.py: 43% -> 51% (DONE)\n- fabric_warehouse.py: 41% -> 42% (remaining for w11)\n- bigquery.py: 49% -> 49% (remaining for w11)\n- redshift.py: 48% -> 48% (remaining for w11)\n- athena.py: 37% -> 37% (remaining for w11)\n- dataframe/__init__.py: 46% -> 46% (structural limit: except blocks\n for installed packages can never execute)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover remaining platform modules to >=50% with mocked cloud clients", "wid": "w10"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w10", "w2", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "For any file still below 50%, add targeted edge/error-path tests and\nrerun coverage-per-file until all listed files pass threshold.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run per-batch tests and close remaining <50% gaps file-by-file", "wid": "w11"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w11"], "notes": "Produce before/after table for each listed file and capture command output\nproving all targets are >=50%.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Finalize coverage evidence and prepare completion report", "wid": "w12"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/results/display.py\n- benchbox/core/results/timing.py\n- benchbox/core/results/anonymization.py\n- benchbox/core/analysis/insights.py\n- benchbox/core/analysis/ranking.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover core results and analysis modules to >=50%", "wid": "w2"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/tpcdi/worker_pool_examples.py\n- benchbox/core/tpcdi/source_generators.py\n- benchbox/core/tpcdi/etl/transformations.py\n- benchbox/core/tpcdi/etl/pipeline.py\n- benchbox/core/tpcdi/etl/batch.py\n- benchbox/core/tpcdi/etl/scd_processor.py\nProgress snapshot (2026-02-10):\n- worker_pool_examples.py: 90.5% (complete for target)\n- source_generators.py: 84.0% (complete for target)\n- pipeline.py: 65.4% (complete for target)\n- transformations.py: 11.8% (remaining)\n- batch.py: 17.1% (remaining)\n- scd_processor.py: 24.2% (remaining)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover TPC-DI ETL and orchestration modules to >=50%", "wid": "w3"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w3"], "notes": "Files:\n- benchbox/core/tpcdi/generator/dimensions.py\n- benchbox/core/tpcdi/generator/facts.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover TPC-DI generator dimension/fact modules to >=50%", "wid": "w4"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/tpcds/reporting.py\n- benchbox/core/tpcds/official_benchmark.py\n- benchbox/core/tpcds/power_test.py\n- benchbox/core/tpcds/c_tools.py\n- benchbox/core/tpcds/generator/runner.py\n- benchbox/core/tpch/reporting.py\n- benchbox/core/tpch/maintenance_test.py\n- benchbox/core/tpch/throughput_test.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover TPC-DS and TPC-H reporting and execution modules to >=50%", "wid": "w5"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/amplab/benchmark.py\n- benchbox/core/amplab/generator.py\n- benchbox/core/nyctaxi/benchmark.py\n- benchbox/core/nyctaxi/downloader.py\n- benchbox/core/ssb/benchmark.py\n- benchbox/core/ssb/generator.py\n- benchbox/core/clickbench/benchmark.py\n- benchbox/core/h2odb/benchmark.py\n- benchbox/core/write_primitives/benchmark.py\n- benchbox/core/transaction_primitives/benchmark.py\n- benchbox/core/metadata_primitives/benchmark.py\n- benchbox/core/metadata_primitives/generator.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover benchmark orchestrator modules for remaining suites to >=50%", "wid": "w6"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/tpcds/dataframe_queries/queries.py\n- benchbox/core/datavault/dataframe_queries/queries.py\n- benchbox/core/ssb/dataframe_queries/queries.py\n- benchbox/core/amplab/dataframe_queries/queries.py\n- benchbox/core/nyctaxi/dataframe_queries/queries.py\n- benchbox/core/clickbench/dataframe_queries/queries.py\n- benchbox/core/coffeeshop/dataframe_queries/queries.py\n- benchbox/core/read_primitives/dataframe_queries.py\n- benchbox/core/ai_primitives/dataframe_operations.py\n- benchbox/core/metadata_primitives/dataframe_operations.py\nCovered via PolarsDataFrameAdapter-backed context for AMPLab (8 expression +\n2 pandas), SSB (13), CoffeeShop (11, 3 xfail due to production bugs).\nAI primitives: enums, capabilities, results, manager, pure functions\n(classify, extract_entities, cosine/euclidean/top_k).\nMetadata primitives: enums, capabilities, presets, results, manager,\nschema introspection execution.\nLarge modules (TPC-DS, read_primitives, NYC Taxi, ClickBench, Data Vault)\ndeferred to w11 gap-closing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover dataframe query modules with representative assertions to >=50%", "wid": "w7"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/concurrency/pool_tester.py (already has 61 tests, >=50%)\n- benchbox/core/concurrency/executor.py (42 tests added)\n- benchbox/core/joinorder/generator.py (42 tests added)\n- benchbox/utils/cloud_storage.py (already has comprehensive test suite, >=50%)\npool_tester.py and cloud_storage.py already had sufficient coverage.\nexecutor.py: ConcurrentLoadExecutor run, failures, connection errors,\nqueue metrics, resource monitoring. joinorder/generator.py: init variants,\nrow counts, all _generate_* methods, dimension/relationship tables, CSV/manifest.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover concurrency, joinorder, and utility modules to >=50%", "wid": "w8"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/cli/exceptions.py (72% -> 88%)\n- benchbox/cli/commands/run.py (47% -> 50%)\n52 tests: PlatformOptionParamType, _build_execution_context,\n_render_post_run_charts, _describe_platform_options, setup_verbose_logging,\nErrorHandler branches, ValidationRules, ErrorContext, BenchboxCLIError.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover CLI exception/run command paths with parser and error tests to >=50%", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use deterministic mocks/fakes", "dont": "add integration tests that require cloud credentials or external services", "why": "CI must run offline"}, {"INSTEAD": "use unittest.mock.patch or lazy imports", "dont": "import heavy cloud SDKs at module level in test files", "why": "they add 2-5s import overhead and may not be installed"}, {"INSTEAD": "assert observable behavior and error-path contracts", "dont": "raise coverage by weakening assertions", "why": "it masks regressions"}, {"INSTEAD": "add any new deferment as explicit TODO update", "dont": "silently defer additional modules beyond the approved list", "why": "scope creep undermines the gate"}, {"INSTEAD": "test query structure, table references, and parameters instead", "dont": "test auto-generated SQL strings character-by-character", "why": "SQL formatting changes would cause false failures"}], "approach": "Follow existing test patterns in tests/unit/{cli,core,tpcdi,platforms} and\nprefer fast unit tests with mocks over integration tests. Implement coverage\nlift in waves ordered by fastest impact (near-50% quick wins first, then\nmedium-effort modules, then large modules requiring 5-10 tests each, then\nCLI, then TPC-DI). Recompute and checkpoint coverage after each wave using\ncoverage.json plus the per-file checker to keep progress measurable.\n\nNaming convention: new files are named test_{module}_coverage.py when an\nexisting test file covers the same source module. All tests marked\n@pytest.mark.fast and @pytest.mark.unit, pure mocks only.\n\nWithin each phase, batches are independent and can be parallelized via\nTask agents. Each batch creates all its test files, then runs verification\nbefore proceeding to the next phase.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-08T00:00:00Z", "deferrals": [], "deps": [], "description": "Close remaining test coverage gaps by enforcing a per-module >=50% floor for\nactive benchbox/ modules while temporarily deferring a small set of\nlegacy/demo/heavy-parser files.\n\nThe HTML coverage report (2026-02-08, 59.59% overall) identifies ~67 modules\nbelow 50%. This plan provides a concrete per-file implementation sequence\nwith specific test file names, estimated test counts, and batch groupings\nfor parallel execution.\n\nImplementation sequence (8 phases):\n 1. Add active-scope per-file coverage checker and Makefile workflow.\n 2. Quick wins - modules at 43-49% needing 2-4 tests each (~21 files).\n 3. Medium effort - tuning, credentials, TPC runners, platform adapters (~20 files).\n 4. High effort - large modules requiring 5-10 tests each (~11 files).\n 5. CLI coverage - zero-coverage through near-50% CLI modules (~20 files).\n 6. Active TPC-DI modules under 50% (not deferred).\n 7. Final sweep - fill any remaining gaps found after verification.\n 8. Enforce the gate in local CI workflow and keep deferred-module tracking.\n\nTotal: ~87 new test files, ~405 test functions.\n\nDeferred scope for this campaign:\n - benchbox/core/tpcdi/tools/file_parsers.py\n - benchbox/core/tpcdi/tools/data_cleaners.py\n - benchbox/core/tpcdi/source_generators.py\n - benchbox/core/tpcdi/etl/transformations.py\n - benchbox/core/tpcdi/etl/batch.py\n - benchbox/core/tpchavoc/variants.py\n - benchbox/core/tpc_validation.py\n - benchbox/core/tpc_patterns.py\n - benchbox/core/tpc_metrics.py\n - benchbox/core/tpcdi/streaming.py\n - benchbox/core/tpcdi/worker_pool_examples.py\n - benchbox/core/tpcds/reporting.py\n - benchbox/core/tpch/reporting.py\n - benchbox/platforms/dataframe/ducklake_maintenance.py\n - benchbox/core/tpcds/dataframe_queries/rollup_helper.py\n - benchbox/core/tpcds/dataframe_queries/channel_helper.py\n - benchbox/core/visualization/chart_generator.py\n - benchbox/mcp/__main__.py\n - benchbox/core/results/display.py\n - benchbox/core/metadata_primitives/generator.py\n - benchbox/core/tpcds/dataframe_queries/queries.py\n - benchbox/core/tpch/maintenance_test.py\n - benchbox/core/tpcdi/etl/pipeline.py\n - benchbox/core/query_plans/parsers/sqlite.py\n - benchbox/core/analysis/insights.py\n - benchbox/core/runner/conversion.py\n - benchbox/core/datavault/dataframe_queries/queries.py\n - benchbox/cli/commands/results.py\n - benchbox/core/amplab/benchmark.py\n - benchbox/core/analysis/ranking.py\n - benchbox/core/ssb/benchmark.py\n - benchbox/core/nyctaxi/benchmark.py\n - benchbox/core/read_primitives/dataframe_queries.py\n - benchbox/core/ssb/dataframe_queries/queries.py\n - benchbox/core/joinorder/generator.py\n - benchbox/core/nyctaxi/downloader.py\n - benchbox/core/nyctaxi/dataframe_queries/queries.py\n - benchbox/core/amplab/dataframe_queries/queries.py\n - benchbox/core/clickbench/benchmark.py\n - benchbox/core/tpcdi/etl/scd_processor.py\n - benchbox/core/transaction_primitives/schema.py\n - benchbox/core/transaction_primitives/benchmark.py\n - benchbox/core/h2odb/benchmark.py\n - benchbox/core/tpcdi/generator/dimensions.py\n - benchbox/core/results/timing.py\n - benchbox/core/concurrency/pool_tester.py\n - benchbox/core/amplab/generator.py\n - benchbox/core/metadata_primitives/schema.py\n - benchbox/core/tpcdi/generator/facts.py\n - benchbox/core/concurrency/executor.py\n - benchbox/core/tpcds/official_benchmark.py\n - benchbox/core/clickbench/dataframe_queries/queries.py\n - benchbox/core/coffeeshop/dataframe_queries/queries.py\n - benchbox/cli/exceptions.py\n - benchbox/core/clickbench/generator.py\n - benchbox/utils/cloud_storage.py\n - benchbox/cli/commands/plot.py\n - benchbox/core/joinorder/queries.py\n - benchbox/platforms/athena.py\n - benchbox/core/metadata_primitives/benchmark.py\n - benchbox/platforms/influxdb/setup.py\n - benchbox/platforms/fabric_warehouse.py\n - benchbox/platforms/influxdb/client.py\n - benchbox/core/write_primitives/benchmark.py\n - benchbox/core/ssb/generator.py\n - benchbox/core/tpch/throughput_test.py\n - benchbox/core/tpcds/generator/runner.py\n - benchbox/core/ai_primitives/dataframe_operations.py\n - benchbox/core/amplab/queries.py\n - benchbox/core/joinorder/schema.py\n - benchbox/core/metadata_primitives/dataframe_operations.py\n - benchbox/core/h2odb/generator.py\n - benchbox/platforms/dataframe/__init__.py\n - benchbox/core/tpcds/c_tools.py\n - benchbox/cli/commands/run.py\n - benchbox/platforms/redshift.py\n - benchbox/core/tpchavoc/queries.py\n - benchbox/core/results/anonymization.py\n - benchbox/platforms/clickhouse/workload.py\n - benchbox/platforms/bigquery.py\n - benchbox/core/tpcds/power_test.py\n - benchbox/platforms/databricks/adapter.py", "id": "coverage-per-module-50-temporary-split", "preserves": ["Current CLI command behavior and argument contracts remain unchanged", "Existing fast/unit test suite remains green with no live platform dependency", "Existing overall --cov-fail-under workflow remains available during migration", "TPC-DI SQL/DataFrame ETL parity behavior verified by existing parity tests"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Raise benchbox/ per-module coverage to >=50% with temporary split", "verifications": [{"command": "uv run -- python scripts/check_per_file_coverage.py --min 50 --coverage-json coverage.json", "description": "Active-scope per-file checker reports and fails correctly", "expected": "No active-scope module below 50%, exit code 0", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 1}, {"command": "make ci-test", "description": "Fast coverage workflow remains green at project floor", "expected": "pytest fast suite passes and coverage gate remains >=50%", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit -m fast --tb=short", "description": "Targeted coverage wave tests pass", "expected": "all selected fast unit tests pass, 0 failures", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests -p pytest_cov --cov=benchbox --cov-report=term-missing:skip-covered -n auto -m 'not slow and not stress and not live_integration' --tb=short -q --benchmark-disable", "description": "Per-phase verification", "expected": "no benchbox/ module below 50% in output", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add scripts/check_per_file_coverage.py for coverage.json-based active-scope checks", "wid": "w1"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_dataframe_init.py \u2192 platforms/dataframe/__init__.py (46% \u2192 50%, ~2 tests)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record before/after failing-module snapshots per wave", "wid": "w100"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm no active-scope module remains below 50%", "wid": "w101"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2B: Utils/core near-50% (5 files, ~15 tests)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/utils/test_verbosity_coverage.py \u2192 utils/verbosity.py (49% \u2192 50%, ~3 tests)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/visualization/test_visualization_exporters.py \u2192 core/visualization/exporters.py (44% \u2192 50%, ~4 tests)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/examples/test_dryrun_utils.py \u2192 examples/dryrun_utils.py (47% \u2192 50%, ~2 tests)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tsbs_devops/test_tsbs_benchmark_coverage.py \u2192 core/tsbs_devops/benchmark.py (43% \u2192 50%, ~4 tests)", "wid": "w15"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_clickhouse_cloud_coverage.py \u2192 platforms/clickhouse_cloud.py (46% \u2192 50%, ~2 tests)", "wid": "w16"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2C: Platform adapters near-50% (6 files, ~16 tests)", "wid": "w17"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_azure_synapse_coverage.py \u2192 platforms/azure_synapse.py (49% \u2192 50%, ~3 tests)", "wid": "w18"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/gcp/test_dataproc_serverless_coverage.py \u2192 platforms/gcp/dataproc_serverless_adapter.py (49% \u2192 50%, ~2 tests)", "wid": "w19"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fail when any active-scope benchbox/*.py file is below 50%", "wid": "w2"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_timescaledb_coverage.py \u2192 platforms/timescaledb.py (48% \u2192 50%, ~2 tests)", "wid": "w20"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/aws/test_athena_spark_coverage.py \u2192 platforms/aws/athena_spark_adapter.py (45% \u2192 50%, ~3 tests)", "wid": "w21"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/azure/test_synapse_spark_coverage.py \u2192 platforms/azure/synapse_spark_adapter.py (44% \u2192 50%, ~3 tests)", "wid": "w22"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/gcp/test_dataproc_coverage.py \u2192 platforms/gcp/dataproc_adapter.py (43% \u2192 50%, ~3 tests)", "wid": "w23"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2D: Core benchmark modules near-50% (5 files, ~18 tests)", "wid": "w24"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/test_tpcds_maintenance_coverage.py \u2192 core/tpcds/maintenance_test.py (48% \u2192 50%, ~3 tests)", "wid": "w25"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/generator/test_streaming_coverage.py \u2192 core/tpcds/generator/streaming.py (45% \u2192 50%, ~4 tests)", "wid": "w26"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_pyspark_maintenance_coverage.py \u2192 platforms/dataframe/pyspark_maintenance.py (44% \u2192 50%, ~3 tests)", "wid": "w27"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch_skew/test_tpch_skew_benchmark_coverage.py \u2192 core/tpch_skew/benchmark.py (39% \u2192 50%, ~4 tests)", "wid": "w28"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_cudf_coverage.py \u2192 platforms/cudf.py (42% \u2192 50%, ~4 tests)", "wid": "w29"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Report sorted failing files with percentage and statement count", "wid": "w3"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2E: MCP and misc (2 files, ~6 tests)", "wid": "w30"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w30"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/mcp/test_mcp_init_coverage.py \u2192 mcp/__init__.py (31% \u2192 50%, ~3 tests)", "wid": "w31"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w30"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/utils/format_converters/test_vortex_converter_coverage.py \u2192 utils/format_converters/vortex_converter.py (14% \u2192 50%, ~3 tests)", "wid": "w32"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3A: Tuning subsystem (5 files, ~32 tests)", "wid": "w33"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/test_tuning_validation.py \u2192 core/tuning/validation.py (13% \u2192 50%, ~10 tests)", "wid": "w34"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/test_tuning_metadata.py \u2192 core/tuning/metadata.py (13% \u2192 50%, ~10 tests)", "wid": "w35"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/generators/test_azure_synapse_ddl.py \u2192 core/tuning/generators/azure_synapse.py (24% \u2192 50%, ~4 tests)", "wid": "w36"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/generators/test_clickhouse_ddl.py \u2192 core/tuning/generators/clickhouse.py (22% \u2192 50%, ~4 tests)", "wid": "w37"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/generators/test_firebolt_ddl.py \u2192 core/tuning/generators/firebolt.py (19% \u2192 50%, ~4 tests)", "wid": "w38"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3B: Credentials (3 files, ~21 tests)", "wid": "w39"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support deferred list and configurable threshold via CLI flags", "wid": "w4"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_snowflake_coverage.py \u2192 platforms/credentials/snowflake.py (32% \u2192 50%, ~6 tests)", "wid": "w40"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_bigquery_coverage.py \u2192 platforms/credentials/bigquery.py (29% \u2192 50%, ~6 tests)", "wid": "w41"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_databricks_credentials_coverage.py \u2192 platforms/databricks/credentials.py (28% \u2192 50%, ~6 tests)", "wid": "w42"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3C: TPC benchmark runners (4 files, ~22 tests)", "wid": "w43"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/benchmark/test_tpcds_runner_coverage.py \u2192 core/tpcds/benchmark/runner.py (34% \u2192 50%, ~7 tests)", "wid": "w44"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/test_throughput_test_coverage.py \u2192 core/tpcds/throughput_test.py (32% \u2192 50%, ~5 tests)", "wid": "w45"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch/test_official_benchmark_coverage.py \u2192 core/tpch/official_benchmark.py (32% \u2192 50%, ~4 tests)", "wid": "w46"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/transaction_primitives/test_dataframe_ops_coverage.py \u2192 core/transaction_primitives/dataframe_operations.py (38% \u2192 50%, ~6 tests)", "wid": "w47"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3D: Platform adapters (5 files, ~24 tests)", "wid": "w48"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_motherduck_coverage.py \u2192 platforms/motherduck.py (32% \u2192 50%, ~5 tests)", "wid": "w49"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make coverage-per-file target and document usage", "wid": "w5"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_influxdb_adapter_coverage.py \u2192 platforms/influxdb/adapter.py (31% \u2192 50%, ~5 tests)", "wid": "w50"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_clickhouse_metadata_coverage.py \u2192 platforms/clickhouse/metadata.py (31% \u2192 50%, ~4 tests)", "wid": "w51"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/aws/test_emr_serverless_coverage.py \u2192 platforms/aws/emr_serverless_adapter.py (38% \u2192 50%, ~5 tests)", "wid": "w52"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_iceberg_maintenance_coverage.py \u2192 platforms/dataframe/iceberg_maintenance.py (38% \u2192 50%, ~5 tests)", "wid": "w53"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3E: DataFrame adapters and remaining credentials (3 files, ~20 tests)", "wid": "w54"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w54"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_expression_family_coverage.py \u2192 platforms/dataframe/expression_family.py (35% \u2192 50%, ~6 tests)", "wid": "w55"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w54"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_modin_df_coverage.py \u2192 platforms/dataframe/modin_df.py (27% \u2192 50%, ~5 tests)", "wid": "w56"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w54"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_redshift_coverage.py \u2192 platforms/credentials/redshift.py (24% \u2192 50%, ~9 tests)", "wid": "w57"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 4A: Largest core modules (5 files, ~40 tests)", "wid": "w58"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_unified_frame_coverage.py \u2192 platforms/dataframe/unified_frame.py (35% \u2192 50%, ~10 tests)", "wid": "w59"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update pyproject coverage omit/deferred notes for temporary split", "wid": "w6"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch/test_dataframe_queries_coverage.py \u2192 core/tpch/dataframe_queries.py (12% \u2192 50%, ~10 tests)", "wid": "w60"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/transaction_primitives/test_generator_coverage.py \u2192 core/transaction_primitives/generator.py (11% \u2192 50%, ~8 tests)", "wid": "w61"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch/test_parameter_extractor_coverage.py \u2192 core/tpch/parameter_extractor.py (22% \u2192 50%, ~6 tests)", "wid": "w62"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py \u2192 core/tpchavoc/validation.py (14% \u2192 50%, ~6 tests)", "wid": "w63"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 4B: More large modules (6 files, ~35 tests)", "wid": "w64"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch_skew/test_skew_generator_coverage.py \u2192 core/tpch_skew/generator.py (17% \u2192 50%, ~7 tests)", "wid": "w65"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/test_parameter_extractor_coverage.py \u2192 core/tpcds/parameter_extractor.py (15% \u2192 50%, ~4 tests)", "wid": "w66"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tsbs_devops/test_tsbs_df_queries_coverage.py \u2192 core/tsbs_devops/dataframe_queries/queries.py (19% \u2192 50%, ~7 tests)", "wid": "w67"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_cudf_df_coverage.py \u2192 platforms/dataframe/cudf_df.py (24% \u2192 50%, ~5 tests)", "wid": "w68"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_dask_df_coverage.py \u2192 platforms/dataframe/dask_df.py (20% \u2192 50%, ~6 tests)", "wid": "w69"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2A: Trivial small modules (3 files, ~8 tests)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/databricks/test_databricks_df_adapter_coverage.py \u2192 platforms/databricks/dataframe_adapter.py (18% \u2192 50%, ~6 tests)", "wid": "w70"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5A: Zero-coverage CLI modules (2 files, ~12 tests)", "wid": "w71"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w71"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_tuning_wizard.py \u2192 cli/tuning.py (0% \u2192 50%, ~6 tests)", "wid": "w72"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w71"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_onboarding_coverage.py \u2192 cli/onboarding.py (0% \u2192 50%, ~6 tests)", "wid": "w73"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5B: Low-coverage CLI commands (6 files, ~32 tests)", "wid": "w74"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_compare_plans_coverage.py \u2192 cli/commands/compare_plans.py (8% \u2192 50%, ~5 tests)", "wid": "w75"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_compare_dataframes_coverage.py \u2192 cli/commands/compare_dataframes.py (9% \u2192 50%, ~5 tests)", "wid": "w76"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_df_tuning_coverage.py \u2192 cli/commands/df_tuning.py (11% \u2192 50%, ~5 tests)", "wid": "w77"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_export_coverage.py \u2192 cli/commands/export.py (13% \u2192 50%, ~5 tests)", "wid": "w78"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_preferences_coverage.py \u2192 cli/preferences.py (15% \u2192 50%, ~7 tests)", "wid": "w79"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/visualization/test_visualization_exceptions.py \u2192 core/visualization/exceptions.py (33% \u2192 50%, ~3 tests)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_plan_history_coverage.py \u2192 cli/commands/plan_history.py (19% \u2192 50%, ~5 tests)", "wid": "w80"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5C: Medium-coverage CLI (7 files, ~33 tests)", "wid": "w81"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_datagen_coverage.py \u2192 cli/commands/datagen.py (22% \u2192 50%, ~5 tests)", "wid": "w82"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_show_plan_coverage.py \u2192 cli/commands/show_plan.py (24% \u2192 50%, ~4 tests)", "wid": "w83"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_compare_coverage.py \u2192 cli/commands/compare.py (28% \u2192 50%, ~5 tests)", "wid": "w84"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_shell_coverage.py \u2192 cli/commands/shell.py (30% \u2192 50%, ~5 tests)", "wid": "w85"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_run_official_coverage.py \u2192 cli/commands/run_official.py (33% \u2192 50%, ~5 tests)", "wid": "w86"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_system_coverage.py \u2192 cli/system.py (36% \u2192 50%, ~4 tests)", "wid": "w87"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_composite_params_coverage.py \u2192 cli/composite_params.py (39% \u2192 50%, ~5 tests)", "wid": "w88"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5D: Near-50% CLI modules (5 files, ~21 tests)", "wid": "w89"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/schema/test_tpcds_schema_registry.py \u2192 core/tpcds/schema/registry.py (27% \u2192 50%, ~3 tests)", "wid": "w9"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_config_coverage.py \u2192 cli/config.py (41% \u2192 50%, ~5 tests)", "wid": "w90"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_output_coverage.py \u2192 cli/output.py (43% \u2192 50%, ~5 tests)", "wid": "w91"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_tuning_group_coverage.py \u2192 cli/commands/tuning_group.py (40% \u2192 50%, ~4 tests)", "wid": "w92"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_tuning_cmd_coverage.py \u2192 cli/commands/tuning.py (46% \u2192 50%, ~2 tests)", "wid": "w93"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/presentation/test_system_presentation_coverage.py \u2192 cli/presentation/system.py (6% \u2192 50%, ~5 tests)", "wid": "w94"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Raise non-deferred TPC-DI modules under 50%: benchmark/generator/queries/loader/schema/validation/metrics/config/etl.py/etl/validation.py", "wid": "w95"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep ETL tests unit-heavy with mocked pipeline and deterministic fixtures", "wid": "w96"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run full coverage report and identify any modules still below 50%", "wid": "w97"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add targeted tests for remaining gaps discovered after phases 2-6", "wid": "w98"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enable failing mode for coverage-per-file in local CI/test workflow", "wid": "w99"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use unittest.mock.patch or lazy imports", "dont": "import heavy cloud SDKs at module level in test files", "why": "they add 2-5s import overhead and may not be installed"}, {"INSTEAD": "mock all SDK clients (boto3, google-cloud, snowflake-connector)", "dont": "require cloud credentials for unit tests", "why": "CI has no cloud access and tests must be runnable offline"}, {"INSTEAD": "test adapter configuration, connection logic, and error handling instead", "dont": "test platform-specific SQL dialect generation", "why": "dialect translation is covered by separate dialect tests"}], "approach": "Follow existing patterns in tests/unit/platforms/. For cloud adapters,\nmock all SDK clients (boto3, google-cloud, snowflake-connector). For\ncredential modules, test env var parsing, config file loading, and\nvalidation logic. For DataFrame adapters, use small in-memory DataFrames.\nunified_frame.py is the largest file - focus on the public API methods\nand expression translation.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Several platform adapter modules are below 50% test coverage. These include\nboth DataFrame platform abstractions and cloud/remote platform adapters.\n\nCurrent coverage breakdown:\n - platforms/dataframe/unified_frame.py: 31.0% (930 missing)\n - platforms/dataframe/benchmark_mixin.py: 38.5% (232 missing)\n - platforms/dataframe/expression_family.py: 45.7% (215 missing)\n - platforms/dataframe/dask_df.py: 26.1% (139 missing)\n - platforms/dataframe/ducklake_maintenance.py: 0.0% (126 missing)\n - platforms/credentials/redshift.py: 25.2% (333 missing)\n - platforms/credentials/bigquery.py: 31.1% (171 missing)\n - platforms/credentials/snowflake.py: 34.6% (123 missing)\n - platforms/databricks/credentials.py: 31.7% (127 missing)\n - platforms/databricks/dataframe_adapter.py: 23.6% (120 missing)\n - platforms/athena.py: 39.7% (356 missing)\n - platforms/fabric_warehouse.py: 43.9% (252 missing)\n - platforms/influxdb/adapter.py: 26.3% (101 missing)\n - platforms/influxdb/client.py: 44.0% (94 missing)\n - platforms/cudf.py: 48.1% (175 missing)\n - platforms/motherduck.py: 35.9% (84 missing)\n - platforms/clickhouse_cloud.py: 49.3% (35 missing)\n\nCloud platform tests should mock all external API calls. DataFrame platform\ntests can use in-memory data.", "id": "coverage-platforms", "preserves": ["All existing platform tests continue to pass", "No changes to production code"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Improve test coverage for platform adapter modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "All platform tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-platforms", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/unified_frame.py (31.0% \u2192 50%)", "wid": "w1"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/athena.py (39.7% \u2192 50%)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/fabric_warehouse.py (43.9% \u2192 50%)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/databricks/dataframe_adapter.py (23.6% \u2192 50%)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/influxdb/ (26-44% \u2192 50%)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/cudf.py (48.1% \u2192 50%)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/motherduck.py (35.9% \u2192 50%)", "wid": "w15"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/clickhouse_cloud.py (49.3% \u2192 50%)", "wid": "w16"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/benchmark_mixin.py (38.5% \u2192 50%)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/expression_family.py (45.7% \u2192 50%)", "wid": "w3"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/dask_df.py (26.1% \u2192 50%)", "wid": "w4"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/ducklake_maintenance.py (0% \u2192 50%)", "wid": "w5"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/credentials/redshift.py (25.2% \u2192 50%)", "wid": "w6"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/credentials/bigquery.py (31.1% \u2192 50%)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/credentials/snowflake.py (34.6% \u2192 50%)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/databricks/credentials.py (31.7% \u2192 50%)", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "every test should fail if the code under test is broken", "dont": "add tests just to hit line coverage", "why": "a test that can never fail is worse than no test"}, {"INSTEAD": "use real DuckDB/Polars/DataFusion wherever possible", "dont": "mock internal methods to test dispatchers", "why": "you're testing mock wiring not behavior"}, {"INSTEAD": "only omit genuinely untestable code (external infrastructure, experimental, bulk mechanical translations)", "dont": "omit files from coverage just because they are hard to test", "why": "that hides real gaps"}, {"INSTEAD": "name after the behavior being tested", "dont": "put 'coverage' in test filenames", "why": "test names should describe what they verify"}, {"INSTEAD": "assert the generated SQL string contains expected clauses", "dont": "test cloud SQL generation by asserting mock.called", "why": "that doesn't verify SQL correctness"}, {"INSTEAD": "every test must call a function and assert on its output", "dont": "write import-only tests", "why": "importing a module is not testing it"}], "approach": null, "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-26T00:00:00Z", "completed_pr": null, "created_at": "2026-03-25T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 131, "reason": "Requires full CDC/SCD pipeline infrastructure with staged incremental loads. 2,226 missing statements but extremely high effort. Omitted from scope and deferred until TPC-DI ETL testing infrastructure is built.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Gated on TPC-DI ETL testing infrastructure being built first; no evidence that infra exists yet, premature to promote.", "summary": "TPC-DI ETL deep coverage (transformations.py, batch.py, scd_processor.py)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 132, "reason": "Require real cloud Spark sessions or complex mocking. Better covered by platform-live-integration-tests TODO.", "resolution": "promoted", "resolved_item": "cloud-spark-adapters-emr-serverless-dataproc-serverless-syna", "resolved_reason": null, "summary": "Cloud Spark adapters (EMR Serverless, Dataproc Serverless, Synapse Spark)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 133, "reason": "Require external engines (Spark, LakeSail server). Config/validation logic was covered in dataframe-test-coverage w13; remaining gaps are engine-specific.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "PySpark and LakeSail DataFrame adapters"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 134, "reason": "Experimental code changes frequently. Testing to 80% is wasted effort if modules are restructured. Cover after modules graduate from experimental.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly deferred until GPU-benchmark/NL2SQL modules graduate from experimental; no evidence that has happened.", "summary": "Experimental modules (GPU benchmark, NL2SQL evaluator)"}], "deps": ["dataframe-test-coverage", "non-dataframe-test-coverage"], "description": "Coverage is at 71.4% (81,722 / 109,707 stmts). Target is 80% (87,766 covered).\nGap: 6,044 more lines to cover.\n\nTwo-pronged strategy:\n1. SCOPE REFINEMENT: Omit 3,526 stmts of untestable bulk (TPC-DI ETL complex\n transformations that require full CDC infrastructure, experimental concurrency\n code, ClickBench DF queries). This immediately adjusts the denominator and\n boosts measured coverage to ~76.3%. Gap after omit: 3,928 lines.\n2. TARGETED TESTS: Cover ~4,130 additional lines (5% buffer over gap) from\n high-value locally-testable files across 5 categories:\n - Core/CLI/DF framework: ~1,600 lines (base adapter, unified_frame, CLI\n commands, DF benchmark_suite, runner, base.py, utils)\n - Cloud SQL generation (top 4): ~1,000 lines (Redshift, Databricks, BigQuery,\n Snowflake - config validation + SQL string assertions)\n - Cloud SQL generation (tier 2): ~500 lines (Athena, Synapse, Firebolt,\n ClickHouse, PostgreSQL, Trino)\n - DF query implementations: ~280 lines (read_primitives DF queries)\n - Remaining high-miss modules: ~350 lines (metadata_primitives, DataFusion\n adapter, TPC-DS runner gaps)\n\nAnti-coverage-theater principles:\n- Every test must call a function and assert on specific output values\n- Use real engines (DuckDB, Polars, DataFusion) not mocks where possible\n- Test behavior not implementation - if the function changes how it works but\n produces the same result, the test should still pass\n- No import-only tests, no `assert result is not None`, no `assert True`\n- Cloud platform tests assert on generated SQL strings, not mock.called\n- Mark genuinely untestable code for exclusion rather than writing useless tests", "id": "coverage-to-80-percent", "preserves": ["All existing tests must continue to pass - new tests are additive", "Cloud adapter tests must not require real credentials", "Coverage config in pyproject.toml remains source of truth for local dev", "No `assert True`, `assert result is not None`, or `hasattr`-only assertions", "Test suite speed: `make test-fast` must complete in < 90 seconds"], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Reach 80% test coverage through scope refinement and targeted behavioral tests", "verifications": [], "work": [{"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Omit untestable bulk from coverage scope", "wid": "w1"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test BigQuery SQL generation and config validation", "wid": "w10"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Snowflake SQL generation and config validation", "wid": "w11"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test remaining cloud platforms and adapters", "wid": "w12"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test base.py BaseBenchmark abstract methods", "wid": "w13"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test remaining utils and supporting modules", "wid": "w14"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test read_primitives remaining DF query implementations", "wid": "w15"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFusion adapter expression and write paths", "wid": "w16"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w1", "w10", "w11", "w12", "w13", "w14", "w15", "w16", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align CI coverage config and ratchet fail_under to 80", "wid": "w17"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test base adapter concrete methods to 85%", "wid": "w2"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py string, date, and subquery operations", "wid": "w3"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CLI run command option parsing and validation", "wid": "w4"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CLI compare and benchmarks commands", "wid": "w5"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFrame benchmark_suite orchestration", "wid": "w6"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test core runner orchestration and config", "wid": "w7"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Redshift SQL generation and config validation", "wid": "w8"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Databricks SQL generation and config validation", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "assert entire generated SQL strings verbatim", "instead": "assert key clauses, table refs, and parameter handling", "why": "formatting churn creates brittle tests"}, {"dont": "modify production code only to satisfy coverage targets", "instead": "add focused tests for existing behavior", "why": "this risks behavioral drift"}, {"dont": "write tests that require live databases or cloud credentials", "instead": "mock network/client layers", "why": "CI is offline for these integrations"}], "approach": "Implement tests in focused batches by domain. Reuse existing fixture and\nmocking patterns in tests/unit/benchmarks/, tests/unit/core/, and\ntests/unit/platforms/. For dataframe_queries modules, validate representative\nquery builder behavior (structure, required references, deterministic outputs)\ninstead of asserting full SQL text equality. For cloud/platform modules,\nisolate credential parsing, request construction, and error mapping with\nmocks only.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-10T00:00:00Z", "completed_pr": null, "created_at": "2026-02-10T00:00:00Z", "deferrals": [], "deps": [], "description": "The following files are still below the 50% per-file coverage floor and need\ntargeted unit tests. This item defines a detailed execution plan that groups\nfiles by domain so coverage can be raised systematically without production\ncode churn.\n\nTarget files:\n- benchbox/core/tpcdi/worker_pool_examples.py\n- benchbox/core/tpcds/reporting.py\n- benchbox/core/tpch/reporting.py\n- benchbox/platforms/dataframe/ducklake_maintenance.py\n- benchbox/core/results/display.py\n- benchbox/core/metadata_primitives/generator.py\n- benchbox/core/tpcds/dataframe_queries/queries.py\n- benchbox/core/tpcdi/etl/transformations.py\n- benchbox/core/tpcdi/source_generators.py\n- benchbox/core/tpcdi/etl/pipeline.py\n- benchbox/core/tpch/maintenance_test.py\n- benchbox/core/analysis/insights.py\n- benchbox/core/datavault/dataframe_queries/queries.py\n- benchbox/core/amplab/benchmark.py\n- benchbox/core/tpcdi/etl/batch.py\n- benchbox/core/analysis/ranking.py\n- benchbox/core/nyctaxi/benchmark.py\n- benchbox/core/ssb/benchmark.py\n- benchbox/core/read_primitives/dataframe_queries.py\n- benchbox/core/ssb/dataframe_queries/queries.py\n- benchbox/core/amplab/dataframe_queries/queries.py\n- benchbox/core/joinorder/generator.py\n- benchbox/core/nyctaxi/dataframe_queries/queries.py\n- benchbox/core/nyctaxi/downloader.py\n- benchbox/core/clickbench/benchmark.py\n- benchbox/core/tpcdi/etl/scd_processor.py\n- benchbox/core/h2odb/benchmark.py\n- benchbox/core/transaction_primitives/benchmark.py\n- benchbox/core/results/timing.py\n- benchbox/core/tpcdi/generator/dimensions.py\n- benchbox/platforms/base/data_loading.py\n- benchbox/core/amplab/generator.py\n- benchbox/core/concurrency/pool_tester.py\n- benchbox/core/tpcdi/generator/facts.py\n- benchbox/core/concurrency/executor.py\n- benchbox/core/clickbench/dataframe_queries/queries.py\n- benchbox/core/tpcds/official_benchmark.py\n- benchbox/core/coffeeshop/dataframe_queries/queries.py\n- benchbox/cli/exceptions.py\n- benchbox/utils/cloud_storage.py\n- benchbox/core/metadata_primitives/benchmark.py\n- benchbox/platforms/athena.py\n- benchbox/platforms/fabric_warehouse.py\n- benchbox/core/ssb/generator.py\n- benchbox/core/tpch/throughput_test.py\n- benchbox/core/write_primitives/benchmark.py\n- benchbox/platforms/influxdb/client.py\n- benchbox/core/ai_primitives/dataframe_operations.py\n- benchbox/core/tpcds/generator/runner.py\n- benchbox/core/metadata_primitives/dataframe_operations.py\n- benchbox/core/tpcds/c_tools.py\n- benchbox/platforms/dataframe/__init__.py\n- benchbox/cli/commands/run.py\n- benchbox/core/results/anonymization.py\n- benchbox/platforms/redshift.py\n- benchbox/core/tpcds/power_test.py\n- benchbox/platforms/bigquery.py\n\nCoverage goal: each listed file reports >=50% line coverage in the per-file\nreport generated by the CI-aligned coverage workflow.", "id": "coverage-per-module-50-remaining-wave-2026-02-10", "preserves": ["Benchmark query generation semantics remain intact for TPC-H, TPC-DS, SSB, Amplab, NYC Taxi, and ClickBench", "No live cloud access is required for unit tests (all cloud SDK behavior is mocked)", "benchbox CLI behavior and option parsing in run commands remains unchanged"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/integration/"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/coverage-per-module-50-remaining-wave-2026-02-10.yaml"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Raise remaining sub-50% files to >=50% coverage (wave 2026-02-10)", "verifications": [{"command": "make coverage-per-file", "description": "Coverage report runs in CI profile without regressions", "expected": "Updated per-file report shows each listed file at >=50%", "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit -m 'fast or unit' -q", "description": "Fast/unit tests remain green for changed coverage areas", "expected": "all tests pass, 0 failures", "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": [], "notes": "Run per-file coverage and record exact missing ranges for each file to\nprioritize high-yield branches and pure helper functions first.\nBaseline captured from `make coverage-per-file` on 2026-02-10.\nSummary:\n- Target files: 57\n- Below 50%: 49\n- Already >=50%: 8\nHighest-priority quick wins by smallest line delta to 50%:\n- benchbox/platforms/dataframe/__init__.py: need +2 lines\n- benchbox/core/ai_primitives/dataframe_operations.py: need +4 lines\n- benchbox/core/tpcds/generator/runner.py: need +7 lines\n- benchbox/platforms/influxdb/client.py: need +9 lines\n- benchbox/core/ssb/generator.py: need +14 lines\nLargest effort files by line delta to 50%:\n- benchbox/core/tpcds/dataframe_queries/queries.py: need +1930 lines\n- benchbox/core/read_primitives/dataframe_queries.py: need +635 lines\n- benchbox/core/tpcdi/etl/transformations.py: need +461 lines\n- benchbox/core/tpcdi/etl/batch.py: need +350 lines\n- benchbox/core/datavault/dataframe_queries/queries.py: need +314 lines\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline missing lines for all listed files and rank by easiest gains", "wid": "w1"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "95 tests added. Results:\n- ducklake_maintenance.py: 0% -> 86% (DONE)\n- data_loading.py: 45% -> 50% (DONE)\n- influxdb/client.py: 43% -> 51% (DONE)\n- fabric_warehouse.py: 41% -> 42% (remaining for w11)\n- bigquery.py: 49% -> 49% (remaining for w11)\n- redshift.py: 48% -> 48% (remaining for w11)\n- athena.py: 37% -> 37% (remaining for w11)\n- dataframe/__init__.py: 46% -> 46% (structural limit: except blocks\n for installed packages can never execute)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover remaining platform modules to >=50% with mocked cloud clients", "wid": "w10"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w10", "w2", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": "For any file still below 50%, add targeted edge/error-path tests and\nrerun coverage-per-file until all listed files pass threshold.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run per-batch tests and close remaining <50% gaps file-by-file", "wid": "w11"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w11"], "notes": "Produce before/after table for each listed file and capture command output\nproving all targets are >=50%.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Finalize coverage evidence and prepare completion report", "wid": "w12"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/results/display.py\n- benchbox/core/results/timing.py\n- benchbox/core/results/anonymization.py\n- benchbox/core/analysis/insights.py\n- benchbox/core/analysis/ranking.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover core results and analysis modules to >=50%", "wid": "w2"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/tpcdi/worker_pool_examples.py\n- benchbox/core/tpcdi/source_generators.py\n- benchbox/core/tpcdi/etl/transformations.py\n- benchbox/core/tpcdi/etl/pipeline.py\n- benchbox/core/tpcdi/etl/batch.py\n- benchbox/core/tpcdi/etl/scd_processor.py\nProgress snapshot (2026-02-10):\n- worker_pool_examples.py: 90.5% (complete for target)\n- source_generators.py: 84.0% (complete for target)\n- pipeline.py: 65.4% (complete for target)\n- transformations.py: 11.8% (remaining)\n- batch.py: 17.1% (remaining)\n- scd_processor.py: 24.2% (remaining)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover TPC-DI ETL and orchestration modules to >=50%", "wid": "w3"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w3"], "notes": "Files:\n- benchbox/core/tpcdi/generator/dimensions.py\n- benchbox/core/tpcdi/generator/facts.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover TPC-DI generator dimension/fact modules to >=50%", "wid": "w4"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/tpcds/reporting.py\n- benchbox/core/tpcds/official_benchmark.py\n- benchbox/core/tpcds/power_test.py\n- benchbox/core/tpcds/c_tools.py\n- benchbox/core/tpcds/generator/runner.py\n- benchbox/core/tpch/reporting.py\n- benchbox/core/tpch/maintenance_test.py\n- benchbox/core/tpch/throughput_test.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover TPC-DS and TPC-H reporting and execution modules to >=50%", "wid": "w5"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/amplab/benchmark.py\n- benchbox/core/amplab/generator.py\n- benchbox/core/nyctaxi/benchmark.py\n- benchbox/core/nyctaxi/downloader.py\n- benchbox/core/ssb/benchmark.py\n- benchbox/core/ssb/generator.py\n- benchbox/core/clickbench/benchmark.py\n- benchbox/core/h2odb/benchmark.py\n- benchbox/core/write_primitives/benchmark.py\n- benchbox/core/transaction_primitives/benchmark.py\n- benchbox/core/metadata_primitives/benchmark.py\n- benchbox/core/metadata_primitives/generator.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover benchmark orchestrator modules for remaining suites to >=50%", "wid": "w6"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/tpcds/dataframe_queries/queries.py\n- benchbox/core/datavault/dataframe_queries/queries.py\n- benchbox/core/ssb/dataframe_queries/queries.py\n- benchbox/core/amplab/dataframe_queries/queries.py\n- benchbox/core/nyctaxi/dataframe_queries/queries.py\n- benchbox/core/clickbench/dataframe_queries/queries.py\n- benchbox/core/coffeeshop/dataframe_queries/queries.py\n- benchbox/core/read_primitives/dataframe_queries.py\n- benchbox/core/ai_primitives/dataframe_operations.py\n- benchbox/core/metadata_primitives/dataframe_operations.py\nCovered via PolarsDataFrameAdapter-backed context for AMPLab (8 expression +\n2 pandas), SSB (13), CoffeeShop (11, 3 xfail due to production bugs).\nAI primitives: enums, capabilities, results, manager, pure functions\n(classify, extract_entities, cosine/euclidean/top_k).\nMetadata primitives: enums, capabilities, presets, results, manager,\nschema introspection execution.\nLarge modules (TPC-DS, read_primitives, NYC Taxi, ClickBench, Data Vault)\ndeferred to w11 gap-closing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover dataframe query modules with representative assertions to >=50%", "wid": "w7"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/core/concurrency/pool_tester.py (already has 61 tests, >=50%)\n- benchbox/core/concurrency/executor.py (42 tests added)\n- benchbox/core/joinorder/generator.py (42 tests added)\n- benchbox/utils/cloud_storage.py (already has comprehensive test suite, >=50%)\npool_tester.py and cloud_storage.py already had sufficient coverage.\nexecutor.py: ConcurrentLoadExecutor run, failures, connection errors,\nqueue metrics, resource monitoring. joinorder/generator.py: init variants,\nrow counts, all _generate_* methods, dimension/relationship tables, CSV/manifest.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover concurrency, joinorder, and utility modules to >=50%", "wid": "w8"}, {"evidence": null, "item_id": "coverage-per-module-50-remaining-wave-2026-02-10", "needs": ["w1"], "notes": "Files:\n- benchbox/cli/exceptions.py (72% -> 88%)\n- benchbox/cli/commands/run.py (47% -> 50%)\n52 tests: PlatformOptionParamType, _build_execution_context,\n_render_post_run_charts, _describe_platform_options, setup_verbose_logging,\nErrorHandler branches, ValidationRules, ErrorContext, BenchboxCLIError.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cover CLI exception/run command paths with parser and error tests to >=50%", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add integration tests that require cloud credentials or external services", "instead": "use deterministic mocks/fakes", "why": "CI must run offline"}, {"dont": "import heavy cloud SDKs at module level in test files", "instead": "use unittest.mock.patch or lazy imports", "why": "they add 2-5s import overhead and may not be installed"}, {"dont": "raise coverage by weakening assertions", "instead": "assert observable behavior and error-path contracts", "why": "it masks regressions"}, {"dont": "silently defer additional modules beyond the approved list", "instead": "add any new deferment as explicit TODO update", "why": "scope creep undermines the gate"}, {"dont": "test auto-generated SQL strings character-by-character", "instead": "test query structure, table references, and parameters instead", "why": "SQL formatting changes would cause false failures"}], "approach": "Follow existing test patterns in tests/unit/{cli,core,tpcdi,platforms} and\nprefer fast unit tests with mocks over integration tests. Implement coverage\nlift in waves ordered by fastest impact (near-50% quick wins first, then\nmedium-effort modules, then large modules requiring 5-10 tests each, then\nCLI, then TPC-DI). Recompute and checkpoint coverage after each wave using\ncoverage.json plus the per-file checker to keep progress measurable.\n\nNaming convention: new files are named test_{module}_coverage.py when an\nexisting test file covers the same source module. All tests marked\n@pytest.mark.fast and @pytest.mark.unit, pure mocks only.\n\nWithin each phase, batches are independent and can be parallelized via\nTask agents. Each batch creates all its test files, then runs verification\nbefore proceeding to the next phase.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-08T00:00:00Z", "deferrals": [], "deps": [], "description": "Close remaining test coverage gaps by enforcing a per-module >=50% floor for\nactive benchbox/ modules while temporarily deferring a small set of\nlegacy/demo/heavy-parser files.\n\nThe HTML coverage report (2026-02-08, 59.59% overall) identifies ~67 modules\nbelow 50%. This plan provides a concrete per-file implementation sequence\nwith specific test file names, estimated test counts, and batch groupings\nfor parallel execution.\n\nImplementation sequence (8 phases):\n 1. Add active-scope per-file coverage checker and Makefile workflow.\n 2. Quick wins - modules at 43-49% needing 2-4 tests each (~21 files).\n 3. Medium effort - tuning, credentials, TPC runners, platform adapters (~20 files).\n 4. High effort - large modules requiring 5-10 tests each (~11 files).\n 5. CLI coverage - zero-coverage through near-50% CLI modules (~20 files).\n 6. Active TPC-DI modules under 50% (not deferred).\n 7. Final sweep - fill any remaining gaps found after verification.\n 8. Enforce the gate in local CI workflow and keep deferred-module tracking.\n\nTotal: ~87 new test files, ~405 test functions.\n\nDeferred scope for this campaign:\n - benchbox/core/tpcdi/tools/file_parsers.py\n - benchbox/core/tpcdi/tools/data_cleaners.py\n - benchbox/core/tpcdi/source_generators.py\n - benchbox/core/tpcdi/etl/transformations.py\n - benchbox/core/tpcdi/etl/batch.py\n - benchbox/core/tpchavoc/variants.py\n - benchbox/core/tpc_validation.py\n - benchbox/core/tpc_patterns.py\n - benchbox/core/tpc_metrics.py\n - benchbox/core/tpcdi/streaming.py\n - benchbox/core/tpcdi/worker_pool_examples.py\n - benchbox/core/tpcds/reporting.py\n - benchbox/core/tpch/reporting.py\n - benchbox/platforms/dataframe/ducklake_maintenance.py\n - benchbox/core/tpcds/dataframe_queries/rollup_helper.py\n - benchbox/core/tpcds/dataframe_queries/channel_helper.py\n - benchbox/core/visualization/chart_generator.py\n - benchbox/mcp/__main__.py\n - benchbox/core/results/display.py\n - benchbox/core/metadata_primitives/generator.py\n - benchbox/core/tpcds/dataframe_queries/queries.py\n - benchbox/core/tpch/maintenance_test.py\n - benchbox/core/tpcdi/etl/pipeline.py\n - benchbox/core/query_plans/parsers/sqlite.py\n - benchbox/core/analysis/insights.py\n - benchbox/core/runner/conversion.py\n - benchbox/core/datavault/dataframe_queries/queries.py\n - benchbox/cli/commands/results.py\n - benchbox/core/amplab/benchmark.py\n - benchbox/core/analysis/ranking.py\n - benchbox/core/ssb/benchmark.py\n - benchbox/core/nyctaxi/benchmark.py\n - benchbox/core/read_primitives/dataframe_queries.py\n - benchbox/core/ssb/dataframe_queries/queries.py\n - benchbox/core/joinorder/generator.py\n - benchbox/core/nyctaxi/downloader.py\n - benchbox/core/nyctaxi/dataframe_queries/queries.py\n - benchbox/core/amplab/dataframe_queries/queries.py\n - benchbox/core/clickbench/benchmark.py\n - benchbox/core/tpcdi/etl/scd_processor.py\n - benchbox/core/transaction_primitives/schema.py\n - benchbox/core/transaction_primitives/benchmark.py\n - benchbox/core/h2odb/benchmark.py\n - benchbox/core/tpcdi/generator/dimensions.py\n - benchbox/core/results/timing.py\n - benchbox/core/concurrency/pool_tester.py\n - benchbox/core/amplab/generator.py\n - benchbox/core/metadata_primitives/schema.py\n - benchbox/core/tpcdi/generator/facts.py\n - benchbox/core/concurrency/executor.py\n - benchbox/core/tpcds/official_benchmark.py\n - benchbox/core/clickbench/dataframe_queries/queries.py\n - benchbox/core/coffeeshop/dataframe_queries/queries.py\n - benchbox/cli/exceptions.py\n - benchbox/core/clickbench/generator.py\n - benchbox/utils/cloud_storage.py\n - benchbox/cli/commands/plot.py\n - benchbox/core/joinorder/queries.py\n - benchbox/platforms/athena.py\n - benchbox/core/metadata_primitives/benchmark.py\n - benchbox/platforms/influxdb/setup.py\n - benchbox/platforms/fabric_warehouse.py\n - benchbox/platforms/influxdb/client.py\n - benchbox/core/write_primitives/benchmark.py\n - benchbox/core/ssb/generator.py\n - benchbox/core/tpch/throughput_test.py\n - benchbox/core/tpcds/generator/runner.py\n - benchbox/core/ai_primitives/dataframe_operations.py\n - benchbox/core/amplab/queries.py\n - benchbox/core/joinorder/schema.py\n - benchbox/core/metadata_primitives/dataframe_operations.py\n - benchbox/core/h2odb/generator.py\n - benchbox/platforms/dataframe/__init__.py\n - benchbox/core/tpcds/c_tools.py\n - benchbox/cli/commands/run.py\n - benchbox/platforms/redshift.py\n - benchbox/core/tpchavoc/queries.py\n - benchbox/core/results/anonymization.py\n - benchbox/platforms/clickhouse/workload.py\n - benchbox/platforms/bigquery.py\n - benchbox/core/tpcds/power_test.py\n - benchbox/platforms/databricks/adapter.py", "id": "coverage-per-module-50-temporary-split", "preserves": ["Current CLI command behavior and argument contracts remain unchanged", "Existing fast/unit test suite remains green with no live platform dependency", "Existing overall --cov-fail-under workflow remains available during migration", "TPC-DI SQL/DataFrame ETL parity behavior verified by existing parity tests"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/"}, {"kind": "only_modify", "path_glob": "tests/unit/"}], "state": "done", "title": "Raise benchbox/ per-module coverage to >=50% with temporary split", "verifications": [{"command": "uv run -- python scripts/check_per_file_coverage.py --min 50 --coverage-json coverage.json", "description": "Active-scope per-file checker reports and fails correctly", "expected": "No active-scope module below 50%, exit code 0", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 1}, {"command": "make ci-test", "description": "Fast coverage workflow remains green at project floor", "expected": "pytest fast suite passes and coverage gate remains >=50%", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit -m fast --tb=short", "description": "Targeted coverage wave tests pass", "expected": "all selected fast unit tests pass, 0 failures", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests -p pytest_cov --cov=benchbox --cov-report=term-missing:skip-covered -n auto -m 'not slow and not stress and not live_integration' --tb=short -q --benchmark-disable", "description": "Per-phase verification", "expected": "no benchbox/ module below 50% in output", "item_id": "coverage-per-module-50-temporary-split", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add scripts/check_per_file_coverage.py for coverage.json-based active-scope checks", "wid": "w1"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_dataframe_init.py \u2192 platforms/dataframe/__init__.py (46% \u2192 50%, ~2 tests)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record before/after failing-module snapshots per wave", "wid": "w100"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm no active-scope module remains below 50%", "wid": "w101"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2B: Utils/core near-50% (5 files, ~15 tests)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/utils/test_verbosity_coverage.py \u2192 utils/verbosity.py (49% \u2192 50%, ~3 tests)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/visualization/test_visualization_exporters.py \u2192 core/visualization/exporters.py (44% \u2192 50%, ~4 tests)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/examples/test_dryrun_utils.py \u2192 examples/dryrun_utils.py (47% \u2192 50%, ~2 tests)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tsbs_devops/test_tsbs_benchmark_coverage.py \u2192 core/tsbs_devops/benchmark.py (43% \u2192 50%, ~4 tests)", "wid": "w15"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w11"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_clickhouse_cloud_coverage.py \u2192 platforms/clickhouse_cloud.py (46% \u2192 50%, ~2 tests)", "wid": "w16"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2C: Platform adapters near-50% (6 files, ~16 tests)", "wid": "w17"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_azure_synapse_coverage.py \u2192 platforms/azure_synapse.py (49% \u2192 50%, ~3 tests)", "wid": "w18"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/gcp/test_dataproc_serverless_coverage.py \u2192 platforms/gcp/dataproc_serverless_adapter.py (49% \u2192 50%, ~2 tests)", "wid": "w19"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fail when any active-scope benchbox/*.py file is below 50%", "wid": "w2"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_timescaledb_coverage.py \u2192 platforms/timescaledb.py (48% \u2192 50%, ~2 tests)", "wid": "w20"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/aws/test_athena_spark_coverage.py \u2192 platforms/aws/athena_spark_adapter.py (45% \u2192 50%, ~3 tests)", "wid": "w21"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/azure/test_synapse_spark_coverage.py \u2192 platforms/azure/synapse_spark_adapter.py (44% \u2192 50%, ~3 tests)", "wid": "w22"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w17"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/gcp/test_dataproc_coverage.py \u2192 platforms/gcp/dataproc_adapter.py (43% \u2192 50%, ~3 tests)", "wid": "w23"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2D: Core benchmark modules near-50% (5 files, ~18 tests)", "wid": "w24"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/test_tpcds_maintenance_coverage.py \u2192 core/tpcds/maintenance_test.py (48% \u2192 50%, ~3 tests)", "wid": "w25"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/generator/test_streaming_coverage.py \u2192 core/tpcds/generator/streaming.py (45% \u2192 50%, ~4 tests)", "wid": "w26"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_pyspark_maintenance_coverage.py \u2192 platforms/dataframe/pyspark_maintenance.py (44% \u2192 50%, ~3 tests)", "wid": "w27"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch_skew/test_tpch_skew_benchmark_coverage.py \u2192 core/tpch_skew/benchmark.py (39% \u2192 50%, ~4 tests)", "wid": "w28"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w24"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_cudf_coverage.py \u2192 platforms/cudf.py (42% \u2192 50%, ~4 tests)", "wid": "w29"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Report sorted failing files with percentage and statement count", "wid": "w3"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2E: MCP and misc (2 files, ~6 tests)", "wid": "w30"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w30"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/mcp/test_mcp_init_coverage.py \u2192 mcp/__init__.py (31% \u2192 50%, ~3 tests)", "wid": "w31"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w30"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/utils/format_converters/test_vortex_converter_coverage.py \u2192 utils/format_converters/vortex_converter.py (14% \u2192 50%, ~3 tests)", "wid": "w32"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3A: Tuning subsystem (5 files, ~32 tests)", "wid": "w33"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/test_tuning_validation.py \u2192 core/tuning/validation.py (13% \u2192 50%, ~10 tests)", "wid": "w34"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/test_tuning_metadata.py \u2192 core/tuning/metadata.py (13% \u2192 50%, ~10 tests)", "wid": "w35"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/generators/test_azure_synapse_ddl.py \u2192 core/tuning/generators/azure_synapse.py (24% \u2192 50%, ~4 tests)", "wid": "w36"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/generators/test_clickhouse_ddl.py \u2192 core/tuning/generators/clickhouse.py (22% \u2192 50%, ~4 tests)", "wid": "w37"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w33"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tuning/generators/test_firebolt_ddl.py \u2192 core/tuning/generators/firebolt.py (19% \u2192 50%, ~4 tests)", "wid": "w38"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3B: Credentials (3 files, ~21 tests)", "wid": "w39"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Support deferred list and configurable threshold via CLI flags", "wid": "w4"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_snowflake_coverage.py \u2192 platforms/credentials/snowflake.py (32% \u2192 50%, ~6 tests)", "wid": "w40"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_bigquery_coverage.py \u2192 platforms/credentials/bigquery.py (29% \u2192 50%, ~6 tests)", "wid": "w41"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w39"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_databricks_credentials_coverage.py \u2192 platforms/databricks/credentials.py (28% \u2192 50%, ~6 tests)", "wid": "w42"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3C: TPC benchmark runners (4 files, ~22 tests)", "wid": "w43"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/benchmark/test_tpcds_runner_coverage.py \u2192 core/tpcds/benchmark/runner.py (34% \u2192 50%, ~7 tests)", "wid": "w44"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/test_throughput_test_coverage.py \u2192 core/tpcds/throughput_test.py (32% \u2192 50%, ~5 tests)", "wid": "w45"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch/test_official_benchmark_coverage.py \u2192 core/tpch/official_benchmark.py (32% \u2192 50%, ~4 tests)", "wid": "w46"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w43"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/transaction_primitives/test_dataframe_ops_coverage.py \u2192 core/transaction_primitives/dataframe_operations.py (38% \u2192 50%, ~6 tests)", "wid": "w47"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3D: Platform adapters (5 files, ~24 tests)", "wid": "w48"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_motherduck_coverage.py \u2192 platforms/motherduck.py (32% \u2192 50%, ~5 tests)", "wid": "w49"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make coverage-per-file target and document usage", "wid": "w5"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_influxdb_adapter_coverage.py \u2192 platforms/influxdb/adapter.py (31% \u2192 50%, ~5 tests)", "wid": "w50"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/test_clickhouse_metadata_coverage.py \u2192 platforms/clickhouse/metadata.py (31% \u2192 50%, ~4 tests)", "wid": "w51"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/aws/test_emr_serverless_coverage.py \u2192 platforms/aws/emr_serverless_adapter.py (38% \u2192 50%, ~5 tests)", "wid": "w52"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w48"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_iceberg_maintenance_coverage.py \u2192 platforms/dataframe/iceberg_maintenance.py (38% \u2192 50%, ~5 tests)", "wid": "w53"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 3E: DataFrame adapters and remaining credentials (3 files, ~20 tests)", "wid": "w54"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w54"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_expression_family_coverage.py \u2192 platforms/dataframe/expression_family.py (35% \u2192 50%, ~6 tests)", "wid": "w55"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w54"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_modin_df_coverage.py \u2192 platforms/dataframe/modin_df.py (27% \u2192 50%, ~5 tests)", "wid": "w56"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w54"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/credentials/test_redshift_coverage.py \u2192 platforms/credentials/redshift.py (24% \u2192 50%, ~9 tests)", "wid": "w57"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 4A: Largest core modules (5 files, ~40 tests)", "wid": "w58"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_unified_frame_coverage.py \u2192 platforms/dataframe/unified_frame.py (35% \u2192 50%, ~10 tests)", "wid": "w59"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update pyproject coverage omit/deferred notes for temporary split", "wid": "w6"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch/test_dataframe_queries_coverage.py \u2192 core/tpch/dataframe_queries.py (12% \u2192 50%, ~10 tests)", "wid": "w60"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/transaction_primitives/test_generator_coverage.py \u2192 core/transaction_primitives/generator.py (11% \u2192 50%, ~8 tests)", "wid": "w61"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch/test_parameter_extractor_coverage.py \u2192 core/tpch/parameter_extractor.py (22% \u2192 50%, ~6 tests)", "wid": "w62"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w58"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpchavoc/test_tpchavoc_validation_coverage.py \u2192 core/tpchavoc/validation.py (14% \u2192 50%, ~6 tests)", "wid": "w63"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 4B: More large modules (6 files, ~35 tests)", "wid": "w64"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpch_skew/test_skew_generator_coverage.py \u2192 core/tpch_skew/generator.py (17% \u2192 50%, ~7 tests)", "wid": "w65"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/test_parameter_extractor_coverage.py \u2192 core/tpcds/parameter_extractor.py (15% \u2192 50%, ~4 tests)", "wid": "w66"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tsbs_devops/test_tsbs_df_queries_coverage.py \u2192 core/tsbs_devops/dataframe_queries/queries.py (19% \u2192 50%, ~7 tests)", "wid": "w67"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_cudf_df_coverage.py \u2192 platforms/dataframe/cudf_df.py (24% \u2192 50%, ~5 tests)", "wid": "w68"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/dataframe/test_dask_df_coverage.py \u2192 platforms/dataframe/dask_df.py (20% \u2192 50%, ~6 tests)", "wid": "w69"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 2A: Trivial small modules (3 files, ~8 tests)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w64"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/platforms/databricks/test_databricks_df_adapter_coverage.py \u2192 platforms/databricks/dataframe_adapter.py (18% \u2192 50%, ~6 tests)", "wid": "w70"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5A: Zero-coverage CLI modules (2 files, ~12 tests)", "wid": "w71"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w71"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_tuning_wizard.py \u2192 cli/tuning.py (0% \u2192 50%, ~6 tests)", "wid": "w72"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w71"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_onboarding_coverage.py \u2192 cli/onboarding.py (0% \u2192 50%, ~6 tests)", "wid": "w73"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5B: Low-coverage CLI commands (6 files, ~32 tests)", "wid": "w74"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_compare_plans_coverage.py \u2192 cli/commands/compare_plans.py (8% \u2192 50%, ~5 tests)", "wid": "w75"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_compare_dataframes_coverage.py \u2192 cli/commands/compare_dataframes.py (9% \u2192 50%, ~5 tests)", "wid": "w76"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_df_tuning_coverage.py \u2192 cli/commands/df_tuning.py (11% \u2192 50%, ~5 tests)", "wid": "w77"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_export_coverage.py \u2192 cli/commands/export.py (13% \u2192 50%, ~5 tests)", "wid": "w78"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_preferences_coverage.py \u2192 cli/preferences.py (15% \u2192 50%, ~7 tests)", "wid": "w79"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/visualization/test_visualization_exceptions.py \u2192 core/visualization/exceptions.py (33% \u2192 50%, ~3 tests)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w74"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_plan_history_coverage.py \u2192 cli/commands/plan_history.py (19% \u2192 50%, ~5 tests)", "wid": "w80"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5C: Medium-coverage CLI (7 files, ~33 tests)", "wid": "w81"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_datagen_coverage.py \u2192 cli/commands/datagen.py (22% \u2192 50%, ~5 tests)", "wid": "w82"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_show_plan_coverage.py \u2192 cli/commands/show_plan.py (24% \u2192 50%, ~4 tests)", "wid": "w83"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_compare_coverage.py \u2192 cli/commands/compare.py (28% \u2192 50%, ~5 tests)", "wid": "w84"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_shell_coverage.py \u2192 cli/commands/shell.py (30% \u2192 50%, ~5 tests)", "wid": "w85"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_run_official_coverage.py \u2192 cli/commands/run_official.py (33% \u2192 50%, ~5 tests)", "wid": "w86"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_system_coverage.py \u2192 cli/system.py (36% \u2192 50%, ~4 tests)", "wid": "w87"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w81"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_composite_params_coverage.py \u2192 cli/composite_params.py (39% \u2192 50%, ~5 tests)", "wid": "w88"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch 5D: Near-50% CLI modules (5 files, ~21 tests)", "wid": "w89"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/core/tpcds/schema/test_tpcds_schema_registry.py \u2192 core/tpcds/schema/registry.py (27% \u2192 50%, ~3 tests)", "wid": "w9"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_config_coverage.py \u2192 cli/config.py (41% \u2192 50%, ~5 tests)", "wid": "w90"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/test_output_coverage.py \u2192 cli/output.py (43% \u2192 50%, ~5 tests)", "wid": "w91"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_tuning_group_coverage.py \u2192 cli/commands/tuning_group.py (40% \u2192 50%, ~4 tests)", "wid": "w92"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/commands/test_tuning_cmd_coverage.py \u2192 cli/commands/tuning.py (46% \u2192 50%, ~2 tests)", "wid": "w93"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": ["w89"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "tests/unit/cli/presentation/test_system_presentation_coverage.py \u2192 cli/presentation/system.py (6% \u2192 50%, ~5 tests)", "wid": "w94"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Raise non-deferred TPC-DI modules under 50%: benchmark/generator/queries/loader/schema/validation/metrics/config/etl.py/etl/validation.py", "wid": "w95"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep ETL tests unit-heavy with mocked pipeline and deterministic fixtures", "wid": "w96"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-run full coverage report and identify any modules still below 50%", "wid": "w97"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add targeted tests for remaining gaps discovered after phases 2-6", "wid": "w98"}, {"evidence": null, "item_id": "coverage-per-module-50-temporary-split", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enable failing mode for coverage-per-file in local CI/test workflow", "wid": "w99"}], "worktree": "main"} +{"anti_patterns": [{"dont": "import heavy cloud SDKs at module level in test files", "instead": "use unittest.mock.patch or lazy imports", "why": "they add 2-5s import overhead and may not be installed"}, {"dont": "require cloud credentials for unit tests", "instead": "mock all SDK clients (boto3, google-cloud, snowflake-connector)", "why": "CI has no cloud access and tests must be runnable offline"}, {"dont": "test platform-specific SQL dialect generation", "instead": "test adapter configuration, connection logic, and error handling instead", "why": "dialect translation is covered by separate dialect tests"}], "approach": "Follow existing patterns in tests/unit/platforms/. For cloud adapters,\nmock all SDK clients (boto3, google-cloud, snowflake-connector). For\ncredential modules, test env var parsing, config file loading, and\nvalidation logic. For DataFrame adapters, use small in-memory DataFrames.\nunified_frame.py is the largest file - focus on the public API methods\nand expression translation.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Several platform adapter modules are below 50% test coverage. These include\nboth DataFrame platform abstractions and cloud/remote platform adapters.\n\nCurrent coverage breakdown:\n - platforms/dataframe/unified_frame.py: 31.0% (930 missing)\n - platforms/dataframe/benchmark_mixin.py: 38.5% (232 missing)\n - platforms/dataframe/expression_family.py: 45.7% (215 missing)\n - platforms/dataframe/dask_df.py: 26.1% (139 missing)\n - platforms/dataframe/ducklake_maintenance.py: 0.0% (126 missing)\n - platforms/credentials/redshift.py: 25.2% (333 missing)\n - platforms/credentials/bigquery.py: 31.1% (171 missing)\n - platforms/credentials/snowflake.py: 34.6% (123 missing)\n - platforms/databricks/credentials.py: 31.7% (127 missing)\n - platforms/databricks/dataframe_adapter.py: 23.6% (120 missing)\n - platforms/athena.py: 39.7% (356 missing)\n - platforms/fabric_warehouse.py: 43.9% (252 missing)\n - platforms/influxdb/adapter.py: 26.3% (101 missing)\n - platforms/influxdb/client.py: 44.0% (94 missing)\n - platforms/cudf.py: 48.1% (175 missing)\n - platforms/motherduck.py: 35.9% (84 missing)\n - platforms/clickhouse_cloud.py: 49.3% (35 missing)\n\nCloud platform tests should mock all external API calls. DataFrame platform\ntests can use in-memory data.", "id": "coverage-platforms", "preserves": ["All existing platform tests continue to pass", "No changes to production code"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Improve test coverage for platform adapter modules to \u226550%", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "All platform tests pass", "expected": "all tests pass, 0 failures", "item_id": "coverage-platforms", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/unified_frame.py (31.0% \u2192 50%)", "wid": "w1"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/athena.py (39.7% \u2192 50%)", "wid": "w10"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/fabric_warehouse.py (43.9% \u2192 50%)", "wid": "w11"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/databricks/dataframe_adapter.py (23.6% \u2192 50%)", "wid": "w12"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/influxdb/ (26-44% \u2192 50%)", "wid": "w13"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/cudf.py (48.1% \u2192 50%)", "wid": "w14"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/motherduck.py (35.9% \u2192 50%)", "wid": "w15"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/clickhouse_cloud.py (49.3% \u2192 50%)", "wid": "w16"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/benchmark_mixin.py (38.5% \u2192 50%)", "wid": "w2"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/expression_family.py (45.7% \u2192 50%)", "wid": "w3"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/dask_df.py (26.1% \u2192 50%)", "wid": "w4"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/dataframe/ducklake_maintenance.py (0% \u2192 50%)", "wid": "w5"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/credentials/redshift.py (25.2% \u2192 50%)", "wid": "w6"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/credentials/bigquery.py (31.1% \u2192 50%)", "wid": "w7"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/credentials/snowflake.py (34.6% \u2192 50%)", "wid": "w8"}, {"evidence": null, "item_id": "coverage-platforms", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests for platforms/databricks/credentials.py (31.7% \u2192 50%)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add tests just to hit line coverage", "instead": "every test should fail if the code under test is broken", "why": "a test that can never fail is worse than no test"}, {"dont": "mock internal methods to test dispatchers", "instead": "use real DuckDB/Polars/DataFusion wherever possible", "why": "you're testing mock wiring not behavior"}, {"dont": "omit files from coverage just because they are hard to test", "instead": "only omit genuinely untestable code (external infrastructure, experimental, bulk mechanical translations)", "why": "that hides real gaps"}, {"dont": "put 'coverage' in test filenames", "instead": "name after the behavior being tested", "why": "test names should describe what they verify"}, {"dont": "test cloud SQL generation by asserting mock.called", "instead": "assert the generated SQL string contains expected clauses", "why": "that doesn't verify SQL correctness"}, {"dont": "write import-only tests", "instead": "every test must call a function and assert on its output", "why": "importing a module is not testing it"}], "approach": null, "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-26T00:00:00Z", "completed_pr": null, "created_at": "2026-03-25T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 131, "reason": "Requires full CDC/SCD pipeline infrastructure with staged incremental loads. 2,226 missing statements but extremely high effort. Omitted from scope and deferred until TPC-DI ETL testing infrastructure is built.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Gated on TPC-DI ETL testing infrastructure being built first; no evidence that infra exists yet, premature to promote.", "summary": "TPC-DI ETL deep coverage (transformations.py, batch.py, scd_processor.py)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 132, "reason": "Require real cloud Spark sessions or complex mocking. Better covered by platform-live-integration-tests TODO.", "resolution": "promoted", "resolved_item": "cloud-spark-adapters-emr-serverless-dataproc-serverless-syna", "resolved_reason": null, "summary": "Cloud Spark adapters (EMR Serverless, Dataproc Serverless, Synapse Spark)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 133, "reason": "Require external engines (Spark, LakeSail server). Config/validation logic was covered in dataframe-test-coverage w13; remaining gaps are engine-specific.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "PySpark and LakeSail DataFrame adapters"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "coverage-to-80-percent", "id": 134, "reason": "Experimental code changes frequently. Testing to 80% is wasted effort if modules are restructured. Cover after modules graduate from experimental.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly deferred until GPU-benchmark/NL2SQL modules graduate from experimental; no evidence that has happened.", "summary": "Experimental modules (GPU benchmark, NL2SQL evaluator)"}], "deps": ["dataframe-test-coverage", "non-dataframe-test-coverage"], "description": "Coverage is at 71.4% (81,722 / 109,707 stmts). Target is 80% (87,766 covered).\nGap: 6,044 more lines to cover.\n\nTwo-pronged strategy:\n1. SCOPE REFINEMENT: Omit 3,526 stmts of untestable bulk (TPC-DI ETL complex\n transformations that require full CDC infrastructure, experimental concurrency\n code, ClickBench DF queries). This immediately adjusts the denominator and\n boosts measured coverage to ~76.3%. Gap after omit: 3,928 lines.\n2. TARGETED TESTS: Cover ~4,130 additional lines (5% buffer over gap) from\n high-value locally-testable files across 5 categories:\n - Core/CLI/DF framework: ~1,600 lines (base adapter, unified_frame, CLI\n commands, DF benchmark_suite, runner, base.py, utils)\n - Cloud SQL generation (top 4): ~1,000 lines (Redshift, Databricks, BigQuery,\n Snowflake - config validation + SQL string assertions)\n - Cloud SQL generation (tier 2): ~500 lines (Athena, Synapse, Firebolt,\n ClickHouse, PostgreSQL, Trino)\n - DF query implementations: ~280 lines (read_primitives DF queries)\n - Remaining high-miss modules: ~350 lines (metadata_primitives, DataFusion\n adapter, TPC-DS runner gaps)\n\nAnti-coverage-theater principles:\n- Every test must call a function and assert on specific output values\n- Use real engines (DuckDB, Polars, DataFusion) not mocks where possible\n- Test behavior not implementation - if the function changes how it works but\n produces the same result, the test should still pass\n- No import-only tests, no `assert result is not None`, no `assert True`\n- Cloud platform tests assert on generated SQL strings, not mock.called\n- Mark genuinely untestable code for exclusion rather than writing useless tests", "id": "coverage-to-80-percent", "preserves": ["All existing tests must continue to pass - new tests are additive", "Cloud adapter tests must not require real credentials", "Coverage config in pyproject.toml remains source of truth for local dev", "No `assert True`, `assert result is not None`, or `hasattr`-only assertions", "Test suite speed: `make test-fast` must complete in < 90 seconds"], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Reach 80% test coverage through scope refinement and targeted behavioral tests", "verifications": [], "work": [{"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Omit untestable bulk from coverage scope", "wid": "w1"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test BigQuery SQL generation and config validation", "wid": "w10"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Snowflake SQL generation and config validation", "wid": "w11"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test remaining cloud platforms and adapters", "wid": "w12"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test base.py BaseBenchmark abstract methods", "wid": "w13"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test remaining utils and supporting modules", "wid": "w14"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test read_primitives remaining DF query implementations", "wid": "w15"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFusion adapter expression and write paths", "wid": "w16"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w1", "w10", "w11", "w12", "w13", "w14", "w15", "w16", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align CI coverage config and ratchet fail_under to 80", "wid": "w17"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test base adapter concrete methods to 85%", "wid": "w2"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py string, date, and subquery operations", "wid": "w3"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CLI run command option parsing and validation", "wid": "w4"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test CLI compare and benchmarks commands", "wid": "w5"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFrame benchmark_suite orchestration", "wid": "w6"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test core runner orchestration and config", "wid": "w7"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Redshift SQL generation and config validation", "wid": "w8"}, {"evidence": null, "item_id": "coverage-to-80-percent", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test Databricks SQL generation and config validation", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Complete user and developer documentation for DataFrame support.", "id": "create-comprehensive-dataframe-documentation", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Create Comprehensive DataFrame Documentation", "verifications": [], "work": [], "worktree": "dataframe-docs"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (COMPLETED)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Successfully created comprehensive release infrastructure including changelog, CI/CD pipelines, and corrected repository URLs throughout the project.", "id": "create-missing-release-infrastructure", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "\u2705 Create Missing Release Infrastructure", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Create 2 `tuning.yaml` variants for each benchmark example in `examples/` - a fully tuned and no tunings variant named after the system and benchmark\"\n- \"**Tasks**:\"\n- \"**Files Affected**: 31 new files in `examples/` directory: 30 tuning configuration files + 1 comprehensive documentation file\"\n- \"**Implementation Results**:\"\n- \"**Impact**: \u2705 Successfully provides users with ready-to-use tuning configurations for performance testing, enables easy comparison between optimized and baseline configurations, demonstrates full tuning system capabilities with platform-specific optimizations. Users can now easily compare performance impact of different optimization strategies across all major benchmarks.\"", "id": "create-tuning-configuration-variants-for-benchmark-examples-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Create Tuning Configuration Variants for Benchmark Examples \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Data Quality and Compatibility Issues", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Completed (commit 98f4b96)", "id": "create-unified-multi-platform-benchmark-runner-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Create Unified Multi-Platform Benchmark Runner - COMPLETED \u2705", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Performance and Efficiency", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-13T00:00:00Z", "deferrals": [], "deps": ["modin-cudf-dask-production-ready"], "description": "Create comprehensive benchmark suite comparing DataFrame platform performance.\nThis was deferred from the dataframe-performance Phase 9.3 work pending\ncompletion of Modin, cuDF, and Dask adapters.\n\nCurrent state:\n- Basic profiling exists (execute_query_profiled, MemoryTracker)\n- Cross-platform example exists: examples/dataframe/cross_platform_comparison.py\n- Missing: Systematic benchmark suite with standardized comparisons\n\nGoal: Generate reproducible performance comparisons across all DataFrame\nplatforms for blog posts and documentation.", "id": "cross-platform-benchmark-suite", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "DataFrame Cross-Platform Benchmark Suite", "verifications": [], "work": [], "worktree": "dataframe-performance"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "auto-merge the prune PR", "why": "soundness path, Code Owner review required."}, {"INSTEAD": "(not specified)", "dont": "reimplement the pruning logic; invoke the #935 writer/target.", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Developer Tooling", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-17T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up to cross-surface-baseline-update-flag (#935), which shipped the manual\n`--update-baseline` maintenance affordance (a YAML baseline in\ncross_surface_baseline.yaml, pruned by known_divergences_baseline.py via the\n`make cross-surface-update-baseline` target). That item deferred the automation:\ndetection and pruning are still operator-initiated, so a resolved (no-longer-\nreproducing) known-divergence entry can linger between manual sweeps.\n\nAdd a scheduled job that runs the enforced cross-surface gates, detects resolved\nentries (the gate already reports \"previously-known divergences now equivalent\"),\nruns the existing `--update-baseline` writer, and opens a PR with the pruned\nbaseline. Removal stays explicit and reviewable (a PR), never a side effect of a\nblocking gate run.\n", "id": "cross-surface-baseline-stale-entry-autodetect", "preserves": ["A still-reproducing divergence is never removed; only resolved entries are dropped; idempotent.", "The blocking gate run never prunes; pruning happens only in the scheduled PR path (as #935 established)."], "prior_art": [{"concept": "{'ref': '.github/workflows/release-canary.yml', 'note': 'Existing scheduled-workflow + RULESET_DRIFT_TOKEN pattern to model the cadence and bot-PR mechanics on.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py (run_gate resolved-key reporting)', 'note': 'The gate already computes resolved keys; the scheduler reads them rather than re-deriving.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/equivalence/known_divergences_baseline.py + Makefile cross-surface-update-baseline', 'note': 'The load/prune/dump writer and Make target shipped by #935; the scheduled job invokes these, does not reimplement pruning.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}, {"kind": "only_modify", "path_glob": "benchbox/core/equivalence/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "planning", "title": "Auto-detect and PR resolved cross-surface known-divergence baseline entries on a schedule", "verifications": [{"command": "seed a resolved entry, run the detection step -> it identifies exactly that entry and the writer's diff drops only it", "description": "Dry-run detects a seeded resolved entry and produces the expected prune diff", "expected": "resolved entry surfaced and pruned; no live entry touched; no-op when none resolved", "item_id": "cross-surface-baseline-stale-entry-autodetect", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "cross-surface-baseline-stale-entry-autodetect", "needs": [], "notes": "A scheduled workflow (or an extension of an existing canary) that runs each\nenforced gate and captures the resolved-entry report. No-op when nothing\nresolved.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Scheduled job runs the enforced gates and collects resolved baseline entries per gate", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-baseline-stale-entry-autodetect", "needs": ["w1"], "notes": "Invoke `make cross-surface-update-baseline BENCHMARK=` for each affected\ngate, then open a bot-authored, labeled PR with the pruned cross_surface_\nbaseline.yaml. Idempotent: no PR when nothing changed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "When resolved entries exist, run --update-baseline and open a PR with the pruned baseline", "wid": "w2"}, {"evidence": null, "item_id": "cross-surface-baseline-stale-entry-autodetect", "needs": ["w2"], "notes": "cross_surface.py / the baseline are CODEOWNERS soundness paths, so the prune PR\nmust leave auto-merge OFF and route to Code Owner review (reuse\nauto_merge_soundness_paths.py).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Guardrails: never auto-merge the prune PR; it touches a soundness path", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "auto-prune the baseline as a side effect of the normal gate run", "why": "keep removal explicit and operator-driven."}, {"INSTEAD": "(not specified)", "dont": "reformat or reorder surviving baseline entries", "why": "minimal, surgical rewrite only."}], "approach": null, "blocked_reason": null, "category": "Developer Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-05T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-baseline-update-flag", "id": 365, "reason": "Out of scope; this item only provides the manual affordance. Automation is a separate hygiene item.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Exact match: open item is literally 'auto-detect and PR resolved cross-surface baseline entries on a schedule'. (tracked by cross-surface-baseline-stale-entry-autodetect)", "summary": "Auto-detect-and-PR stale baseline entries on a schedule."}], "deps": [], "description": "Follow-up to cross-surface-known-divergence-baseline-hygiene (PR #903, DONE). The\ndefault-FAIL half (a known divergence that no longer reproduces fails the gate\ninstead of silently passing) landed correctly. But the YAML also promised a\nmaintenance affordance \u2014 a flag/Make target to drop a resolved baseline entry \u2014\nand that was NOT built: removing a resolved known-divergence entry from the\nbaseline is fully manual. Makefile was in the item's only_modify list but is\nuntouched in the #903 diff. Manual baseline pruning is exactly the toil that lets\nbaselines rot, so this completes the promised affordance.\n\nCOMPLETED via #935 (merged 2026-07-05). Shipped as a YAML data-file baseline\n(benchbox/core/equivalence/cross_surface_baseline.yaml) loaded and pruned by\nbenchbox/core/equivalence/known_divergences_baseline.py, plus the\n`cross-surface-update-baseline` Make target and golden tests -- NOT the in-source\nAST rewrite the original prior_art assumed (that first cut was redesigned in review).\nThe writer lives under core/equivalence/, not benchbox/cli/ as scope_limit listed,\nto satisfy the utils it identifies exactly that entry and the writer's diff drops only it", "description": "Dry-run detects a seeded resolved entry and produces the expected prune diff", "expected": "resolved entry surfaced and pruned; no live entry touched; no-op when none resolved", "item_id": "cross-surface-baseline-stale-entry-autodetect", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "cross-surface-baseline-stale-entry-autodetect", "needs": [], "notes": "A scheduled workflow (or an extension of an existing canary) that runs each\nenforced gate and captures the resolved-entry report. No-op when nothing\nresolved.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Scheduled job runs the enforced gates and collects resolved baseline entries per gate", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-baseline-stale-entry-autodetect", "needs": ["w1"], "notes": "Invoke `make cross-surface-update-baseline BENCHMARK=` for each affected\ngate, then open a bot-authored, labeled PR with the pruned cross_surface_\nbaseline.yaml. Idempotent: no PR when nothing changed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "When resolved entries exist, run --update-baseline and open a PR with the pruned baseline", "wid": "w2"}, {"evidence": null, "item_id": "cross-surface-baseline-stale-entry-autodetect", "needs": ["w2"], "notes": "cross_surface.py / the baseline are CODEOWNERS soundness paths, so the prune PR\nmust leave auto-merge OFF and route to Code Owner review (reuse\nauto_merge_soundness_paths.py).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Guardrails: never auto-merge the prune PR; it touches a soundness path", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "auto-prune the baseline as a side effect of the normal gate run", "instead": "(not specified)", "why": "keep removal explicit and operator-driven."}, {"dont": "reformat or reorder surviving baseline entries", "instead": "(not specified)", "why": "minimal, surgical rewrite only."}], "approach": null, "blocked_reason": null, "category": "Developer Tooling", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-05T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-baseline-update-flag", "id": 365, "reason": "Out of scope; this item only provides the manual affordance. Automation is a separate hygiene item.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Exact match: open item is literally 'auto-detect and PR resolved cross-surface baseline entries on a schedule'. (tracked by cross-surface-baseline-stale-entry-autodetect)", "summary": "Auto-detect-and-PR stale baseline entries on a schedule."}], "deps": [], "description": "Follow-up to cross-surface-known-divergence-baseline-hygiene (PR #903, DONE). The\ndefault-FAIL half (a known divergence that no longer reproduces fails the gate\ninstead of silently passing) landed correctly. But the YAML also promised a\nmaintenance affordance \u2014 a flag/Make target to drop a resolved baseline entry \u2014\nand that was NOT built: removing a resolved known-divergence entry from the\nbaseline is fully manual. Makefile was in the item's only_modify list but is\nuntouched in the #903 diff. Manual baseline pruning is exactly the toil that lets\nbaselines rot, so this completes the promised affordance.\n\nCOMPLETED via #935 (merged 2026-07-05). Shipped as a YAML data-file baseline\n(benchbox/core/equivalence/cross_surface_baseline.yaml) loaded and pruned by\nbenchbox/core/equivalence/known_divergences_baseline.py, plus the\n`cross-surface-update-baseline` Make target and golden tests -- NOT the in-source\nAST rewrite the original prior_art assumed (that first cut was redesigned in review).\nThe writer lives under core/equivalence/, not benchbox/cli/ as scope_limit listed,\nto satisfy the utilsDataFrame validator so it is tracked rather than\nforgotten.\n\n`ResultValidator.validate_results_exact(order_aware=True, tie_aware=True)`\n(benchbox/core/tpchavoc/validation.py, `_validate_order_aware`) partitions the\nreturned rows into consecutive same-order-key groups and, for the FINAL group\nunder a trailing `LIMIT`, tolerates a membership difference as an\nequally-valid tie pick across the LIMIT cutoff. #878 tightened that acceptance\nto require a genuine tie of `len(orig_rows) >= 2` rows sharing the final key,\nso a deterministic single-row final-group difference is CAUGHT instead of\nmasked.\n\nCodex review on #878 (discussion r3462372977) correctly observed the residual\nedge: when a `LIMIT` cuts mid-tie, a legitimate boundary tie can leave exactly\nONE of the tied rows visible (keys `10, 5a, 5b` with `LIMIT 2` -> `[10, 5]`).\nTwo surfaces picking different `5` rows is valid SQL, but the `>= 2` guard now\nflags it as a mismatch.\n\nThe root cause is fundamental: from the TRUNCATED result alone, a one-visible-\nrow boundary tie and a genuine unique-final-key value change (e.g.\n`(2,\"good\")` vs `(2,\"bad\")`) are structurally identical - same final order\nkey, differing non-key column, a single row in the group. The comparator\ncannot distinguish them without signal from outside the LIMIT.\n\nAccepted tradeoff (decided on #878): land the `>= 2` guard as-is. Its failure\nmode is a LOUD false positive (a gate fails when it should not), which is\nrecoverable and safer than the prior behavior (#872 accepted ALL final-group\ndifferences, silently MASKING real single-row value bugs). All five enforced\ncross-surface gates currently pass (0 unclassified divergent), so the false\npositive is latent and data-dependent, not an active breakage.", "id": "cross-surface-comparator-boundary-tie-soundness", "preserves": [], "prior_art": [{"concept": "{'ref': 'PR #872', 'note': 'Introduced the order-aware/tie-aware comparator; original final-group accept was unconditional (masked single-row value bugs).'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'PR #878 + Codex discussion r3462372977', 'note': 'Tightened the final-group accept to len(orig_rows) >= 2; surfaced the one-visible-row boundary-tie false-positive edge documented here.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py:_validate_order_aware', 'note': 'The guard site (`if i == last_index and tie_aware and len(orig_rows) >= 2`).'}", "decision": "reuse", "path": "{'ref'"}], "priority": "low", "scope": [], "state": "done", "title": "Cross-surface comparator: make the final-tie-group guard boundary-aware (one-visible-row LIMIT ties)", "verifications": [], "work": [{"evidence": null, "item_id": "cross-surface-comparator-boundary-tie-soundness", "needs": [], "notes": "The comparator needs signal it does not have today to tell a truncated\nboundary tie from a genuine deterministic last row. Candidate approaches,\nto be evaluated for cost vs. value:\n - Have the cross-surface gate run each trailing-LIMIT query at `LIMIT N+1`\n (or unbounded) to learn whether the final order key is genuinely tied\n beyond the cutoff, and pass that fact into validate_results_exact so a\n single-visible-row final group can be accepted ONLY when the key is\n actually tied past the LIMIT.\n - Or thread the resolved `ORDER BY` / `LIMIT` structure plus a\n tie-multiplicity hint from the SQL surface into the comparator.\nKeep the change ADDITIVE: the TPC-Havoc strict path (no order_aware/\ntie_aware) must stay byte-identical, and the default must remain catching\nsingle-row value bugs. Add a regression test for the keys `10,5,5`\n`LIMIT 2` boundary-tie case (currently a false positive) alongside the\nexisting `(2,\"good\")` vs `(2,\"bad\")` true-positive case.\n\nRESOLVED: took the first approach (LIMIT N+1 probe). The cross-surface gate\nnow re-runs each trailing-LIMIT order-aware query one row wider\n(`cross_surface._bump_trailing_limit` / `_final_key_tied_beyond_limit`) and\nthreads `final_key_tied_beyond_limit` into\n`ResultValidator.validate_results_exact`. `_validate_order_aware` accepts a\nsingle-visible-row final tie group ONLY when that probe confirms the final\norder key recurs past the cutoff; otherwise the lone row is treated as a\ndeterministic last row and a value change there is still CAUGHT. The same\nchange also lands #878's `len(orig_rows) >= 2` tightening of the order-aware\nfinal-group accept (#878 was still open against develop), so a >= 2-row\nvisible tie remains an accepted boundary swap. ADDITIVE: callers that pass\nneither `order_aware` nor `tie_aware` (the TPC-Havoc strict path) are\nuntouched - verified 220/220 variant and 440/440 DataFrame equivalence, and\nall five enforced cross-surface gates green (0 unclassified divergent). The\nprobe runs once per query (memoized across backends). Regression tests:\ntests/integration/test_validator_order_aware.py (one-visible-row tie accepted\nwith the probe, caught without; the `(2,\"good\")` vs `(2,\"bad\")` unique-key\nbug still raises) and tests/unit/core/equivalence/test_cross_surface.py\n(`_bump_trailing_limit`, `_final_key_tied_beyond_limit`, and the end-to-end\ngate accept/catch cases).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Give the comparator a boundary-tie probe so a one-visible-row final tie is not a false positive", "wid": "w1"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "broaden scope into a general LIMIT-removal or untruncated-result comparison rewrite.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "disable tie-aware validation or weaken mismatch reporting to preserve current green gates.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "keep accepting a multi-row final tie group on row count alone; require truncation evidence.", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-comparator-complete-final-tie-group-soundness", "id": 135, "reason": "Out of scope - changing query shape would change what the cross-surface gate measures; the per-query probe is the targeted, additive fix (same rationale as #886).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Explicitly rejected: reworking the gate to drop LIMITs was ruled out of scope in favor of the per-query probe fix that shipped instead.", "summary": "Reworking the gate to compare untruncated result sets generally (dropping LIMITs)"}], "deps": [], "description": "Follow-up spun out of PR #886 (the boundary-tie probe, w1 of\ncross-surface-comparator-boundary-tie-soundness) and its self-review. It records\na known, accepted residual limitation in the order-aware tie-aware validator so\nit is tracked rather than forgotten.\n\n`ResultValidator._validate_order_aware` accepts a differing FINAL tie group under\na trailing `LIMIT` when:\n\n if i == last_index and tie_aware and (len(orig_rows) >= 2 or final_key_tied_beyond_limit):\n continue\n\nThe `len(orig_rows) >= 2` branch (landed by #878, preserved by #886) accepts a\nmembership difference in any final group that has >= 2 visible rows sharing the\norder key - WITHOUT evidence that the `LIMIT` actually truncated that tie. That\nis unsound for a COMPLETE, non-truncated final tie group: e.g.\n`... ORDER BY a DESC LIMIT 10` over a table with exactly 10 rows whose last two\nrows tie at `a`. There is no truncation, so both surfaces should return the SAME\ntwo tied rows; a membership difference like `[(5,'a'),(5,'b')]` vs\n`[(5,'a'),(5,'c')]` is a genuine value bug, yet the `>= 2` branch accepts it as a\n\"boundary swap\" and the gate goes silently green.\n\n#886's boundary-tie probe (`final_key_tied_beyond_limit`) is the exact signal\nneeded to distinguish a truncated tie from a complete one, but #886 scoped it to\nthe single-visible-row case only (its stated charter). For the >= 2 case the\nprobe is deliberately NOT consulted, and `cross_surface._final_key_tied_beyond_limit`\neven short-circuits (returns False, skipping the extra query) whenever the\nreference's final group already spans >= 2 rows - so today the >= 2 branch has no\ntruncation evidence at all.\n\nAccepted tradeoff (decided on #886): the failure mode is a MASKED bug (a false\nnegative) confined to a narrow shape - a complete, non-truncated, multi-row final\ntie group whose members differ between surfaces. All six enforced cross-surface\ngates currently pass with 0 unclassified divergent, so this is latent and\ndata-dependent, not an active breakage. It is documented in-code at the guard\nsite. Closing it is a deliberate, separately-scoped change (below), not a\ndrive-by, because it alters the #872/#878 established semantics and forces a\ntest + cost change.\n\nPriority raised Low -> Medium (2026-06-28, holistic-review remediation): for a\nbenchmark tool, false confidence in a correctness gate is the worst outcome, so a\ndocumented masked-value-bug path in the comparator warrants Medium and should be\nclosed before the next published cross-platform comparison claim. (The fix is\nalready scoped in w1; this is the only change to this item.)", "id": "cross-surface-comparator-complete-final-tie-group-soundness", "preserves": ["All enforced cross-surface gates remain green with 0 unclassified divergences after the tighter final-tie check.", "TPC-Havoc strict validation behavior is byte-identical when order_aware/tie_aware is disabled.", "The fix remains targeted to boundary-tie soundness; it does not rewrite queries or compare untruncated result sets.", "Truncated ORDER BY/LIMIT boundary swaps remain accepted only when the probe proves the final key extends past the LIMIT."], "prior_art": [{"concept": "{'ref': 'PR #872', 'note': 'Introduced the order-aware/tie-aware comparator; original final-group accept was unconditional.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'PR #878', 'note': 'Tightened the final-group accept to len(orig_rows) >= 2 (catches the single-row value bug).'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'PR #886', 'note': 'Added the boundary-tie probe (final_key_tied_beyond_limit) for the single-visible-row case and documented this residual >= 2 gap at the guard site.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py:_validate_order_aware', 'note': 'The guard site: `if i == last_index and tie_aware and (len(orig_rows) >= 2 or final_key_tied_beyond_limit)`.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/equivalence/cross_surface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/validation.py"}, {"kind": "only_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "tests/integration/test_validator_order_aware.py"}], "state": "done", "title": "Cross-surface comparator: close the >=2 final-tie-group gap for a complete (non-truncated) final tie", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_validator_order_aware.py -q -k 'final_tie or boundary'", "description": "Complete final tie with differing members now fails", "expected": "complete non-truncated >=2 final tie membership difference raises; genuinely truncated boundary swap still passes", "item_id": "cross-surface-comparator-complete-final-tie-group-soundness", "last_result": null, "last_run": null, "seq": 1}, {"command": "make ssb-cross-surface-equivalence-report && make clickbench-cross-surface-equivalence-report && make tpchavoc-equivalence-report", "description": "Cross-surface and TPC-Havoc behavior stays green", "expected": "cross-surface gates remain 0 unclassified divergent; tpchavoc strict result counts unchanged", "item_id": "cross-surface-comparator-complete-final-tie-group-soundness", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cross-surface-comparator-complete-final-tie-group-soundness", "needs": [], "notes": "Make the order-aware final-group acceptance require evidence that the LIMIT\nactually truncated the tie, for BOTH the single-row and the >= 2 case, so a\ncomplete (non-truncated) final tie group difference is CAUGHT. The probe\nalready answers exactly this (\"does the final order key recur past the\ncutoff?\"), so the targeted shape is to consult `final_key_tied_beyond_limit`\nfor the >= 2 branch too rather than accepting on row count alone.\n\nScope to evaluate (this is NOT a one-liner - it ripples through three places):\n 1. validation.py `_validate_order_aware`: change the guard so the >= 2\n branch also requires truncation evidence. Keep it ADDITIVE for callers\n that do not opt into the probe.\n 2. cross_surface.py `_final_key_tied_beyond_limit`: today it short-circuits\n (returns False, skips the extra LIMIT n+1 query) when the reference final\n group already has >= 2 rows - precisely the case w1 now needs the probe\n for. That optimization must be removed/relaxed for the >= 2 case, at the\n cost of one extra bounded probe query for those queries.\n 3. tests/integration/test_validator_order_aware.py\n `test_order_aware_truncated_limit_boundary_swap_is_not_flagged` encodes\n the CURRENT behavior (a 2-row final swap accepted with tie_aware but NO\n probe). It must be re-framed to pass the probe signal, and a NEW test\n added for the complete (table-exhausted, non-truncated) 2-row final tie\n that must now RAISE.\n\nHard constraints (same as #886): the TPC-Havoc strict path (no order_aware/\ntie_aware) stays byte-identical; all six enforced cross-surface gates stay\ngreen with 0 unclassified divergent; tpchavoc 220/220 and 440/440 unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate the >=2 final-tie-group accept on truncation evidence so a complete final tie no longer masks a value bug", "wid": "w1"}], "worktree": "main"} +{"anti_patterns": [{"dont": "broaden scope into a general LIMIT-removal or untruncated-result comparison rewrite.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "disable tie-aware validation or weaken mismatch reporting to preserve current green gates.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "keep accepting a multi-row final tie group on row count alone; require truncation evidence.", "instead": "(not specified)", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-comparator-complete-final-tie-group-soundness", "id": 135, "reason": "Out of scope - changing query shape would change what the cross-surface gate measures; the per-query probe is the targeted, additive fix (same rationale as #886).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Explicitly rejected: reworking the gate to drop LIMITs was ruled out of scope in favor of the per-query probe fix that shipped instead.", "summary": "Reworking the gate to compare untruncated result sets generally (dropping LIMITs)"}], "deps": [], "description": "Follow-up spun out of PR #886 (the boundary-tie probe, w1 of\ncross-surface-comparator-boundary-tie-soundness) and its self-review. It records\na known, accepted residual limitation in the order-aware tie-aware validator so\nit is tracked rather than forgotten.\n\n`ResultValidator._validate_order_aware` accepts a differing FINAL tie group under\na trailing `LIMIT` when:\n\n if i == last_index and tie_aware and (len(orig_rows) >= 2 or final_key_tied_beyond_limit):\n continue\n\nThe `len(orig_rows) >= 2` branch (landed by #878, preserved by #886) accepts a\nmembership difference in any final group that has >= 2 visible rows sharing the\norder key - WITHOUT evidence that the `LIMIT` actually truncated that tie. That\nis unsound for a COMPLETE, non-truncated final tie group: e.g.\n`... ORDER BY a DESC LIMIT 10` over a table with exactly 10 rows whose last two\nrows tie at `a`. There is no truncation, so both surfaces should return the SAME\ntwo tied rows; a membership difference like `[(5,'a'),(5,'b')]` vs\n`[(5,'a'),(5,'c')]` is a genuine value bug, yet the `>= 2` branch accepts it as a\n\"boundary swap\" and the gate goes silently green.\n\n#886's boundary-tie probe (`final_key_tied_beyond_limit`) is the exact signal\nneeded to distinguish a truncated tie from a complete one, but #886 scoped it to\nthe single-visible-row case only (its stated charter). For the >= 2 case the\nprobe is deliberately NOT consulted, and `cross_surface._final_key_tied_beyond_limit`\neven short-circuits (returns False, skipping the extra query) whenever the\nreference's final group already spans >= 2 rows - so today the >= 2 branch has no\ntruncation evidence at all.\n\nAccepted tradeoff (decided on #886): the failure mode is a MASKED bug (a false\nnegative) confined to a narrow shape - a complete, non-truncated, multi-row final\ntie group whose members differ between surfaces. All six enforced cross-surface\ngates currently pass with 0 unclassified divergent, so this is latent and\ndata-dependent, not an active breakage. It is documented in-code at the guard\nsite. Closing it is a deliberate, separately-scoped change (below), not a\ndrive-by, because it alters the #872/#878 established semantics and forces a\ntest + cost change.\n\nPriority raised Low -> Medium (2026-06-28, holistic-review remediation): for a\nbenchmark tool, false confidence in a correctness gate is the worst outcome, so a\ndocumented masked-value-bug path in the comparator warrants Medium and should be\nclosed before the next published cross-platform comparison claim. (The fix is\nalready scoped in w1; this is the only change to this item.)", "id": "cross-surface-comparator-complete-final-tie-group-soundness", "preserves": ["All enforced cross-surface gates remain green with 0 unclassified divergences after the tighter final-tie check.", "TPC-Havoc strict validation behavior is byte-identical when order_aware/tie_aware is disabled.", "The fix remains targeted to boundary-tie soundness; it does not rewrite queries or compare untruncated result sets.", "Truncated ORDER BY/LIMIT boundary swaps remain accepted only when the probe proves the final key extends past the LIMIT."], "prior_art": [{"concept": "{'ref': 'PR #872', 'note': 'Introduced the order-aware/tie-aware comparator; original final-group accept was unconditional.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'PR #878', 'note': 'Tightened the final-group accept to len(orig_rows) >= 2 (catches the single-row value bug).'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'PR #886', 'note': 'Added the boundary-tie probe (final_key_tied_beyond_limit) for the single-visible-row case and documented this residual >= 2 gap at the guard site.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py:_validate_order_aware', 'note': 'The guard site: `if i == last_index and tie_aware and (len(orig_rows) >= 2 or final_key_tied_beyond_limit)`.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/equivalence/cross_surface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/validation.py"}, {"kind": "only_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "tests/integration/test_validator_order_aware.py"}], "state": "done", "title": "Cross-surface comparator: close the >=2 final-tie-group gap for a complete (non-truncated) final tie", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_validator_order_aware.py -q -k 'final_tie or boundary'", "description": "Complete final tie with differing members now fails", "expected": "complete non-truncated >=2 final tie membership difference raises; genuinely truncated boundary swap still passes", "item_id": "cross-surface-comparator-complete-final-tie-group-soundness", "last_result": null, "last_run": null, "seq": 1}, {"command": "make ssb-cross-surface-equivalence-report && make clickbench-cross-surface-equivalence-report && make tpchavoc-equivalence-report", "description": "Cross-surface and TPC-Havoc behavior stays green", "expected": "cross-surface gates remain 0 unclassified divergent; tpchavoc strict result counts unchanged", "item_id": "cross-surface-comparator-complete-final-tie-group-soundness", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cross-surface-comparator-complete-final-tie-group-soundness", "needs": [], "notes": "Make the order-aware final-group acceptance require evidence that the LIMIT\nactually truncated the tie, for BOTH the single-row and the >= 2 case, so a\ncomplete (non-truncated) final tie group difference is CAUGHT. The probe\nalready answers exactly this (\"does the final order key recur past the\ncutoff?\"), so the targeted shape is to consult `final_key_tied_beyond_limit`\nfor the >= 2 branch too rather than accepting on row count alone.\n\nScope to evaluate (this is NOT a one-liner - it ripples through three places):\n 1. validation.py `_validate_order_aware`: change the guard so the >= 2\n branch also requires truncation evidence. Keep it ADDITIVE for callers\n that do not opt into the probe.\n 2. cross_surface.py `_final_key_tied_beyond_limit`: today it short-circuits\n (returns False, skips the extra LIMIT n+1 query) when the reference final\n group already has >= 2 rows - precisely the case w1 now needs the probe\n for. That optimization must be removed/relaxed for the >= 2 case, at the\n cost of one extra bounded probe query for those queries.\n 3. tests/integration/test_validator_order_aware.py\n `test_order_aware_truncated_limit_boundary_swap_is_not_flagged` encodes\n the CURRENT behavior (a 2-row final swap accepted with tie_aware but NO\n probe). It must be re-framed to pass the probe signal, and a NEW test\n added for the complete (table-exhausted, non-truncated) 2-row final tie\n that must now RAISE.\n\nHard constraints (same as #886): the TPC-Havoc strict path (no order_aware/\ntie_aware) stays byte-identical; all six enforced cross-surface gates stay\ngreen with 0 unclassified divergent; tpchavoc 220/220 and 440/440 unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate the >=2 final-tie-group accept on truncation evidence so a complete final tie no longer masks a value bug", "wid": "w1"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-gate-rollout-followups", "id": 16, "reason": "DuckDB-backed SF=0.1 cells are sufficient as the reference oracle for now; a second engine is extra coverage, not a correctness gap.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly concludes a second engine is not a correctness gap; DuckDB SF=0.1 oracle deemed sufficient for now.", "summary": "Cross-surface equivalence on a second engine (beyond DuckDB-backed cells)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-gate-rollout-followups", "id": 17, "reason": "Per-benchmark id mapping is small and explicit; auto-derivation is a convenience optimization deferred until the mappings prove burdensome.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Deferral's own parent item is still open/planning; auto-derivation is a documented optimization within that item's existing scope. (tracked by cross-surface-gate-rollout-followups)", "summary": "Auto-deriving the SQL<->DataFrame id correspondence instead of per-benchmark mapping"}], "deps": [], "description": "Breadth-only follow-up to the cross-surface SQL<->DataFrame equivalence gate.\nIt tracks ONE thing: extending the gate to the dual-surface benchmarks that do\nnot yet have one, plus a fallback oracle for the benchmarks that ship no\nDataFrame query surface at all.\n\nSTATE OF THE CAMPAIGN (so this file is not misread as still-open hardening\nwork). The enforced GATES are now ssb, amplab, coffeeshop, clickbench, and\njoinorder_synthetic; STAGED_GATES is empty. The oracle's correctness and\ntrustworthiness - schema-type-faithful loading, the order/tie-aware comparator,\ngate-boundary determinism, the vacuous-empty-vs-empty guard, and\nmutation-testing the oracle's SENSITIVITY - was the cross-surface-oracle-\nremediation campaign (w1-w10), which is COMPLETE. This file does not revisit any\nof that; it only adds gate COVERAGE for more benchmarks, assuming that hardened\nloader/comparator is in place.\n\nAuthoritative inventory: _project/analysis/cross-surface-applicability.md\n(registry-detection sweep). Of the dual-surface unguarded candidates, 9 ship a\nDataFrame query registry and are not yet gated (4 with ids that overlap as-is, 5\nneeding id normalization), and 4 ship no DataFrame query surface and need a\nfallback oracle. Those three groups are w0, w1, w2 below.\n", "id": "cross-surface-gate-rollout-followups", "preserves": [], "prior_art": [{"concept": "_project/TODO/main/active/benchmark-cross-surface-equivalence-gate.yaml - the parent campaign (harness + the first enforced gates); its w4 also lists remaining benchmarks, which this file makes concrete.", "decision": "reuse", "path": "_project/TODO/main/active/benchmark-cross-surface-equivalence-gate.yaml"}, {"concept": "_project/TODO/main/active/cross-surface-oracle-remediation.yaml - the COMPLETED oracle correctness/trust campaign (w1-w10: loader types, tie-aware comparator, determinism, vacuous-cell guard, mutation testing). This breadth rollout assumes its fixes, and does not duplicate them.", "decision": "reuse", "path": "_project/TODO/main/active/cross-surface-oracle-remediation.yaml"}, {"concept": "_project/analysis/cross-surface-applicability.md - the authoritative gateable / needs-id-normalization / no-DataFrame-registry classification used below.", "decision": "reuse", "path": "_project/analysis/cross-surface-applicability.md"}, {"concept": "_project/analysis/gateable-generator-seeding-audit.md - a gate is flaky unless its generator is seeded/stable; check each benchmark here before wiring.", "decision": "reuse", "path": "_project/analysis/gateable-generator-seeding-audit.md"}, {"concept": "benchbox/core/equivalence/cross_surface.py - build__duckdb + the GATES registry; copy an existing builder (build_ssb_duckdb / build_amplab_duckdb) per new benchmark, and map ids in the builder where the sweep flags normalization.", "decision": "reuse", "path": "benchbox/core/equivalence/cross_surface.py"}, {"concept": "tests/integration/test_amplab_cross_surface_equivalence.py - the medium-marker fast-lane regression to mirror per newly-gated benchmark.", "decision": "reuse", "path": "tests/integration/test_amplab_cross_surface_equivalence.py"}], "priority": "medium", "scope": [], "state": "planning", "title": "Cross-surface gate breadth rollout: wire the remaining gateable benchmarks + fallback oracle", "verifications": [], "work": [{"evidence": null, "item_id": "cross-surface-gate-rollout-followups", "needs": [], "notes": "Per cross-surface-applicability.md, ids overlap as-is (no normalization):\nflightdata (20), h2odb (10), joinorder (113), read_primitives (157 SQL vs\n152 DF - gate the overlapping 152). One focused gate PR each, copying the\nbuild__duckdb + GATES pattern and adding a medium-marker fast-lane\ntest. CAVEATS to resolve per benchmark before gating:\n - flightdata and joinorder use real/downloaded data (no synthetic\n generator) - pin a bounded, stable data source so the gate is\n deterministic; joinorder may be SF=1.0-only, so confirm a cheap bounded\n cell is even feasible before committing to gate it.\n - read_primitives: gate only the 152 ids present on both surfaces; record\n the 5 SQL-only ids as intentionally out of cross-surface scope.\nIf a new gate surfaces a NEW oracle-correctness bug (not a benchmark logic\nbug), reopen cross-surface-oracle-remediation rather than patching here.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire cross-surface gates for the benchmarks whose SQL and DataFrame ids already overlap", "wid": "w0"}, {"evidence": null, "item_id": "cross-surface-gate-rollout-followups", "needs": [], "notes": "Per cross-surface-applicability.md, a DataFrame query registry exists but its\nids differ from the SQL ids by a naming convention (e.g. `1` vs `Q1`):\ndatavault (22), nyctaxi (25), tsbs_devops (18). Plus two partial/low-value\ncases: tpcds_obt (only 3 DF queries vs 89 SQL - gate the 3, mark the rest\nout of scope) and tpch_skew (22; low value, TPC-H-derived - gate only if the\nid mapping is cheap). Normalize the id mapping inside the builder (as\nbuild_amplab_duckdb maps `1`->`Q1`), then proceed exactly as w0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Wire cross-surface gates for the benchmarks that need SQL<->DataFrame id normalization first", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-gate-rollout-followups", "needs": [], "notes": "Four benchmarks load into DataFrames but ship NO DataFrame query registry,\nso cross-surface equivalence cannot reach them: metadata_primitives, tpcdi,\ntransaction_primitives, write_primitives. They have NO correctness oracle of\nany kind today. Give each a maintenance-light oracle - a differential\nsecond-engine comparison or a curated bounded expected-results subset - so\nthe coverage map does not silently leave them UNGUARDED. Lowest priority of\nthis file.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Provide a fallback oracle for the dual-loadable benchmarks that ship no DataFrame query surface", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "hand-maintain a second list of expected gate names", "why": "derive from GATES, or the guard itself drifts."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "A benchmark can be enforced in code (registered in cross_surface.GATES) yet not\npinned in the Make/CI surface - the standardized-command test does not assert that\nevery gate has a corresponding *-cross-surface-equivalence-report Make target and\na pr.yml correctness-gate step. The newer gates (ClickBench, JoinOrder-synthetic,\nH2O-DB) are exposed to exactly this drift: enforced in GATES, but their\nMake/CI pinning is maintained by hand and can fall out of sync.\n\nAdd a check that DERIVES the expected Make targets and pr.yml steps from the live\nGATES registry and fails when a gate lacks its target or CI step. This closes the\nsame drift class as the mutation-target guard (set(_TARGETS)==set(GATES)), one\nlayer up at the Make/CI boundary.", "id": "cross-surface-gate-target-ci-ratchet", "preserves": ["No change to gate behavior; this is a drift guard over the Make/CI surface."], "prior_art": [{"concept": "{'ref': '.github/workflows/pr.yml:404', 'note': 'The per-gate correctness-gate steps (ssb/amplab/coffeeshop/clickbench/joinorder/h2odb) that must be derivable from GATES.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py (GATES)', 'note': 'The single source of truth the check derives from.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:332', 'note': 'test_targets_cover_every_enforced_gate is the precedent: assert a registry-derived set equals the live GATES set. Mirror it for Make/CI targets.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/test_standardized_test_commands.py', 'note': 'Existing standardized-command test; extend it (or add a sibling) to derive expectations from GATES rather than a hand-maintained list.'}", "decision": "extend", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Derive expected cross-surface Make/CI targets from GATES so newer gates cannot be unpinned", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_standardized_test_commands.py -q", "description": "Adding a gate without its target/step fails the guard", "expected": "passes for current GATES; fails if a GATES entry lacks a Make target or pr.yml correctness-gate step", "item_id": "cross-surface-gate-target-ci-ratchet", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "cross-surface-gate-target-ci-ratchet", "needs": [], "notes": "Add a test that, for each name in cross_surface.GATES, asserts a\n`-cross-surface-equivalence-report` Make target exists AND a\ncorresponding step is present in pr.yml's correctness-gate job. Parse the\nMakefile and pr.yml as text/data; key off GATES so adding a gate without its\ntarget/step fails. Account for the name normalization already used\n(joinorder_synthetic -> joinorder-synthetic-...).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Assert every GATES entry has a Make target and a pr.yml correctness-gate step", "wid": "w1"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "silently auto-prune resolved entries on a normal run", "why": "default to FAIL so the removal is a reviewed change; auto-prune only behind an explicit --update-baseline."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "Cross-surface gates carry classified known_divergences (e.g. ClickBench Q18's\norder-less LIMIT, _H2ODB_KNOWN_DIVERGENCES). These entries are reported but do\nNOT fail the gate when they no longer reproduce. That asymmetry lets a stale\nexception linger: once a known divergence is silently resolved (by a loader fix,\na query change, or a comparator improvement), the baseline entry stays, and a\nFUTURE regression on that same cell would be masked as \"known\" instead of caught.\n\nAdd baseline hygiene: a known-divergence entry whose cell now matches must either\nfail the gate or force a baseline update (an --update-baseline discipline), so the\nexception list cannot drift ahead of reality. This mirrors the empty-baseline,\nfail-on-divergence discipline the TPC-Havoc gates already enforce, extended to the\n\"resolved exception\" direction.", "id": "cross-surface-known-divergence-baseline-hygiene", "preserves": ["A new unclassified divergence still fails the gate (no weakening of detection).", "Genuine, still-reproducing classified divergences (e.g. ClickBench Q18 order-less LIMIT) remain accepted.", "ResultValidator semantics unchanged; reuse, do not fork."], "prior_art": [{"concept": "{'ref': '_project/analysis/clickbench-cross-surface-divergences.md', 'note': 'Documents the single justified ClickBench Q18 baseline entry - the kind of entry that must be re-validated, not assumed permanent.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py (known_divergences, GATES)', 'note': 'Where classified exceptions live; the gate already reports them - add the reverse check (resolved-but-still-listed).'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/equivalence.py (KNOWN_DIVERGENCES, empty-baseline fail-on-divergence)', 'note': 'The discipline to mirror: a baseline is for intentional differences only; reuse its pattern, do not fork the comparator.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "benchbox/core/equivalence/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Fail (or force baseline update) when a known cross-surface divergence no longer reproduces", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/equivalence/test_cross_surface.py -q -k baseline", "description": "Stale known-divergence entry is caught (unit)", "expected": "a resolved-but-listed known_divergences entry fails the gate; a still-reproducing one is accepted; a new unclassified divergence fails", "item_id": "cross-surface-known-divergence-baseline-hygiene", "last_result": null, "last_run": null, "seq": 1}, {"command": "add a fake resolved entry to a gate's known_divergences, run `make ssb-cross-surface-equivalence-report`", "description": "Manual end-to-end check", "expected": "gate FAILS naming the resolved entry; passes again once removed (or under --update-baseline)", "item_id": "cross-surface-known-divergence-baseline-hygiene", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cross-surface-known-divergence-baseline-hygiene", "needs": [], "notes": "In the gate runner, after comparison, check each known_divergences entry: if\nthe cell it names now MATCHES (no divergence), that entry is stale. Default\nbehavior: FAIL the gate with a clear message naming the resolved entry and\ninstructing removal; provide an --update-baseline (or make target flag) to\ndrop resolved entries deterministically. Keep genuine, still-reproducing\nclassified divergences accepted exactly as today. Apply to every gate carrying\nknown_divergences.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Detect resolved-but-listed known divergences and fail (or require --update-baseline)", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-known-divergence-baseline-hygiene", "needs": ["w1"], "notes": "Add tests: (a) a known-divergence entry whose cell now matches fails the gate\n(or is removed under --update-baseline); (b) a still-reproducing classified\ndivergence is still accepted; (c) a NEW unclassified divergence still fails.\nReuse the planted-defect style of the existing gate tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test both directions: stale entry fails, live entry still accepted", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "pick an h2odb target that is vacuous (empty/all-NULL) at _H2ODB_SCALE", "why": "the sensitivity result would be meaningless (a BS3-vacuous pass)."}, {"INSTEAD": "(not specified)", "dont": "use `gate.scale_factor or EQUIVALENCE_SCALE`", "why": "rely on the dataclass default; the `or` form silently substitutes policy and defeats the contract."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The cross-surface mutation-sensitivity suite measures whether each enforced\ngate's comparator actually catches a planted divergence. Two coupled defects make\nit test a DIFFERENT dataset than CI gates, and leave one enforced gate uncovered:\n\n1. Wrong scale. _build_gate_cell (tests/integration/test_cross_surface_mutation_\n sensitivity.py:269) builds every cell with the GLOBAL EQUIVALENCE_SCALE,\n ignoring the gate's own scale_factor. H2O-DB explicitly carries\n scale_factor=_H2ODB_SCALE because its generator base is the 10M-row small tier\n - the global 0.1 would emit ~1M rows (cross_surface.py:973 comment). So the\n mutation harness measures sensitivity on a dataset that does not match the\n enforced gate, and cannot host H2O-DB at all as written.\n\n2. Missing target. _TARGETS (test:100) lists ssb/amplab/coffeeshop/clickbench/\n joinorder_synthetic but OMITS h2odb, even though h2odb is enforced in GATES\n (cross_surface.py:973). The drift guard test_targets_cover_every_enforced_gate\n (test:332) already asserts set(_TARGETS) == set(GATES) - so it is (or should\n be) RED for h2odb. The work is to make that EXISTING guard pass and ensure its\n lane runs in CI, not to add a new guard.", "id": "cross-surface-mutation-harness-scale-fidelity", "preserves": ["Existing gate mutation cells stay non-vacuous and their planted-mutation catches are unchanged.", "No change to production gate behavior or to CrossSurfaceGate semantics - this is test-harness fidelity only (plus possibly a CI lane wiring)."], "prior_art": [{"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py:973', 'note': 'h2odb gate definition with scale_factor=_H2ODB_SCALE and the ~100k-row bounded-cell rationale - pick an h2odb target query that is non-vacuous at that scale.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:100', 'note': '_TARGETS map and the per-target caveat comments - add h2odb with a discriminating GROUP BY + ORDER BY query returning multiple non-NULL rows.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:269', 'note': '_build_gate_cell uses gate.build(EQUIVALENCE_SCALE, ...). Change to gate.build(gate.scale_factor, ...) - scale_factor is part of the gate contract (CrossSurfaceGate dataclass default), so use it directly, NOT `gate.scale_factor or EQUIVALENCE_SCALE` (that re-leaks policy into the test).'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:332', 'note': 'test_targets_cover_every_enforced_gate already asserts set(_TARGETS)==set(GATES). This item makes it pass; do not add a duplicate guard.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "tests/integration/test_cross_surface_mutation_sensitivity.py"}], "state": "done", "title": "Mutation-sensitivity harness: build cells at each gate's own scale and cover H2O-DB (make the existing drift guard pass)", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_cross_surface_mutation_sensitivity.py -q", "description": "Every enforced gate is mutation-covered at its own scale", "expected": "test_targets_cover_every_enforced_gate passes (6 gates incl h2odb); all mutation kinds caught; unmutated targets green", "item_id": "cross-surface-mutation-harness-scale-fidelity", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "cross-surface-mutation-harness-scale-fidelity", "needs": [], "notes": "Change _build_gate_cell to call gate.build(gate.scale_factor, tmp_path) so the\nmutation cell matches the enforced-gate cell. Use the dataclass default as the\nsingle source of truth; do not re-implement an `or EQUIVALENCE_SCALE` fallback.\nRe-verify the existing parametrized gates still build non-vacuous reference\nrows at their own scale.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build each mutation cell at the gate's own scale_factor", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-mutation-harness-scale-fidelity", "needs": ["w1"], "notes": "Pick an h2odb query (GROUP BY + ORDER BY) that returns multiple non-NULL rows\nat _H2ODB_SCALE (~100k rows), add it to _TARGETS, and confirm\ntest_targets_cover_every_enforced_gate (test:332) and\ntest_unmutated_target_is_green pass for h2odb. The unmutated cell must be\nnon-empty and discriminating; each mutation kind (flip_comparator,\ndrop_group_key, reverse_sort, drop_join) must be CAUGHT (and reverse_sort's\ndocumented order-blind miss for non-ORDER-BY shapes still holds where it\nlegitimately should).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a non-vacuous H2O-DB mutation target and make the drift guard green", "wid": "w2"}, {"evidence": null, "item_id": "cross-surface-mutation-harness-scale-fidelity", "needs": ["w2"], "notes": "The drift guard only protects if its lane executes. Confirm the integration\nlane that collects test_cross_surface_mutation_sensitivity.py runs in\n.github/workflows (or add it to the appropriate required job). If it is a\nslow/integration marker excluded from the default lane, ensure at least the\nlightweight set(_TARGETS)==set(GATES) guard runs in a required lane so a future\ngate promotion cannot silently skip mutation coverage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm the mutation-sensitivity lane actually runs in CI", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "count an empty-vs-empty or unverified-id-mapping cell as coverage", "why": "that is the exact 'theater' the review flagged."}, {"INSTEAD": "fix the loader contract (w1) and verify on the real run path.", "dont": "fix the null bug only in the gate path or only in the generated factories", "why": "production and hand-written impls would stay broken"}, {"INSTEAD": "extend it with an opt-in order/tie-aware mode.", "dont": "fork ResultValidator for cross-surface", "why": "TPC-Havoc depends on it"}, {"INSTEAD": "green+empty+deterministic first.", "dont": "promote a staged gate to GATES while it is nondeterministic or has a non-empty baseline", "why": "GATES membership is the coverage signal"}, {"INSTEAD": "fix the wrong surface (this is exactly how #848 went wrong).", "dont": "silence a divergence by adding it to known_divergences", "why": "the baseline is for intentional presentational differences only"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-oracle-remediation", "id": 136, "reason": "Large; cross-surface proves agreement not correctness. Tracked as a deeper question in the review, not required to close these findings.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Independent SQL oracle for non-tpch/tpcds benchmarks is owned by active benchmark-correctness-oracle-coverage-map ('small curated expected-results subset') and cross-surface-gate-rollout-followups w2 fallback oracle.", "summary": "A truly independent SQL-side oracle for non-tpch/tpcds benchmarks (e.g. upstream published answer sets)."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-oracle-remediation", "id": 137, "reason": "Additive enhancement beyond w6's acceptance and M4's core concern (which is resolved: the gate drives the real production loader and already caught the TEXT->null / TIMESTAMP->string dtype bugs via value comparison). Designing an acceptable cross-engine dtype-equivalence map is a separable piece.", "resolution": "promoted", "resolved_item": "a-dedicated-dtype-asserting-cross-surface-gate-cell-compare", "resolved_reason": null, "summary": "A dedicated DTYPE-asserting cross-surface gate cell (compare each loaded column's dtype, not just its values, against the SQL reference)."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-oracle-remediation", "id": 138, "reason": "A separate SSB data-generation bug, not a cross-surface oracle defect; the vacuous cells are honestly classified in the ssb gate's allowlist. Tracked in #870.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Fixed: benchbox/core/ssb/generator.py:567 emits canonical 2-digit MFGR#12; DONE item ssb-generator-p-category-bug.yaml (Completed) + test_ssb_cross_surface_equivalence.py:89-98 pins the six queries non-empty. Allowlist removed.", "summary": "SSB generator emits a 3-digit p_category ('MFGR#212') so the standard SSB Q2.x predicates ('MFGR#12') match no rows at any scale (Q2.1-2.3/Q3.3/Q3.4/Q4.3 empty to SF=1.0)."}], "deps": [], "description": "An adversarial review of the cross-surface SQL<->DataFrame equivalence campaign\n(PRs #826-#848 on develop) reproduced every finding by running the gates and\nfound the oracle is real but narrow, oversold, and partly vacuous. This TODO\naddresses ALL of those findings. The committed evidence the review draws on\nlives in _project/analysis/clickbench-cross-surface-divergences.md,\n_project/analysis/joinorder-synthetic-cross-surface-divergences.md, and\n_project/analysis/oracle-coverage-map.md; reproduce any cell with\n`make -cross-surface-equivalence-report`.\n\nThe headline problem: PR #848 concluded \"ClickBench has ~no genuine DataFrame\nlogic bugs\" and triaged 27 divergences as \"tie-ambiguous top-N.\" That is false.\nRe-running the gate shows at least three REAL, deterministic DataFrame bugs that\n#848 rationalized away as ties:\n - Q6_pandas: wrong COUNT(DISTINCT SearchPhrase) (18 SQL vs 17 pandas), a\n scalar with NO ORDER BY and NO LIMIT - a tie is impossible. Root cause: the\n DataFrame loader maps empty-string -> None and pandas .nunique() drops the\n null. SearchPhrase has 74,724 empty strings / 0 nulls.\n - Q17_pandas / Q18_pandas: wrong GROUP BY - pandas default groupby(dropna=True)\n silently drops ~74k null-key rows, so the query runs over ~25% of the data.\n The count multiset differs, which a boundary tie cannot cause.\n - Q24 / Q17 col-1 \"\" vs None: the SAME root cause #848 deferred as a cosmetic\n \"dtype\" issue. It is not cosmetic - it changes COUNT(DISTINCT) and GROUP BY\n membership.\n\nCompounding issues the review proved by reproduction:\n - NON-DETERMINISM: the ClickBench gate returned 27 -> 25 -> 26 divergent cells\n across three identical runs, with per-query values changing run to run. The\n generated DATA is byte-identical across runs (verified by md5), so the wobble\n is ENGINE tie-ordering (DuckDB/Polars thread parallelism under\n LIMIT-without-total-ORDER), NOT data. random.seed(42) cannot fix it, and a\n known_divergences baseline is therefore un-pinnable.\n - ORDER-BY BLINDNESS: ResultValidator.validate_results_exact sorts FULL ROWS\n before comparing (tpchavoc/validation.py:54-79), so ORDER BY is never checked\n except via which rows survive LIMIT. A reversed ORDER BY passes silently;\n equally-valid tie picks are flagged as false divergences; a column mixing\n None and non-None crashes Python sorted() and is mislabeled \"error:\".\n - VACUOUS CELLS: at SF=0.1, 6 of 13 SSB queries (Q2.1-2.3, Q3.3, Q3.4, Q4.3)\n return ZERO rows on both surfaces, so 12 of 26 SSB cells pass empty-vs-empty\n while the report counts them as \"compared\". The build guards empty TABLES\n (cross_surface.py:261) but not empty RESULTS.\n - FALSE INDEPENDENCE: SSB and ~40/43 ClickBench DataFrame queries are GENERATED\n from a compact DSL spec, and BOTH \"independent backends\" are emitted from the\n SAME spec row - so a spec-level logic error appears identically on both, and\n the docs' \"two independent implementations\" claim (cross_surface.py:3-6,\n dataframe_surface.py:48-50) is false for them. CoffeeShop (hand-written) is\n the genuine exception.\n - LOADER BYPASS: the gate materializes contexts straight from typed DuckDB\n Arrow (dataframe_surface.py:74-125), never through DataFrameDataLoader, so a\n green gate proves NOTHING about the production CSV->parquet loader's dtype\n fidelity (leading-zero VARCHAR->int, empty->null). #847 \"deferred dtype\" is\n docs-only and the bug is still live; the #846 DDL parser is a naive\n whitespace tokenizer that breaks on DECIMAL(p, s), quoted names, and\n multi-word types.\n - GOVERNANCE: STAGED gates are not run in CI and have no promotion forcing\n function; \"guarded\" in the coverage map means \"registered in the GATES dict\",\n not \"verified green\"; the \"12 gateable\" applicability headline counts 6\n benchmarks whose SQL<->DF id mapping was never verified (the team's own TODO\n says \"do NOT guess\").", "id": "cross-surface-oracle-remediation", "preserves": ["All TPC-Havoc SQL and DataFrame equivalence gates stay green with empty baselines and byte-identical output - the comparator change (w2) is additive, never a fork.", "SSB and CoffeeShop enforced gates stay green throughout (CoffeeShop is the reference 'good' gate).", "The bounded one-cell-per-benchmark cost model is preserved; no full platform matrix in routine PRs.", "benchbox run default (execution-only) behavior is unchanged for every benchmark; gate and loader changes must keep the real DataFrame run path working (re-verify a DataFrame benchmark run after w1/w6)."], "prior_art": [{"concept": "_project/TODO/main/active/benchmark-cross-surface-equivalence-gate.yaml - the original campaign TODO; w2/w3/w4 there are still 'in_progress' and overlap this remediation.", "decision": "reuse", "path": "_project/TODO/main/active/benchmark-cross-surface-equivalence-gate.yaml"}, {"concept": "_project/analysis/clickbench-cross-surface-divergences.md - committed ClickBench divergence evidence (the Q6/Q17/Q18/Q24 cases this TODO remediates); reproduce with `make clickbench-cross-surface-equivalence-report`.", "decision": "reuse", "path": "_project/analysis/clickbench-cross-surface-divergences.md"}, {"concept": "_project/analysis/joinorder-synthetic-cross-surface-divergences.md - committed joinorder_synthetic divergence evidence (the deferred-dtype cases); reproduce with `make joinorder_synthetic-cross-surface-equivalence-report`.", "decision": "reuse", "path": "_project/analysis/joinorder-synthetic-cross-surface-divergences.md"}, {"concept": "_project/analysis/oracle-coverage-map.md - the coverage/applicability map (guarded == GATES membership, the 12-gateable headline) the M1/M2 honesty fixes target.", "decision": "reuse", "path": "_project/analysis/oracle-coverage-map.md"}, {"concept": "_project/scripts/generate_oracle_coverage_map.py (guarded := GATES membership, 85/111) and cross_surface_applicability_sweep.py (registry presence + id overlap); the honesty fixes for M1/M2.", "decision": "reuse", "path": "_project/scripts/generate_oracle_coverage_map.py"}, {"concept": "benchbox/core/clickbench/dataframe_queries/queries.py - _make_group_pandas (groupby as_index=False, line 291), nunique op (206/285), _SCALAR_ROWS/_GROUP_ROWS/_SELECT_ROWS spec tables (394/404/433); the dropna/nunique null bugs are systemic to these factories.", "decision": "reuse", "path": "benchbox/core/clickbench/dataframe_queries/queries.py"}, {"concept": "benchbox/core/dataframe/data_loader.py - PYARROW_TYPE_MAP (191-199), _extract_arrow_types (1373-1382), convert_csv_to_parquet strings_can_be_null=True (329), cache version (74); the production load path the gate bypasses.", "decision": "reuse", "path": "benchbox/core/dataframe/data_loader.py"}, {"concept": "benchbox/core/dataframe/schema_utils.py - column_name (29-43) / column_sql_type (46-51) naive column.split() DDL parser (#846).", "decision": "reuse", "path": "benchbox/core/dataframe/schema_utils.py"}, {"concept": "benchbox/core/equivalence/cross_surface.py - GATES vs STAGED_GATES, build_*_duckdb, find_cross_surface_divergences, count_executed_cells, run_gate, _report; the empty-RESULT guard belongs here.", "decision": "reuse", "path": "benchbox/core/equivalence/cross_surface.py"}, {"concept": "benchbox/core/equivalence/dataframe_surface.py - materialize_rows, fetch_reference_rows, find_surface_divergences, _normalize_value (lines 239-263), build_dataframe_contexts_from_specs (74-125); the loader-bypass and the None/NaN coercion live here.", "decision": "reuse", "path": "benchbox/core/equivalence/dataframe_surface.py"}, {"concept": "benchbox/core/ssb/dataframe_queries/queries.py - _QUERY_SPECS (158) and _make_expression_impl/_make_pandas_impl; both backends from one spec; SSB parameters that yield 6 empty results at SF=0.1.", "decision": "reuse", "path": "benchbox/core/ssb/dataframe_queries/queries.py"}, {"concept": "benchbox/core/tpchavoc/validation.py - ResultValidator.validate_results_exact (54-79) full-row sort; the order/tie-awareness change must not regress the TPC-Havoc gates that rely on it.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py"}], "priority": "high", "scope": [], "state": "done", "title": "Remediate the cross-surface equivalence oracle: fix the bugs it found-and-dismissed, make it deterministic, order-aware, non-vacuous, and honest", "verifications": [{"command": "python -m benchbox.core.equivalence.cross_surface --benchmark clickbench (x3, expect identical deterministic output)", "description": "Re-run the full review reproduction appendix and confirm each finding is resolved.", "expected": null, "item_id": "cross-surface-oracle-remediation", "last_result": null, "last_run": null, "seq": 1}, {"command": "pytest tests for cross-surface mutation sensitivity (w10)", "description": "Mutation tests prove the gate catches seeded filter/group/sort/join bugs.", "expected": null, "item_id": "cross-surface-oracle-remediation", "last_result": null, "last_run": null, "seq": 2}, {"command": "make tpchavoc-equivalence-report && make tpchavoc-dataframe-equivalence-report", "description": "TPC-Havoc gates unchanged.", "expected": null, "item_id": "cross-surface-oracle-remediation", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "The DataFrame surface conflates SQL '' with None, which changes results, not\njust presentation. Two layers must be decided and fixed:\n\nROOT FIX (preferred) - loader fidelity: a CSV empty field in a declared\nstring/VARCHAR/TEXT column must materialize as '' (matching DuckDB/SQL), not\nArrow null. Today convert_csv_to_parquet passes strings_can_be_null=True\n(data_loader.py:329) and pandas read_csv uses default na_values, so empties\nbecome null on the production path; the gate path (DuckDB->Arrow) keeps ''.\nDecide the canonical contract (recommend: declared-string empties stay '',\nmatching the SQL reference and the upstream ClickBench data) and enforce it on\nBOTH load paths. This is the true fix and also resolves Q24/Q17-col1.\n\nSYMPTOM FIX (only if the loader contract cannot be '' for a real reason):\nmake the generated factories null-correct - groupby(..., dropna=False) in\n_make_group_pandas (clickbench queries.py:291; mirror in ssb) and a\nnull-aware distinct count instead of bare pandas .nunique() (queries.py:206/\n285). But note this only patches generated factories; hand-written impls and\nfuture benchmarks would still be exposed - which is why the loader fix is\npreferred.\n\nWhichever path: verify against the REAL DataFrame run (dataframe_runner /\nbenchmark_mixin), not the gate only - the gate bypasses the loader (see w6),\nso a gate-only fix would be invisible to production.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the empty-string/null conflation in the DataFrame load path (C1: Q6, Q17, Q18, Q24 are REAL bugs)", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": ["w2", "w4"], "notes": "The campaign measured only that the gates PASS (specificity); it never measured\nwhether they CATCH a seeded bug (sensitivity). Add a mutation-test harness that,\nper gated benchmark, injects a known DataFrame error and asserts the gate goes\nRED:\n - flip a filter comparator (< -> <=),\n - drop a GROUP BY key,\n - reverse a sort direction (this is the BS2 regression guard),\n - drop a join.\nFor each mutation that the gate does NOT catch, document why (e.g. an empty-\nresult query from BS3, or an order-only change the comparator ignores) and feed\nit back into w2/w4. This closes the central unanswered question: is the oracle\nstrong enough to find logic bugs, or did it find \"no bugs\" because it is too\nweak/vacuous?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Mutation-test the oracle to measure SENSITIVITY, not just specificity (C2 - the central question)", "wid": "w10"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "validate_results_exact (validation.py:54-79) sorts full rows then compares\npositionally, so it is simultaneously BLIND to real ORDER BY bugs (a reversed\norder passes - DEMO1) and NOISY on legitimate ties (two equally-valid tie\npicks fail - DEMO2), and it crashes on a column mixing None with non-None\nbecause Python sorted() cannot order None vs int (DEMO3), which\nfind_surface_divergences (dataframe_surface.py:234) then mislabels as\n\"error:\" instead of a clean result.\n\nAdd an ORDER-AWARE comparison mode for queries that declare an ORDER BY:\n - compare the ordered prefix up to the first tie group exactly (this\n restores ORDER BY signal), then\n - treat the rows sharing the boundary order-key as a MULTISET (this removes\n the tie noise that #848 mistook for the whole story).\nFor queries with no ORDER BY, keep multiset/set comparison.\nFix the sort key to place None deterministically (e.g. sort on a (is_none,\nvalue) surrogate) so a NULL-bearing column never raises, and so a genuine\nNone-vs-value difference is reported as a MISMATCH, not an \"error:\".\n\nCRITICAL CONSTRAINT: this comparator is shared with the TPC-Havoc SQL and\nDataFrame gates, which rely on its current semantics and run with empty\nbaselines. The order/tie-awareness must be additive (opt-in per query/gate)\nand MUST keep all TPC-Havoc gates byte-identical-green. Do NOT fork the\nvalidator for cross-surface only; extend it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the comparator order-aware and tie-aware; fix the None/NaN sort crash (BS2)", "wid": "w2"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "The wobble is engine tie-ordering, not data. Pin it at the gate:\n - run the gate's DuckDB connection and the Polars/pandas execution\n single-threaded (PRAGMA threads=1; pl.Config / POLARS_MAX_THREADS=1), AND\n - have both surfaces apply a deterministic TOTAL-ORDER tie-breaker before any\n LIMIT (append all remaining columns / a stable key to ORDER BY on the SQL\n reference and the DataFrame impls), so LIMIT truncation is reproducible.\nDocument explicitly (in cross_surface.py and the review's \"determinism\" claim)\nthat random.seed(42) makes the DATA reproducible but does NOT make the GATE\ndeterministic; the tie-breaker + single-thread is what does.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Eliminate engine nondeterminism at the gate boundary (H1)", "wid": "w3"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "6/13 SSB queries return 0 rows at SF=0.1, so 12/26 cells pass empty-vs-empty.\nTwo parts:\n 1. Gate hygiene: track per-query reference row count in\n find_cross_surface_divergences; in _report, count a both-empty cell as\n NON-discriminating. The gate should FAIL (or require an explicit\n classified \"legitimately-empty\" entry) if any gated query's reference is\n empty, and the \"compared N of M cells\" line must exclude vacuous cells\n from the discriminating count.\n 2. Data discrimination: for SSB, choose query parameters (or a per-benchmark\n SF) so Q2.1-2.3/Q3.3/Q3.4/Q4.3 return rows at the bounded cell. Prefer\n parameterization over raising SF globally (keep PRs cheap).\nApply the same empty-result audit to coffeeshop (currently 0/11 empty - good),\nclickbench, and joinorder_synthetic before promotion.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fail (not silently pass) on vacuous empty-vs-empty cells; make every gated query discriminating (BS3)", "wid": "w4"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": ["w1", "w2", "w3"], "notes": "After w1 (null fix), w2 (tie-aware), w3 (determinism): re-run the ClickBench\ngate. Expectation: the genuine ties (Q9, Q15, Q16, Q31, Q32, Q33, Q38) become\nclean (tie-aware comparison), and the real bugs (Q6, Q17, Q18, Q24) are fixed\nby w1. ANY residual divergence is a real bug to fix, verified by checking the\nORDER-BY-key multiset, not extrapolated.\nCorrect _project/analysis/clickbench-cross-surface-divergences.md: the \"~no\ngenuine DataFrame logic bugs\" conclusion was wrong; record that Q6/Q17/Q18\nwere real null-handling bugs sharing root cause with the deferred '' vs None\nissue. Promote clickbench from STAGED_GATES to GATES only once it is green with\nan empty baseline AND deterministic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-triage ClickBench with the fixed loader+comparator and correct the #848 record (C1 closeout)", "wid": "w5"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "The gate's loader bypass (dataframe_surface.py:74-125 materializes from typed\nDuckDB Arrow) means it cannot see production CSV->parquet dtype corruption.\n - Add a gate cell/mode that loads each gated benchmark through the REAL\n DataFrameDataLoader (CSV->parquet->read_parquet) and compares both dtypes\n and values to the SQL reference, so loader corruption is observable.\n - Carry the w1 empty-string->'' contract to the remaining raw-CSV adapters.\n w1 fixed the Parquet path (strings_can_be_null=False) and the pandas CSV\n path; Polars scan_csv / modin / dask / cudf still map an empty string\n field to null on their raw-CSV read, so prefer_parquet=False (or a\n CSV-recommending adapter) reintroduces the Q6/Q17/Q18 divergence on those\n surfaces. Thread declared column types into each adapter's read_csv and\n preserve '' for string columns, mirroring pandas_df.read_csv.\n - Harden schema_utils.column_name/column_sql_type (schema_utils.py:29-51):\n replace the naive column.split() with a paren-aware, quote-aware DDL\n column parser that handles DECIMAL(10, 2) (space after comma), quoted/\n spaced names, and multi-word types (DOUBLE PRECISION, TIMESTAMP WITH TIME\n ZONE, NOT NULL DEFAULT ...). Add unit tests for each shape.\n - Extend PYARROW_TYPE_MAP and the polars/pandas maps (data_loader.py:191-199)\n to cover TEXT, BIGINT, REAL, DOUBLE PRECISION, NUMERIC, BOOLEAN so declared\n string/numeric types are applied at conversion instead of inferred.\n - Apply declared STRING dtypes on the production load path (not just date-\n name suppression) so a leading-zero VARCHAR is not inferred as int.\nRe-verify the real DataFrame benchmark run still works (this is a loader fix,\nnot gate-only special-casing).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a production-loader-path gate cell + harden the DDL parser and type maps (M4, L1, L2)", "wid": "w6"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "Correct the docstrings that assert \"two independent implementations\"\n(cross_surface.py:3-6) and \"independent ... can diverge independently\"\n(dataframe_surface.py:48-50). For SSB and ~40/43 ClickBench queries BOTH\nbackends are generated from one DSL spec row, so they cannot diverge on logic\n- only on engine semantics; the only real independence is SQL-text vs DSL-spec\n(same author, same understanding). State plainly that the oracle catches\nTRANSCRIPTION errors, not CONCEPTUAL ones, for DSL-generated benchmarks.\nRecord per gated benchmark whether its DataFrame surface is hand-written\n(coffeeshop) or generated (ssb, clickbench). Optional but recommended: for at\nleast the enforced gates, ensure one surface is independently authored, or add\na second independent reference (e.g. the upstream published ClickBench answer\nset) so the oracle is not just self-consistency.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop overclaiming independence; document what the oracle actually proves (BS1)", "wid": "w7"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "STAGED_GATES (clickbench, joinorder_synthetic) are not run in CI (pr.yml runs\nonly ssb + coffeeshop) and nothing fails when a staged gate is parked\nindefinitely; test_clickbench_is_staged_not_enforced only BLESSES parking.\n - Add the staged cross-surface reports to a non-blocking CI job so their\n divergence snapshots cannot silently rot (promote to blocking once green\n via w5).\n - Add a governance test: a benchmark may sit in STAGED_GATES only with a\n dated rationale and an owner; fail if a staged gate has been parked past a\n declared review-by date. Delete a staged gate that no one will burn down\n rather than leave a permanent red parking lot.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run staged gates in CI and add a promotion forcing-function (M3)", "wid": "w8"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "- generate_oracle_coverage_map.py derives 'guarded' from GATES dict membership\n (85/111) and never checks the gate is green. Either run each gate (or read\n its CI result) and add a distinct 'verified-green' signal, or rename\n 'guarded' -> 'gate-registered' so a registered-but-unverified gate is not\n reported as coverage.\n- cross_surface_applicability_sweep.py counts 6 'needs-id-mapping' benchmarks\n (amplab, datavault, nyctaxi, tpcds_obt, tpch_skew, tsbs_devops) as 'gateable'\n on ZERO verified id overlap; the campaign's own TODO warns 'do NOT guess'\n the nyctaxi/tsbs mappings and tpcds_obt is 3 DF queries vs 89 SQL. Downgrade\n these from 'gateable' to 'candidate-unverified' until an independent\n SQL<->DF id mapping is confirmed per benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Tighten coverage-map / applicability honesty (M1, M2)", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use CLUSTER BY", "dont": "implement BigQuery CTAS sort", "why": "BigQuery does not preserve row order and the sorted output would revert to arbitrary order on subsequent DML"}, {"INSTEAD": "if research confirms CTAS adds no value, return None", "dont": "implement CTAS sort for Snowflake on top of existing clustering", "why": "Snowflake's Automatic Clustering already handles physical data ordering and a CTAS would recreate the table without the clustering metadata"}, {"INSTEAD": "validate physical sort behavior before coding any SQL", "dont": "skip research units to save time", "why": "incorrect assumptions for cloud warehouses could produce silently incorrect benchmark data"}, {"INSTEAD": "the correct mechanism is OPTIMIZE ZORDER BY", "dont": "use CTAS for Databricks/Delta Lake", "why": "CTAS creates an unmanaged table losing Delta transactional metadata"}], "approach": "Research units (w1, w3, w5, w7, w9) can all run in parallel - they are read-only.\nImplementation follows research findings. Where the answer is \"unsupported / return None\",\nthe implementation is 5-10 lines (override + docstring); prioritize getting the documentation\nright over speed.\n\nFor cloud platforms that are unsupported, rely on the existing base-class skip logging from\nctas-sort-dataloader-connectivity. If gaps are discovered, capture them as a follow-up TODO\ninstead of modifying base adapter code in this item.\n\nDo not attempt live platform tests - unit tests with mocked connections are sufficient\nfor verifying SQL generation. The verification commands use only unit tests.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-19T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": ["ctas-sort-dataloader-connectivity"], "description": "With the centralized apply_ctas_sort() in PlatformAdapter (ctas-sort-dataloader-connectivity),\ncloud data warehouse platforms gain sorted-data loading by overriding only\n_build_ctas_sort_sql(table_name, sort_columns). This item implements those overrides.\n\nEach cloud platform has a distinct approach to physical data ordering:\n\nSnowflake\n Snowflake already applies SORT configuration via clustering keys in apply_unified_tuning().\n A post-load CTAS (CREATE OR REPLACE TABLE t AS SELECT * FROM t ORDER BY col) will\n re-materialize data sorted but without Automatic Clustering - this may be counterproductive.\n Research whether post-load CTAS sort is beneficial in addition to or instead of clustering.\n If clustering already covers the use case, _build_ctas_sort_sql should return None and the\n sort should remain in apply_unified_tuning() / apply_table_tunings().\n\nDatabricks / Delta Lake\n Delta Lake does not guarantee physical sort order on disk via CTAS ORDER BY.\n Physical ordering is achieved post-load via OPTIMIZE ... ZORDER BY (Z-ordering) or\n OPTIMIZE ... with liquid clustering. The _build_ctas_sort_sql hook is not the right\n mechanism; instead, an apply_post_load_optimize() pattern may be needed, or the existing\n Z_ORDERING TuningType support can be extended. Return None from _build_ctas_sort_sql\n and document the correct path.\n\nBigQuery\n BigQuery CTAS supports ORDER BY (CREATE OR REPLACE TABLE ... AS SELECT * FROM ... ORDER BY).\n However, BigQuery is distributed and row order is not preserved after DML, so a post-load\n CTAS sort does not produce persistent physical ordering. Clustering keys (via partition/\n cluster config) are the correct mechanism. Return None and document.\n\nRedshift\n Amazon Redshift supports SORT KEYS defined at table creation. A post-load CTAS\n (CREATE TABLE t_sorted AS SELECT * FROM t ORDER BY col) followed by DROP+RENAME is\n valid but loses distribution style and sort key metadata. The correct approach is to\n define the SORT KEY in CREATE TABLE DDL. Investigate whether a VACUUM SORT ONLY t\n command (which re-sorts in-place) is a better post-load mechanism than CTAS.\n\nClickHouse / ClickHouse Cloud\n ClickHouse table ORDER BY is defined immutably at CREATE TABLE time (ENGINE = MergeTree\n ORDER BY col). Post-load CTAS sort is not applicable. Return None from\n _build_ctas_sort_sql and document. The correct path is to include ORDER BY in CREATE\n TABLE DDL generation - this may require a separate tuning TODO for ClickHouse DDL.\n\nAthena\n Athena (Presto/Trino-based) queries S3 directly. CTAS (CREATE TABLE ... AS SELECT ...\n ORDER BY) writes new Parquet files to S3. This IS meaningful for query performance if\n the output is partitioned or sorted - but Athena's distributed nature means ORDER BY in\n CTAS only produces local file ordering, not global ordering across the dataset. Research\n whether this provides zone-map-equivalent benefits for Athena queries.\n\nAzure Synapse\n Azure Synapse Analytics supports CTAS (CREATE TABLE ... AS SELECT ... ORDER BY).\n Physical ordering depends on the distribution style (hash, round-robin, replicated).\n Investigate whether CTAS ORDER BY produces useful clustering for Synapse's columnar storage.", "id": "ctas-sort-cloud-platforms", "preserves": ["All existing cloud platform tests pass (no credential mocks broken)", "Databricks Z_ORDERING TuningType support in apply_unified_tuning() is not affected", "Platforms returning None from _build_ctas_sort_sql produce no SQL execution side effects", "Snowflake clustering key application in apply_unified_tuning() is not removed or bypassed"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/motherduck.py"}], "state": "done", "title": "Implement _build_ctas_sort_sql for cloud platforms (Snowflake, Databricks, BigQuery, Redshift, ClickHouse, Athena, Azure Synapse)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_cloud_platform_ctas_sort.py -v --tb=short", "description": "Cloud platform CTAS sort unit tests pass (no live credentials needed)", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-cloud-platforms", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing cloud platform tests unaffected", "expected": "0 failures", "item_id": "ctas-sort-cloud-platforms", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check . && uv run ty check", "description": "Lint and type check pass", "expected": "no errors", "item_id": "ctas-sort-cloud-platforms", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Snowflake apply_table_tunings() already handles TuningType.SORTING via clustering keys\n(benchbox/platforms/snowflake.py:~1570). Determine:\n- Does CTAS ORDER BY produce sorted micro-partitions independent of clustering?\n- Does it conflict with Automatic Clustering?\n- Is there a benchmark scenario where CTAS sort is better than ALTER TABLE CLUSTER BY?\nExpected outcome: return None (clustering already covers this) with doc comment.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Snowflake: does post-load CTAS ORDER BY add value beyond clustering keys?", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement AzureSynapseAdapter._build_ctas_sort_sql or document as unsupported", "wid": "w10"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "ClickHouse MergeTree ENGINE requires ORDER BY at CREATE TABLE time. There is no\npost-load CTAS that changes physical ordering. Override _build_ctas_sort_sql in\nClickHouseAdapter to return None with a doc comment. Create a follow-up note\n(or separate TODO) for ensuring the ORDER BY clause from TuningType.SORTING is\nincluded in ClickHouse CREATE TABLE DDL generation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document ClickHouse Cloud as intentionally unsupported via _build_ctas_sort_sql = None", "wid": "w11"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "BigQuery does not preserve row order. Override to return None with a comment directing\nusers to BigQuery clustering (CLUSTER BY) configuration which is separate from SORTING.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document BigQuery as intentionally unsupported via _build_ctas_sort_sql = None", "wid": "w12"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w10", "w11", "w12", "w2", "w4", "w6", "w8"], "notes": "Each platform test: mock connection, call apply_ctas_sort with a tuning_config\ncontaining sort columns, assert correct SQL generated or assert returns False with\nno SQL executed for unsupported platforms. Tests should not require live cloud credentials.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests: verify each cloud platform returns expected SQL or None", "wid": "w13"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w1"], "notes": "If research shows CTAS ORDER BY is beneficial: return\n CREATE OR REPLACE TABLE {t} AS SELECT * FROM {t} ORDER BY col;\nIf clustering already covers it: return None with log_debug message directing users\nto the clustering key configuration instead.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement SnowflakeAdapter._build_ctas_sort_sql based on w1 research", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Delta Lake physical ordering mechanisms:\n- OPTIMIZE t ZORDER BY col (multi-dimensional Z-order clustering)\n- OPTIMIZE t (liquid clustering if enabled)\n- CTAS ORDER BY (creates new unmanaged table, loses Delta metadata)\nConfirm that _build_ctas_sort_sql returning None is correct; document the right\npath for Databricks (likely a separate apply_post_load_optimize hook or extending\nthe existing Z_ORDERING TuningType support in apply_unified_tuning()).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Databricks/Delta Lake post-load sort mechanisms (OPTIMIZE ZORDER BY vs CTAS)", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w3"], "notes": "In DatabricksAdapter (or pyspark adapter), override _build_ctas_sort_sql to return None\nwith an explanatory comment. If the existing Z_ORDERING TuningType handling in\napply_unified_tuning() already covers physical sort, confirm and document the connection.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document Databricks unsupported CTAS sort; note correct Z-ordering path in code", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Redshift options:\n- VACUUM SORT ONLY table_name - re-sorts in-place using the table's defined SORT KEY.\n Requires SORT KEY to be defined in CREATE TABLE. No DROP/RENAME needed.\n- CTAS+DROP+RENAME - loses SORT KEY definition and distribution style.\nIf VACUUM SORT ONLY is viable: implement _build_ctas_sort_sql to return\n \"VACUUM SORT ONLY {table_name};\" (unconventional use of the hook, but works).\nIf not viable: return None and document.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Redshift: VACUUM SORT ONLY vs CTAS+DROP+RENAME for post-load sorting", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement RedshiftAdapter._build_ctas_sort_sql based on w5 research", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Athena CTAS writes Parquet files to S3. Key question: does ORDER BY in Athena CTAS\nproduce single sorted Parquet files, or distributed unsorted output across multiple\nfiles? Row group-level min/max statistics in Parquet enable Athena's predicate pushdown\n(equivalent to DuckDB zone maps). If ORDER BY produces sorted row groups within files,\nthis is beneficial.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Athena CTAS ORDER BY: does it produce zone-map-equivalent file sorting?", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement AthenaAdapter._build_ctas_sort_sql or document as unsupported", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Synapse columnar store uses columnstore indexes for compression and pushdown - physical\nrow order within a distribution affects rowgroup quality. CTAS ORDER BY may improve\nrowgroup delta store promotion. Verify against Synapse docs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Azure Synapse CTAS ORDER BY effectiveness with hash/round-robin distribution", "wid": "w9"}], "worktree": "physical-tuning"} -{"anti_patterns": [{"INSTEAD": "return None by default (opt-in model)", "dont": "make _build_ctas_sort_sql abstract", "why": "not all platforms support CTAS sort and forcing every subclass to implement it would break the many adapters that don't need it"}, {"INSTEAD": "keep it in the base apply_ctas_sort() before calling the hook", "dont": "move the validate_sql_identifier call into _build_ctas_sort_sql", "why": "identifier validation is a shared safety concern, not platform-specific"}, {"INSTEAD": "each platform builds its own SQL string directly", "dont": "pass DuckDBDDLGenerator into _build_ctas_sort_sql", "why": "the hook must be self-contained per platform"}, {"INSTEAD": "(not specified)", "dont": "rename generate_ctas_ddl parameter without grep-checking all callers first", "why": "(unstated)"}], "approach": "Start with w1 (add the hook method signature to base adapter) before touching any\nadapters, so the interface is settled before implementations follow.\n\nFor w2, keep the case-insensitive lookup and validate_sql_identifier call from the existing\nDuckDBAdapter.apply_ctas_sort() - both are correct and should move verbatim.\n\nFor w3, DuckDBAdapter._build_ctas_sort_sql builds the ORDER BY string from sort_columns\nwithout involving DuckDBDDLGenerator - just use f-string formatting directly since the\nbase class owns the logic that already extracted the columns. Only the SQL template string\nbelongs in the platform-specific hook.\n\nFor w4 (parameter rename), run grep first:\n grep -rn \"generate_ctas_ddl\" benchbox/ tests/\nto find all callers before renaming.\n\nFor w8 (MotherDuck), add the apply_ctas_sort call after each table's INSERT INTO completes\ninside MotherDuck's existing load_data() loop - mirror the DataLoader pattern.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-19T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": ["ctas-sorted-data-loading"], "description": "The initial CTAS sort implementation (ctas-sorted-data-loading) placed apply_ctas_sort()\ndirectly on DuckDBAdapter and used a duck-typed hasattr() check in DataLoader. This works\nfor DuckDB today, but extending to additional platforms would require copy-pasting the common\nlookup/validation/logging logic into each adapter.\n\nThis item centralizes apply_ctas_sort() in PlatformAdapter (base class) so that all common\nbehavior - case-insensitive table lookup, sort column extraction, validate_sql_identifier call,\ndry-run capture_sql, success logging, and debug logging on skipped paths - lives in exactly\none place. Each platform adapter overrides only the SQL generation hook\n_build_ctas_sort_sql(table_name, sort_columns) -> str | None, returning None if the platform\ndoes not support CTAS-based sorting.\n\nSecondary items in scope:\n- Wire tuning_config into the DataLoader calls for ClickHouse, StarRocks, SQLite, and\n MotherDuck (currently they construct DataLoader without tuning_config, so CTAS sort\n never fires even when tuning is enabled).\n- Rename the misleadingly named if_not_exists parameter in DuckDBDDLGenerator.generate_ctas_ddl()\n to or_replace (the parameter generates CREATE OR REPLACE, not IF NOT EXISTS).\n- Remove or simplify the hasattr(adapter, \"apply_ctas_sort\") guard in DataLoader now that\n the method is guaranteed by the base class; keep the tuning_config None check as the opt-in.\n- Add debug-level logging on all early-return paths in the base apply_ctas_sort() so\n operators can understand why sorting was skipped.", "id": "ctas-sort-dataloader-connectivity", "preserves": ["All existing test_duckdb_sorted_loading.py tests pass unchanged or are migrated 1-for-1", "DataLoader INSERT INTO path (tuning_config=None) is completely unaffected", "DuckDBAdapter.load_data() produces identical row counts after refactor", "DuckDBDDLGenerator.generate_ctas_ddl() behavioral output is identical after the parameter rename", "apply_ctas_sort returns False (not exception) when platform returns None from _build_ctas_sort_sql"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/duckdb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/workload.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/motherduck.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/sqlite.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/starrocks/workload.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter_ctas_sort.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_duckdb_sorted_loading.py"}], "state": "done", "title": "Centralize apply_ctas_sort in PlatformAdapter base class and wire all DataLoader callers", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_duckdb_sorted_loading.py -v --tb=short", "description": "All existing DuckDB sorted loading tests pass after refactor", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_base_adapter_ctas_sort.py -v --tb=short", "description": "New base adapter CTAS tests pass", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Full platform unit test suite passes", "expected": "0 failures", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run ruff check . && uv run ty check", "description": "Lint and type check pass", "expected": "no errors", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": [], "notes": "Signature: def _build_ctas_sort_sql(self, table_name: str, sort_columns: list[TuningColumn]) -> str | None\nDefault implementation returns None (platform does not support CTAS sort).\nPlace near apply_unified_tuning() in benchbox/platforms/base/adapter.py.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add _build_ctas_sort_sql(table_name, sort_columns) hook to PlatformAdapter base class returning None", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w9"], "notes": "DataLoader docstring: explain that adapters control CTAS sort via _build_ctas_sort_sql().\nPlatformAdapter: document both apply_ctas_sort() (call from DataLoader/load_data) and\n_build_ctas_sort_sql() (override in subclasses to enable platform-specific sorting).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DataLoader and base adapter docstrings to document the hook protocol", "wid": "w10"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2", "w3", "w9"], "notes": "Move/adapt the TestApplyCtasSort and TestDataLoaderTuningConfig tests from\ntest_duckdb_sorted_loading.py so that base class behavior is tested through a\nminimal concrete subclass, and DuckDB-specific tests remain in the DuckDB file.\nAdd test: platform that returns None from _build_ctas_sort_sql \u2192 apply_ctas_sort returns False.\nAdd test: MotherDuck hook returns correct DuckDB SQL.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update / add tests for centralized apply_ctas_sort in base adapter", "wid": "w11"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w1"], "notes": "Move all logic from DuckDBAdapter.apply_ctas_sort() to PlatformAdapter:\n1. Case-insensitive lookup of table_name in tuning_config.table_tunings\n2. Extract sort columns via table_tuning.get_columns_by_type(TuningType.SORTING)\n3. If no sort columns: log_debug(\"No sort columns for {table} - skipping CTAS sort\") and return False\n4. Call self._build_ctas_sort_sql(validated_table, sort_columns)\n5. If None returned: log_debug(\"{platform} does not support CTAS sort - skipping\") and return False\n6. Dry-run path: self.capture_sql(sql, \"ctas_sort\", validated_table); return True\n7. Execute sql via connection\n8. self.log_verbose(\"Applied CTAS sorting to {table_name}: {col_names}\")\n9. Return True\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement centralized apply_ctas_sort() in PlatformAdapter with all common logic", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "DuckDB override returns:\n CREATE OR REPLACE TABLE {table_name} AS SELECT * FROM {table_name} ORDER BY col1, col2;\nBuild ORDER BY string from sort_columns sorted by col.order.\nRemove the now-redundant apply_ctas_sort() method from DuckDBAdapter entirely.\nMotherDuck can be added here as well since it shares DuckDB SQL syntax.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate DuckDBAdapter._build_ctas_sort_sql override; remove duplicated apply_ctas_sort", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w3"], "notes": "The parameter if_not_exists in benchbox/core/tuning/generators/duckdb.py:generate_ctas_ddl()\ngenerates CREATE OR REPLACE TABLE - the name is factually wrong and misleading.\nRename to or_replace. Update all remaining call sites that still use the generator\n(DuckDBAdapter._build_ctas_sort_sql should already be direct SQL by w3). Run grep to\nfind all usages before renaming.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rename generate_ctas_ddl(if_not_exists=...) parameter to or_replace in DuckDBDDLGenerator", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "benchbox/platforms/clickhouse/workload.py constructs DataLoader without tuning_config.\nAdd: tuning_config=self.unified_tuning_configuration if self.tuning_enabled else None\nClickHouse uses its own ORDER BY at table creation; _build_ctas_sort_sql returns None\n(deferred to ctas-sort-cloud-platforms) so this wiring is a no-op for now but enables\nfuture opt-in without another connectivity change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into ClickHouse DataLoader instantiation", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "benchbox/platforms/starrocks/workload.py - same pattern as ClickHouse.\nAdd tuning_config parameter so future _build_ctas_sort_sql can activate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into StarRocks DataLoader instantiation", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "benchbox/platforms/sqlite.py - same pattern.\nSQLite will likely return None from _build_ctas_sort_sql (no efficient CTAS sort path)\nbut the wiring must be present for the hook to have any effect.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into SQLite DataLoader instantiation", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w3"], "notes": "MotherDuck has its own load_data() using raw INSERT INTO SELECT * FROM read_parquet().\nAfter each table loads, add a call to self.apply_ctas_sort(table_name, effective_tuning, connection)\nwhen tuning_enabled. MotherDuck _build_ctas_sort_sql returns DuckDB-compatible SQL (same engine).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into MotherDuck load_data (custom loader, not DataLoader)", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "In benchbox/platforms/base/data_loading.py the guard currently reads:\n if self.tuning_config and hasattr(self.adapter, \"apply_ctas_sort\"):\nSince apply_ctas_sort is now guaranteed on all PlatformAdapter subclasses, simplify to:\n if self.tuning_config:\nUpdate the docstring to remove duck-typing language; describe the base class hook protocol.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove hasattr(adapter, 'apply_ctas_sort') guard in DataLoader; keep tuning_config check", "wid": "w9"}], "worktree": "physical-tuning"} -{"anti_patterns": [{"INSTEAD": "verify that SELECT * FROM {table_name} is valid post-load before generating CTAS SQL", "dont": "assume DataFusion CTAS behavior matches DuckDB without verification", "why": "DataFusion's table registration model (external tables vs in-memory) differs"}, {"INSTEAD": "always wrap in a BEGIN/COMMIT block", "dont": "execute DROP TABLE without a preceding BEGIN transaction in PostgreSQL", "why": "a crash between DROP and RENAME would leave the table destroyed"}, {"INSTEAD": "confirm physical sort behavior before coding", "dont": "skip the research work units (w1, w3, w5) to save time", "why": "incorrect CTAS assumptions could silently produce unsorted data or corrupt the table"}], "approach": "Work research units (w1, w3, w5, w7) first in parallel - they are read-only and independent.\nThen implement based on findings (w2, w4, w6 depend on their research units).\nFor PostgreSQL, if the multi-statement list[str] return type is needed, extend apply_ctas_sort\nin the base class to handle it before implementing the PostgreSQL override.\n\nFor DataFusion, check the existing _load_table_parquet() and _load_table_csv() methods to\nunderstand how the table is registered - CTAS sort requires the table to exist as a\nnamed relation that can be self-joined in SELECT * FROM {table_name}.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-19T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": ["ctas-sort-dataloader-connectivity"], "description": "With the centralized apply_ctas_sort() in PlatformAdapter (ctas-sort-dataloader-connectivity),\neach local SQL platform needs only to override _build_ctas_sort_sql(table_name, sort_columns)\nto gain CTAS-based sorted loading. This item implements those overrides.\n\nPlatforms in scope (MotherDuck is handled in ctas-sort-dataloader-connectivity as it shares\nDuckDB's SQL syntax):\n\nDataFusion\n DataFusion's SQL dialect supports CREATE OR REPLACE TABLE ... AS SELECT ... ORDER BY ...\n (same pattern as DuckDB). Verify the exact syntax against DataFusion's SQL reference and\n implement the hook.\n\nPostgreSQL\n PostgreSQL does not support CREATE OR REPLACE TABLE ... AS SELECT. The closest equivalent is:\n CREATE TABLE {t}_sorted AS SELECT * FROM {t} ORDER BY col;\n DROP TABLE {t};\n ALTER TABLE {t}_sorted RENAME TO {t};\n This is safe within a single connection but requires the table to exist first (post-load).\n Wrap in a transaction to keep it atomic.\n\nStarRocks\n StarRocks supports CREATE TABLE ... AS SELECT (CTAS) with ORDER BY via its sort keys.\n However, StarRocks sorts are defined at table-creation time (via ORDER BY in CREATE TABLE),\n not on a post-load CTAS. Investigate whether a post-load CTAS reorders physical data or\n just creates an unsorted replica. If CTAS does not physically sort: return None and document\n as unsupported; the correct path for StarRocks is to define ORDER BY in CREATE TABLE DDL.\n\nSQLite\n SQLite has no efficient physical sort; INSERT ORDER determines storage order. A CTAS pattern\n (CREATE TABLE t2 AS SELECT * FROM t1 ORDER BY col; DROP TABLE t1; ALTER TABLE t2 RENAME TO t1)\n is syntactically valid but requires recreating indexes and constraints. Given SQLite's use\n as a development/testing platform (not production), assess whether the overhead is justified.\n If not: _build_ctas_sort_sql returns None with a logged debug message.", "id": "ctas-sort-local-sql-platforms", "preserves": ["All existing DataFusion, PostgreSQL, StarRocks, SQLite unit tests pass", "DataFusion load_data() produces identical row counts after adding _build_ctas_sort_sql", "PlatformAdapter.apply_ctas_sort remains backward-compatible for existing str/None hook returns while optionally supporting list[str]", "PostgreSQL COPY-based loading path is unmodified; only the post-load sort step is added", "SQLite DataLoader INSERT INTO path unchanged (CTAS sort returns None)", "StarRocks DataLoader INSERT INTO path unchanged if CTAS sort is determined unsupported"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/sqlite.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/starrocks/workload.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_local_platform_ctas_sort.py"}], "state": "done", "title": "Implement _build_ctas_sort_sql for local SQL platforms (DataFusion, PostgreSQL, StarRocks, SQLite)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_local_platform_ctas_sort.py -v --tb=short", "description": "Local SQL platform unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-local-sql-platforms", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "0 failures", "item_id": "ctas-sort-local-sql-platforms", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_local_platform_ctas_sort.py -k integration -v", "description": "DataFusion integration: sorted table row order matches sort key", "expected": "all tests pass", "item_id": "ctas-sort-local-sql-platforms", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "Check benchbox/platforms/datafusion.py and DataFusion SQL docs. Specifically verify:\n- Does DataFusion support CREATE OR REPLACE TABLE?\n- Does ORDER BY in CTAS physically sort the stored data?\n- What is the exact SQL for a registered in-memory or on-disk table?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research DataFusion CTAS sort syntax; verify CREATE OR REPLACE TABLE ... AS SELECT ... ORDER BY", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w1"], "notes": "If DataFusion supports CREATE OR REPLACE TABLE t AS SELECT * FROM t ORDER BY col:\n return f\"CREATE OR REPLACE TABLE {table_name} AS SELECT * FROM {table_name} ORDER BY {order_by};\"\nIf not, return None and document in code why DataFusion is unsupported.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DataFusionAdapter._build_ctas_sort_sql override", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "PostgreSQL post-load sort strategy:\n BEGIN;\n CREATE TABLE {t}_ctas_sort AS SELECT * FROM {t} ORDER BY col;\n DROP TABLE {t};\n ALTER TABLE {t}_ctas_sort RENAME TO {t};\n COMMIT;\nVerify: does this preserve indexes/constraints created by schema setup? Likely not -\nthe DROP removes the table and its constraints. The base class apply_ctas_sort runs\nafter data load but the schema (CREATE TABLE + indexes) was applied earlier in the\nrun. Determine whether constraints must be re-applied or if COPY-based loading makes\nthem unnecessary for benchmark purposes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research PostgreSQL DROP+CTAS sort pattern; assess transaction safety", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w3"], "notes": "Return a multi-statement SQL string (or list[str]) for the DROP+CTAS pattern.\nThe base class apply_ctas_sort must handle list[str] (execute each statement in order).\nIf apply_ctas_sort currently only handles str, extend it to handle list[str] as well.\nAlternatively, wrap all statements in a DO $$ ... $$ block if the connection supports it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement PostgreSQLAdapter._build_ctas_sort_sql", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "StarRocks uses a Duplicate Key / Sort Key model defined at table creation.\nKey questions:\n- Does CREATE TABLE t2 AS SELECT * FROM t1 ORDER BY col in StarRocks physically sort\n the output table's tablets?\n- Or is the ORDER BY hint advisory only?\nReference: StarRocks docs on CTAS, sort key behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research StarRocks CTAS sort - does post-load CTAS physically reorder data?", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w5"], "notes": "If CTAS physically sorts: implement override returning:\n CREATE TABLE {t}_sorted AS SELECT * FROM {t} ORDER BY col;\n (+ DROP+RENAME steps)\nIf CTAS does NOT physically sort, return None and add a log message explaining that\nStarRocks sort keys must be defined in CREATE TABLE DDL (pre-load, not post-load).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement StarRocksAdapter._build_ctas_sort_sql or document as unsupported", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "SQLite CTAS sort requires DROP + RENAME which loses indexes/constraints, same as PG.\nGiven SQLite is a dev/testing platform (not production performance target), the cost\nlikely outweighs the benefit. Decision: return None with log_debug(\n\"SQLite CTAS sort skipped - physical sort not beneficial for testing platform\").\nDocument the decision in code comments so future maintainers understand it was deliberate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Determine SQLite stance: implement trivial CTAS sort or document as intentionally unsupported", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w2", "w4", "w6", "w7"], "notes": "Each platform test verifies:\n- Correct SQL returned when sort columns are present\n- None returned (or correct skip) when platform doesn't support it\n- SQL contains correct column ordering (sorted by TuningColumn.order)\nFor PostgreSQL multi-statement path: verify all statements are returned and in order.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for DataFusion, PostgreSQL, StarRocks, SQLite _build_ctas_sort_sql", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w2"], "notes": "Mirror tests/unit/platforms/test_duckdb_sorted_loading.py::TestCtasSortIntegration\nfor DataFusion: load a small table, apply CTAS sort, verify row order matches sort columns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration test: DataFusion actual sortedness after load with tuning config", "wid": "w9"}], "worktree": "physical-tuning"} -{"anti_patterns": [{"INSTEAD": "add CTAS as an opt-in path triggered by tuning config", "dont": "modify DataLoader.load() default behavior", "why": "existing platforms depend on INSERT INTO"}, {"INSTEAD": "keep both paths", "dont": "remove INSERT INTO fallback", "why": "not all platforms support CTAS"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-01-07T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox's tuning infrastructure generates sort clauses via DuckDBDDLGenerator\n(e.g., `ORDER BY L_SHIPDATE, L_ORDERKEY`), but the data loader uses INSERT INTO\nstatements that don't preserve sorting.\n\nDuckDB doesn't support inline ORDER BY in CREATE TABLE. Sorted tables must be\ncreated via CTAS patterns:\n CREATE TABLE sorted_table AS SELECT * FROM source ORDER BY col;\n\nThe current DataLoader uses INSERT INTO with VALUES, which doesn't apply sorting.\nThis gap was discovered during sorted data performance experiments showing 5-28x\nspeedups for selective queries when data is properly sorted (zone map pruning).\n\nThe fix requires modifying the data loading path to:\n1. Detect when tuning config specifies sorting for a table\n2. Use CTAS pattern instead of INSERT INTO for those tables\n3. Support both fresh load (CTAS) and append scenarios", "id": "ctas-sorted-data-loading", "preserves": ["DuckDB adapter backward compatible when no tuning config provided", "Existing INSERT INTO loading path unchanged for non-sorted tables", "benchbox run --platform duckdb --benchmark tpch --scale 0.01 produces identical row counts"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Implement CTAS-based sorted data loading for DuckDB", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_duckdb*.py -v --tb=short", "description": "Existing DuckDB tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ctas-sorted-data-loading", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'sorted or ctas' -v --tb=short", "description": "Row counts match after CTAS vs INSERT INTO loading", "expected": "all tests pass", "item_id": "ctas-sorted-data-loading", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document current data loading flow in DataLoader class", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for sorted CTAS loading", "wid": "w10"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration test: verify sortedness after load", "wid": "w11"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Performance validation: confirm zone map pruning benefit", "wid": "w12"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Identify integration point for tuning config in load path", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design CTAS-based loading strategy for DuckDB", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle fresh load scenario (CTAS)", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle append scenario (temp table + ordered insert)", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tuning_config parameter to DataLoader", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDB CTAS handler for sorted tables", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Modify DuckDB adapter to pass tuning config to loader", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add generate_ctas_ddl usage in data loading path", "wid": "w9"}], "worktree": "physical-tuning"} +{"anti_patterns": [{"dont": "hand-maintain a second list of expected gate names", "instead": "(not specified)", "why": "derive from GATES, or the guard itself drifts."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "A benchmark can be enforced in code (registered in cross_surface.GATES) yet not\npinned in the Make/CI surface - the standardized-command test does not assert that\nevery gate has a corresponding *-cross-surface-equivalence-report Make target and\na pr.yml correctness-gate step. The newer gates (ClickBench, JoinOrder-synthetic,\nH2O-DB) are exposed to exactly this drift: enforced in GATES, but their\nMake/CI pinning is maintained by hand and can fall out of sync.\n\nAdd a check that DERIVES the expected Make targets and pr.yml steps from the live\nGATES registry and fails when a gate lacks its target or CI step. This closes the\nsame drift class as the mutation-target guard (set(_TARGETS)==set(GATES)), one\nlayer up at the Make/CI boundary.", "id": "cross-surface-gate-target-ci-ratchet", "preserves": ["No change to gate behavior; this is a drift guard over the Make/CI surface."], "prior_art": [{"concept": "{'ref': '.github/workflows/pr.yml:404', 'note': 'The per-gate correctness-gate steps (ssb/amplab/coffeeshop/clickbench/joinorder/h2odb) that must be derivable from GATES.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py (GATES)', 'note': 'The single source of truth the check derives from.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:332', 'note': 'test_targets_cover_every_enforced_gate is the precedent: assert a registry-derived set equals the live GATES set. Mirror it for Make/CI targets.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/test_standardized_test_commands.py', 'note': 'Existing standardized-command test; extend it (or add a sibling) to derive expectations from GATES rather than a hand-maintained list.'}", "decision": "extend", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Derive expected cross-surface Make/CI targets from GATES so newer gates cannot be unpinned", "verifications": [{"command": "uv run -- python -m pytest tests/unit/test_standardized_test_commands.py -q", "description": "Adding a gate without its target/step fails the guard", "expected": "passes for current GATES; fails if a GATES entry lacks a Make target or pr.yml correctness-gate step", "item_id": "cross-surface-gate-target-ci-ratchet", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "cross-surface-gate-target-ci-ratchet", "needs": [], "notes": "Add a test that, for each name in cross_surface.GATES, asserts a\n`-cross-surface-equivalence-report` Make target exists AND a\ncorresponding step is present in pr.yml's correctness-gate job. Parse the\nMakefile and pr.yml as text/data; key off GATES so adding a gate without its\ntarget/step fails. Account for the name normalization already used\n(joinorder_synthetic -> joinorder-synthetic-...).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Assert every GATES entry has a Make target and a pr.yml correctness-gate step", "wid": "w1"}], "worktree": "main"} +{"anti_patterns": [{"dont": "silently auto-prune resolved entries on a normal run", "instead": "(not specified)", "why": "default to FAIL so the removal is a reviewed change; auto-prune only behind an explicit --update-baseline."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "Cross-surface gates carry classified known_divergences (e.g. ClickBench Q18's\norder-less LIMIT, _H2ODB_KNOWN_DIVERGENCES). These entries are reported but do\nNOT fail the gate when they no longer reproduce. That asymmetry lets a stale\nexception linger: once a known divergence is silently resolved (by a loader fix,\na query change, or a comparator improvement), the baseline entry stays, and a\nFUTURE regression on that same cell would be masked as \"known\" instead of caught.\n\nAdd baseline hygiene: a known-divergence entry whose cell now matches must either\nfail the gate or force a baseline update (an --update-baseline discipline), so the\nexception list cannot drift ahead of reality. This mirrors the empty-baseline,\nfail-on-divergence discipline the TPC-Havoc gates already enforce, extended to the\n\"resolved exception\" direction.", "id": "cross-surface-known-divergence-baseline-hygiene", "preserves": ["A new unclassified divergence still fails the gate (no weakening of detection).", "Genuine, still-reproducing classified divergences (e.g. ClickBench Q18 order-less LIMIT) remain accepted.", "ResultValidator semantics unchanged; reuse, do not fork."], "prior_art": [{"concept": "{'ref': '_project/analysis/clickbench-cross-surface-divergences.md', 'note': 'Documents the single justified ClickBench Q18 baseline entry - the kind of entry that must be re-validated, not assumed permanent.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py (known_divergences, GATES)', 'note': 'Where classified exceptions live; the gate already reports them - add the reverse check (resolved-but-still-listed).'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/equivalence.py (KNOWN_DIVERGENCES, empty-baseline fail-on-divergence)', 'note': 'The discipline to mirror: a baseline is for intentional differences only; reuse its pattern, do not fork the comparator.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "benchbox/core/equivalence/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Fail (or force baseline update) when a known cross-surface divergence no longer reproduces", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/equivalence/test_cross_surface.py -q -k baseline", "description": "Stale known-divergence entry is caught (unit)", "expected": "a resolved-but-listed known_divergences entry fails the gate; a still-reproducing one is accepted; a new unclassified divergence fails", "item_id": "cross-surface-known-divergence-baseline-hygiene", "last_result": null, "last_run": null, "seq": 1}, {"command": "add a fake resolved entry to a gate's known_divergences, run `make ssb-cross-surface-equivalence-report`", "description": "Manual end-to-end check", "expected": "gate FAILS naming the resolved entry; passes again once removed (or under --update-baseline)", "item_id": "cross-surface-known-divergence-baseline-hygiene", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "cross-surface-known-divergence-baseline-hygiene", "needs": [], "notes": "In the gate runner, after comparison, check each known_divergences entry: if\nthe cell it names now MATCHES (no divergence), that entry is stale. Default\nbehavior: FAIL the gate with a clear message naming the resolved entry and\ninstructing removal; provide an --update-baseline (or make target flag) to\ndrop resolved entries deterministically. Keep genuine, still-reproducing\nclassified divergences accepted exactly as today. Apply to every gate carrying\nknown_divergences.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Detect resolved-but-listed known divergences and fail (or require --update-baseline)", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-known-divergence-baseline-hygiene", "needs": ["w1"], "notes": "Add tests: (a) a known-divergence entry whose cell now matches fails the gate\n(or is removed under --update-baseline); (b) a still-reproducing classified\ndivergence is still accepted; (c) a NEW unclassified divergence still fails.\nReuse the planted-defect style of the existing gate tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test both directions: stale entry fails, live entry still accepted", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "pick an h2odb target that is vacuous (empty/all-NULL) at _H2ODB_SCALE", "instead": "(not specified)", "why": "the sensitivity result would be meaningless (a BS3-vacuous pass)."}, {"dont": "use `gate.scale_factor or EQUIVALENCE_SCALE`", "instead": "(not specified)", "why": "rely on the dataclass default; the `or` form silently substitutes policy and defeats the contract."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The cross-surface mutation-sensitivity suite measures whether each enforced\ngate's comparator actually catches a planted divergence. Two coupled defects make\nit test a DIFFERENT dataset than CI gates, and leave one enforced gate uncovered:\n\n1. Wrong scale. _build_gate_cell (tests/integration/test_cross_surface_mutation_\n sensitivity.py:269) builds every cell with the GLOBAL EQUIVALENCE_SCALE,\n ignoring the gate's own scale_factor. H2O-DB explicitly carries\n scale_factor=_H2ODB_SCALE because its generator base is the 10M-row small tier\n - the global 0.1 would emit ~1M rows (cross_surface.py:973 comment). So the\n mutation harness measures sensitivity on a dataset that does not match the\n enforced gate, and cannot host H2O-DB at all as written.\n\n2. Missing target. _TARGETS (test:100) lists ssb/amplab/coffeeshop/clickbench/\n joinorder_synthetic but OMITS h2odb, even though h2odb is enforced in GATES\n (cross_surface.py:973). The drift guard test_targets_cover_every_enforced_gate\n (test:332) already asserts set(_TARGETS) == set(GATES) - so it is (or should\n be) RED for h2odb. The work is to make that EXISTING guard pass and ensure its\n lane runs in CI, not to add a new guard.", "id": "cross-surface-mutation-harness-scale-fidelity", "preserves": ["Existing gate mutation cells stay non-vacuous and their planted-mutation catches are unchanged.", "No change to production gate behavior or to CrossSurfaceGate semantics - this is test-harness fidelity only (plus possibly a CI lane wiring)."], "prior_art": [{"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py:973', 'note': 'h2odb gate definition with scale_factor=_H2ODB_SCALE and the ~100k-row bounded-cell rationale - pick an h2odb target query that is non-vacuous at that scale.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:100', 'note': '_TARGETS map and the per-target caveat comments - add h2odb with a discriminating GROUP BY + ORDER BY query returning multiple non-NULL rows.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:269', 'note': '_build_gate_cell uses gate.build(EQUIVALENCE_SCALE, ...). Change to gate.build(gate.scale_factor, ...) - scale_factor is part of the gate contract (CrossSurfaceGate dataclass default), so use it directly, NOT `gate.scale_factor or EQUIVALENCE_SCALE` (that re-leaks policy into the test).'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_cross_surface_mutation_sensitivity.py:332', 'note': 'test_targets_cover_every_enforced_gate already asserts set(_TARGETS)==set(GATES). This item makes it pass; do not add a duplicate guard.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "tests/integration/test_cross_surface_mutation_sensitivity.py"}], "state": "done", "title": "Mutation-sensitivity harness: build cells at each gate's own scale and cover H2O-DB (make the existing drift guard pass)", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_cross_surface_mutation_sensitivity.py -q", "description": "Every enforced gate is mutation-covered at its own scale", "expected": "test_targets_cover_every_enforced_gate passes (6 gates incl h2odb); all mutation kinds caught; unmutated targets green", "item_id": "cross-surface-mutation-harness-scale-fidelity", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "cross-surface-mutation-harness-scale-fidelity", "needs": [], "notes": "Change _build_gate_cell to call gate.build(gate.scale_factor, tmp_path) so the\nmutation cell matches the enforced-gate cell. Use the dataclass default as the\nsingle source of truth; do not re-implement an `or EQUIVALENCE_SCALE` fallback.\nRe-verify the existing parametrized gates still build non-vacuous reference\nrows at their own scale.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build each mutation cell at the gate's own scale_factor", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-mutation-harness-scale-fidelity", "needs": ["w1"], "notes": "Pick an h2odb query (GROUP BY + ORDER BY) that returns multiple non-NULL rows\nat _H2ODB_SCALE (~100k rows), add it to _TARGETS, and confirm\ntest_targets_cover_every_enforced_gate (test:332) and\ntest_unmutated_target_is_green pass for h2odb. The unmutated cell must be\nnon-empty and discriminating; each mutation kind (flip_comparator,\ndrop_group_key, reverse_sort, drop_join) must be CAUGHT (and reverse_sort's\ndocumented order-blind miss for non-ORDER-BY shapes still holds where it\nlegitimately should).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a non-vacuous H2O-DB mutation target and make the drift guard green", "wid": "w2"}, {"evidence": null, "item_id": "cross-surface-mutation-harness-scale-fidelity", "needs": ["w2"], "notes": "The drift guard only protects if its lane executes. Confirm the integration\nlane that collects test_cross_surface_mutation_sensitivity.py runs in\n.github/workflows (or add it to the appropriate required job). If it is a\nslow/integration marker excluded from the default lane, ensure at least the\nlightweight set(_TARGETS)==set(GATES) guard runs in a required lane so a future\ngate promotion cannot silently skip mutation coverage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm the mutation-sensitivity lane actually runs in CI", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "count an empty-vs-empty or unverified-id-mapping cell as coverage", "instead": "(not specified)", "why": "that is the exact 'theater' the review flagged."}, {"dont": "fix the null bug only in the gate path or only in the generated factories", "instead": "fix the loader contract (w1) and verify on the real run path.", "why": "production and hand-written impls would stay broken"}, {"dont": "fork ResultValidator for cross-surface", "instead": "extend it with an opt-in order/tie-aware mode.", "why": "TPC-Havoc depends on it"}, {"dont": "promote a staged gate to GATES while it is nondeterministic or has a non-empty baseline", "instead": "green+empty+deterministic first.", "why": "GATES membership is the coverage signal"}, {"dont": "silence a divergence by adding it to known_divergences", "instead": "fix the wrong surface (this is exactly how #848 went wrong).", "why": "the baseline is for intentional presentational differences only"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-oracle-remediation", "id": 136, "reason": "Large; cross-surface proves agreement not correctness. Tracked as a deeper question in the review, not required to close these findings.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Independent SQL oracle for non-tpch/tpcds benchmarks is owned by active benchmark-correctness-oracle-coverage-map ('small curated expected-results subset') and cross-surface-gate-rollout-followups w2 fallback oracle.", "summary": "A truly independent SQL-side oracle for non-tpch/tpcds benchmarks (e.g. upstream published answer sets)."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-oracle-remediation", "id": 137, "reason": "Additive enhancement beyond w6's acceptance and M4's core concern (which is resolved: the gate drives the real production loader and already caught the TEXT->null / TIMESTAMP->string dtype bugs via value comparison). Designing an acceptable cross-engine dtype-equivalence map is a separable piece.", "resolution": "promoted", "resolved_item": "a-dedicated-dtype-asserting-cross-surface-gate-cell-compare", "resolved_reason": null, "summary": "A dedicated DTYPE-asserting cross-surface gate cell (compare each loaded column's dtype, not just its values, against the SQL reference)."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "cross-surface-oracle-remediation", "id": 138, "reason": "A separate SSB data-generation bug, not a cross-surface oracle defect; the vacuous cells are honestly classified in the ssb gate's allowlist. Tracked in #870.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Fixed: benchbox/core/ssb/generator.py:567 emits canonical 2-digit MFGR#12; DONE item ssb-generator-p-category-bug.yaml (Completed) + test_ssb_cross_surface_equivalence.py:89-98 pins the six queries non-empty. Allowlist removed.", "summary": "SSB generator emits a 3-digit p_category ('MFGR#212') so the standard SSB Q2.x predicates ('MFGR#12') match no rows at any scale (Q2.1-2.3/Q3.3/Q3.4/Q4.3 empty to SF=1.0)."}], "deps": [], "description": "An adversarial review of the cross-surface SQL<->DataFrame equivalence campaign\n(PRs #826-#848 on develop) reproduced every finding by running the gates and\nfound the oracle is real but narrow, oversold, and partly vacuous. This TODO\naddresses ALL of those findings. The committed evidence the review draws on\nlives in _project/analysis/clickbench-cross-surface-divergences.md,\n_project/analysis/joinorder-synthetic-cross-surface-divergences.md, and\n_project/analysis/oracle-coverage-map.md; reproduce any cell with\n`make -cross-surface-equivalence-report`.\n\nThe headline problem: PR #848 concluded \"ClickBench has ~no genuine DataFrame\nlogic bugs\" and triaged 27 divergences as \"tie-ambiguous top-N.\" That is false.\nRe-running the gate shows at least three REAL, deterministic DataFrame bugs that\n#848 rationalized away as ties:\n - Q6_pandas: wrong COUNT(DISTINCT SearchPhrase) (18 SQL vs 17 pandas), a\n scalar with NO ORDER BY and NO LIMIT - a tie is impossible. Root cause: the\n DataFrame loader maps empty-string -> None and pandas .nunique() drops the\n null. SearchPhrase has 74,724 empty strings / 0 nulls.\n - Q17_pandas / Q18_pandas: wrong GROUP BY - pandas default groupby(dropna=True)\n silently drops ~74k null-key rows, so the query runs over ~25% of the data.\n The count multiset differs, which a boundary tie cannot cause.\n - Q24 / Q17 col-1 \"\" vs None: the SAME root cause #848 deferred as a cosmetic\n \"dtype\" issue. It is not cosmetic - it changes COUNT(DISTINCT) and GROUP BY\n membership.\n\nCompounding issues the review proved by reproduction:\n - NON-DETERMINISM: the ClickBench gate returned 27 -> 25 -> 26 divergent cells\n across three identical runs, with per-query values changing run to run. The\n generated DATA is byte-identical across runs (verified by md5), so the wobble\n is ENGINE tie-ordering (DuckDB/Polars thread parallelism under\n LIMIT-without-total-ORDER), NOT data. random.seed(42) cannot fix it, and a\n known_divergences baseline is therefore un-pinnable.\n - ORDER-BY BLINDNESS: ResultValidator.validate_results_exact sorts FULL ROWS\n before comparing (tpchavoc/validation.py:54-79), so ORDER BY is never checked\n except via which rows survive LIMIT. A reversed ORDER BY passes silently;\n equally-valid tie picks are flagged as false divergences; a column mixing\n None and non-None crashes Python sorted() and is mislabeled \"error:\".\n - VACUOUS CELLS: at SF=0.1, 6 of 13 SSB queries (Q2.1-2.3, Q3.3, Q3.4, Q4.3)\n return ZERO rows on both surfaces, so 12 of 26 SSB cells pass empty-vs-empty\n while the report counts them as \"compared\". The build guards empty TABLES\n (cross_surface.py:261) but not empty RESULTS.\n - FALSE INDEPENDENCE: SSB and ~40/43 ClickBench DataFrame queries are GENERATED\n from a compact DSL spec, and BOTH \"independent backends\" are emitted from the\n SAME spec row - so a spec-level logic error appears identically on both, and\n the docs' \"two independent implementations\" claim (cross_surface.py:3-6,\n dataframe_surface.py:48-50) is false for them. CoffeeShop (hand-written) is\n the genuine exception.\n - LOADER BYPASS: the gate materializes contexts straight from typed DuckDB\n Arrow (dataframe_surface.py:74-125), never through DataFrameDataLoader, so a\n green gate proves NOTHING about the production CSV->parquet loader's dtype\n fidelity (leading-zero VARCHAR->int, empty->null). #847 \"deferred dtype\" is\n docs-only and the bug is still live; the #846 DDL parser is a naive\n whitespace tokenizer that breaks on DECIMAL(p, s), quoted names, and\n multi-word types.\n - GOVERNANCE: STAGED gates are not run in CI and have no promotion forcing\n function; \"guarded\" in the coverage map means \"registered in the GATES dict\",\n not \"verified green\"; the \"12 gateable\" applicability headline counts 6\n benchmarks whose SQL<->DF id mapping was never verified (the team's own TODO\n says \"do NOT guess\").", "id": "cross-surface-oracle-remediation", "preserves": ["All TPC-Havoc SQL and DataFrame equivalence gates stay green with empty baselines and byte-identical output - the comparator change (w2) is additive, never a fork.", "SSB and CoffeeShop enforced gates stay green throughout (CoffeeShop is the reference 'good' gate).", "The bounded one-cell-per-benchmark cost model is preserved; no full platform matrix in routine PRs.", "benchbox run default (execution-only) behavior is unchanged for every benchmark; gate and loader changes must keep the real DataFrame run path working (re-verify a DataFrame benchmark run after w1/w6)."], "prior_art": [{"concept": "_project/TODO/main/active/benchmark-cross-surface-equivalence-gate.yaml - the original campaign TODO; w2/w3/w4 there are still 'in_progress' and overlap this remediation.", "decision": "reuse", "path": "_project/TODO/main/active/benchmark-cross-surface-equivalence-gate.yaml"}, {"concept": "_project/analysis/clickbench-cross-surface-divergences.md - committed ClickBench divergence evidence (the Q6/Q17/Q18/Q24 cases this TODO remediates); reproduce with `make clickbench-cross-surface-equivalence-report`.", "decision": "reuse", "path": "_project/analysis/clickbench-cross-surface-divergences.md"}, {"concept": "_project/analysis/joinorder-synthetic-cross-surface-divergences.md - committed joinorder_synthetic divergence evidence (the deferred-dtype cases); reproduce with `make joinorder_synthetic-cross-surface-equivalence-report`.", "decision": "reuse", "path": "_project/analysis/joinorder-synthetic-cross-surface-divergences.md"}, {"concept": "_project/analysis/oracle-coverage-map.md - the coverage/applicability map (guarded == GATES membership, the 12-gateable headline) the M1/M2 honesty fixes target.", "decision": "reuse", "path": "_project/analysis/oracle-coverage-map.md"}, {"concept": "_project/scripts/generate_oracle_coverage_map.py (guarded := GATES membership, 85/111) and cross_surface_applicability_sweep.py (registry presence + id overlap); the honesty fixes for M1/M2.", "decision": "reuse", "path": "_project/scripts/generate_oracle_coverage_map.py"}, {"concept": "benchbox/core/clickbench/dataframe_queries/queries.py - _make_group_pandas (groupby as_index=False, line 291), nunique op (206/285), _SCALAR_ROWS/_GROUP_ROWS/_SELECT_ROWS spec tables (394/404/433); the dropna/nunique null bugs are systemic to these factories.", "decision": "reuse", "path": "benchbox/core/clickbench/dataframe_queries/queries.py"}, {"concept": "benchbox/core/dataframe/data_loader.py - PYARROW_TYPE_MAP (191-199), _extract_arrow_types (1373-1382), convert_csv_to_parquet strings_can_be_null=True (329), cache version (74); the production load path the gate bypasses.", "decision": "reuse", "path": "benchbox/core/dataframe/data_loader.py"}, {"concept": "benchbox/core/dataframe/schema_utils.py - column_name (29-43) / column_sql_type (46-51) naive column.split() DDL parser (#846).", "decision": "reuse", "path": "benchbox/core/dataframe/schema_utils.py"}, {"concept": "benchbox/core/equivalence/cross_surface.py - GATES vs STAGED_GATES, build_*_duckdb, find_cross_surface_divergences, count_executed_cells, run_gate, _report; the empty-RESULT guard belongs here.", "decision": "reuse", "path": "benchbox/core/equivalence/cross_surface.py"}, {"concept": "benchbox/core/equivalence/dataframe_surface.py - materialize_rows, fetch_reference_rows, find_surface_divergences, _normalize_value (lines 239-263), build_dataframe_contexts_from_specs (74-125); the loader-bypass and the None/NaN coercion live here.", "decision": "reuse", "path": "benchbox/core/equivalence/dataframe_surface.py"}, {"concept": "benchbox/core/ssb/dataframe_queries/queries.py - _QUERY_SPECS (158) and _make_expression_impl/_make_pandas_impl; both backends from one spec; SSB parameters that yield 6 empty results at SF=0.1.", "decision": "reuse", "path": "benchbox/core/ssb/dataframe_queries/queries.py"}, {"concept": "benchbox/core/tpchavoc/validation.py - ResultValidator.validate_results_exact (54-79) full-row sort; the order/tie-awareness change must not regress the TPC-Havoc gates that rely on it.", "decision": "reuse", "path": "benchbox/core/tpchavoc/validation.py"}], "priority": "high", "scope": [], "state": "done", "title": "Remediate the cross-surface equivalence oracle: fix the bugs it found-and-dismissed, make it deterministic, order-aware, non-vacuous, and honest", "verifications": [{"command": "python -m benchbox.core.equivalence.cross_surface --benchmark clickbench (x3, expect identical deterministic output)", "description": "Re-run the full review reproduction appendix and confirm each finding is resolved.", "expected": null, "item_id": "cross-surface-oracle-remediation", "last_result": null, "last_run": null, "seq": 1}, {"command": "pytest tests for cross-surface mutation sensitivity (w10)", "description": "Mutation tests prove the gate catches seeded filter/group/sort/join bugs.", "expected": null, "item_id": "cross-surface-oracle-remediation", "last_result": null, "last_run": null, "seq": 2}, {"command": "make tpchavoc-equivalence-report && make tpchavoc-dataframe-equivalence-report", "description": "TPC-Havoc gates unchanged.", "expected": null, "item_id": "cross-surface-oracle-remediation", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "The DataFrame surface conflates SQL '' with None, which changes results, not\njust presentation. Two layers must be decided and fixed:\n\nROOT FIX (preferred) - loader fidelity: a CSV empty field in a declared\nstring/VARCHAR/TEXT column must materialize as '' (matching DuckDB/SQL), not\nArrow null. Today convert_csv_to_parquet passes strings_can_be_null=True\n(data_loader.py:329) and pandas read_csv uses default na_values, so empties\nbecome null on the production path; the gate path (DuckDB->Arrow) keeps ''.\nDecide the canonical contract (recommend: declared-string empties stay '',\nmatching the SQL reference and the upstream ClickBench data) and enforce it on\nBOTH load paths. This is the true fix and also resolves Q24/Q17-col1.\n\nSYMPTOM FIX (only if the loader contract cannot be '' for a real reason):\nmake the generated factories null-correct - groupby(..., dropna=False) in\n_make_group_pandas (clickbench queries.py:291; mirror in ssb) and a\nnull-aware distinct count instead of bare pandas .nunique() (queries.py:206/\n285). But note this only patches generated factories; hand-written impls and\nfuture benchmarks would still be exposed - which is why the loader fix is\npreferred.\n\nWhichever path: verify against the REAL DataFrame run (dataframe_runner /\nbenchmark_mixin), not the gate only - the gate bypasses the loader (see w6),\nso a gate-only fix would be invisible to production.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the empty-string/null conflation in the DataFrame load path (C1: Q6, Q17, Q18, Q24 are REAL bugs)", "wid": "w1"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": ["w2", "w4"], "notes": "The campaign measured only that the gates PASS (specificity); it never measured\nwhether they CATCH a seeded bug (sensitivity). Add a mutation-test harness that,\nper gated benchmark, injects a known DataFrame error and asserts the gate goes\nRED:\n - flip a filter comparator (< -> <=),\n - drop a GROUP BY key,\n - reverse a sort direction (this is the BS2 regression guard),\n - drop a join.\nFor each mutation that the gate does NOT catch, document why (e.g. an empty-\nresult query from BS3, or an order-only change the comparator ignores) and feed\nit back into w2/w4. This closes the central unanswered question: is the oracle\nstrong enough to find logic bugs, or did it find \"no bugs\" because it is too\nweak/vacuous?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Mutation-test the oracle to measure SENSITIVITY, not just specificity (C2 - the central question)", "wid": "w10"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "validate_results_exact (validation.py:54-79) sorts full rows then compares\npositionally, so it is simultaneously BLIND to real ORDER BY bugs (a reversed\norder passes - DEMO1) and NOISY on legitimate ties (two equally-valid tie\npicks fail - DEMO2), and it crashes on a column mixing None with non-None\nbecause Python sorted() cannot order None vs int (DEMO3), which\nfind_surface_divergences (dataframe_surface.py:234) then mislabels as\n\"error:\" instead of a clean result.\n\nAdd an ORDER-AWARE comparison mode for queries that declare an ORDER BY:\n - compare the ordered prefix up to the first tie group exactly (this\n restores ORDER BY signal), then\n - treat the rows sharing the boundary order-key as a MULTISET (this removes\n the tie noise that #848 mistook for the whole story).\nFor queries with no ORDER BY, keep multiset/set comparison.\nFix the sort key to place None deterministically (e.g. sort on a (is_none,\nvalue) surrogate) so a NULL-bearing column never raises, and so a genuine\nNone-vs-value difference is reported as a MISMATCH, not an \"error:\".\n\nCRITICAL CONSTRAINT: this comparator is shared with the TPC-Havoc SQL and\nDataFrame gates, which rely on its current semantics and run with empty\nbaselines. The order/tie-awareness must be additive (opt-in per query/gate)\nand MUST keep all TPC-Havoc gates byte-identical-green. Do NOT fork the\nvalidator for cross-surface only; extend it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the comparator order-aware and tie-aware; fix the None/NaN sort crash (BS2)", "wid": "w2"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "The wobble is engine tie-ordering, not data. Pin it at the gate:\n - run the gate's DuckDB connection and the Polars/pandas execution\n single-threaded (PRAGMA threads=1; pl.Config / POLARS_MAX_THREADS=1), AND\n - have both surfaces apply a deterministic TOTAL-ORDER tie-breaker before any\n LIMIT (append all remaining columns / a stable key to ORDER BY on the SQL\n reference and the DataFrame impls), so LIMIT truncation is reproducible.\nDocument explicitly (in cross_surface.py and the review's \"determinism\" claim)\nthat random.seed(42) makes the DATA reproducible but does NOT make the GATE\ndeterministic; the tie-breaker + single-thread is what does.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Eliminate engine nondeterminism at the gate boundary (H1)", "wid": "w3"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "6/13 SSB queries return 0 rows at SF=0.1, so 12/26 cells pass empty-vs-empty.\nTwo parts:\n 1. Gate hygiene: track per-query reference row count in\n find_cross_surface_divergences; in _report, count a both-empty cell as\n NON-discriminating. The gate should FAIL (or require an explicit\n classified \"legitimately-empty\" entry) if any gated query's reference is\n empty, and the \"compared N of M cells\" line must exclude vacuous cells\n from the discriminating count.\n 2. Data discrimination: for SSB, choose query parameters (or a per-benchmark\n SF) so Q2.1-2.3/Q3.3/Q3.4/Q4.3 return rows at the bounded cell. Prefer\n parameterization over raising SF globally (keep PRs cheap).\nApply the same empty-result audit to coffeeshop (currently 0/11 empty - good),\nclickbench, and joinorder_synthetic before promotion.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fail (not silently pass) on vacuous empty-vs-empty cells; make every gated query discriminating (BS3)", "wid": "w4"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": ["w1", "w2", "w3"], "notes": "After w1 (null fix), w2 (tie-aware), w3 (determinism): re-run the ClickBench\ngate. Expectation: the genuine ties (Q9, Q15, Q16, Q31, Q32, Q33, Q38) become\nclean (tie-aware comparison), and the real bugs (Q6, Q17, Q18, Q24) are fixed\nby w1. ANY residual divergence is a real bug to fix, verified by checking the\nORDER-BY-key multiset, not extrapolated.\nCorrect _project/analysis/clickbench-cross-surface-divergences.md: the \"~no\ngenuine DataFrame logic bugs\" conclusion was wrong; record that Q6/Q17/Q18\nwere real null-handling bugs sharing root cause with the deferred '' vs None\nissue. Promote clickbench from STAGED_GATES to GATES only once it is green with\nan empty baseline AND deterministic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-triage ClickBench with the fixed loader+comparator and correct the #848 record (C1 closeout)", "wid": "w5"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "The gate's loader bypass (dataframe_surface.py:74-125 materializes from typed\nDuckDB Arrow) means it cannot see production CSV->parquet dtype corruption.\n - Add a gate cell/mode that loads each gated benchmark through the REAL\n DataFrameDataLoader (CSV->parquet->read_parquet) and compares both dtypes\n and values to the SQL reference, so loader corruption is observable.\n - Carry the w1 empty-string->'' contract to the remaining raw-CSV adapters.\n w1 fixed the Parquet path (strings_can_be_null=False) and the pandas CSV\n path; Polars scan_csv / modin / dask / cudf still map an empty string\n field to null on their raw-CSV read, so prefer_parquet=False (or a\n CSV-recommending adapter) reintroduces the Q6/Q17/Q18 divergence on those\n surfaces. Thread declared column types into each adapter's read_csv and\n preserve '' for string columns, mirroring pandas_df.read_csv.\n - Harden schema_utils.column_name/column_sql_type (schema_utils.py:29-51):\n replace the naive column.split() with a paren-aware, quote-aware DDL\n column parser that handles DECIMAL(10, 2) (space after comma), quoted/\n spaced names, and multi-word types (DOUBLE PRECISION, TIMESTAMP WITH TIME\n ZONE, NOT NULL DEFAULT ...). Add unit tests for each shape.\n - Extend PYARROW_TYPE_MAP and the polars/pandas maps (data_loader.py:191-199)\n to cover TEXT, BIGINT, REAL, DOUBLE PRECISION, NUMERIC, BOOLEAN so declared\n string/numeric types are applied at conversion instead of inferred.\n - Apply declared STRING dtypes on the production load path (not just date-\n name suppression) so a leading-zero VARCHAR is not inferred as int.\nRe-verify the real DataFrame benchmark run still works (this is a loader fix,\nnot gate-only special-casing).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a production-loader-path gate cell + harden the DDL parser and type maps (M4, L1, L2)", "wid": "w6"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "Correct the docstrings that assert \"two independent implementations\"\n(cross_surface.py:3-6) and \"independent ... can diverge independently\"\n(dataframe_surface.py:48-50). For SSB and ~40/43 ClickBench queries BOTH\nbackends are generated from one DSL spec row, so they cannot diverge on logic\n- only on engine semantics; the only real independence is SQL-text vs DSL-spec\n(same author, same understanding). State plainly that the oracle catches\nTRANSCRIPTION errors, not CONCEPTUAL ones, for DSL-generated benchmarks.\nRecord per gated benchmark whether its DataFrame surface is hand-written\n(coffeeshop) or generated (ssb, clickbench). Optional but recommended: for at\nleast the enforced gates, ensure one surface is independently authored, or add\na second independent reference (e.g. the upstream published ClickBench answer\nset) so the oracle is not just self-consistency.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop overclaiming independence; document what the oracle actually proves (BS1)", "wid": "w7"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "STAGED_GATES (clickbench, joinorder_synthetic) are not run in CI (pr.yml runs\nonly ssb + coffeeshop) and nothing fails when a staged gate is parked\nindefinitely; test_clickbench_is_staged_not_enforced only BLESSES parking.\n - Add the staged cross-surface reports to a non-blocking CI job so their\n divergence snapshots cannot silently rot (promote to blocking once green\n via w5).\n - Add a governance test: a benchmark may sit in STAGED_GATES only with a\n dated rationale and an owner; fail if a staged gate has been parked past a\n declared review-by date. Delete a staged gate that no one will burn down\n rather than leave a permanent red parking lot.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run staged gates in CI and add a promotion forcing-function (M3)", "wid": "w8"}, {"evidence": null, "item_id": "cross-surface-oracle-remediation", "needs": [], "notes": "- generate_oracle_coverage_map.py derives 'guarded' from GATES dict membership\n (85/111) and never checks the gate is green. Either run each gate (or read\n its CI result) and add a distinct 'verified-green' signal, or rename\n 'guarded' -> 'gate-registered' so a registered-but-unverified gate is not\n reported as coverage.\n- cross_surface_applicability_sweep.py counts 6 'needs-id-mapping' benchmarks\n (amplab, datavault, nyctaxi, tpcds_obt, tpch_skew, tsbs_devops) as 'gateable'\n on ZERO verified id overlap; the campaign's own TODO warns 'do NOT guess'\n the nyctaxi/tsbs mappings and tpcds_obt is 3 DF queries vs 89 SQL. Downgrade\n these from 'gateable' to 'candidate-unverified' until an independent\n SQL<->DF id mapping is confirmed per benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Tighten coverage-map / applicability honesty (M1, M2)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "implement BigQuery CTAS sort", "instead": "use CLUSTER BY", "why": "BigQuery does not preserve row order and the sorted output would revert to arbitrary order on subsequent DML"}, {"dont": "implement CTAS sort for Snowflake on top of existing clustering", "instead": "if research confirms CTAS adds no value, return None", "why": "Snowflake's Automatic Clustering already handles physical data ordering and a CTAS would recreate the table without the clustering metadata"}, {"dont": "skip research units to save time", "instead": "validate physical sort behavior before coding any SQL", "why": "incorrect assumptions for cloud warehouses could produce silently incorrect benchmark data"}, {"dont": "use CTAS for Databricks/Delta Lake", "instead": "the correct mechanism is OPTIMIZE ZORDER BY", "why": "CTAS creates an unmanaged table losing Delta transactional metadata"}], "approach": "Research units (w1, w3, w5, w7, w9) can all run in parallel - they are read-only.\nImplementation follows research findings. Where the answer is \"unsupported / return None\",\nthe implementation is 5-10 lines (override + docstring); prioritize getting the documentation\nright over speed.\n\nFor cloud platforms that are unsupported, rely on the existing base-class skip logging from\nctas-sort-dataloader-connectivity. If gaps are discovered, capture them as a follow-up TODO\ninstead of modifying base adapter code in this item.\n\nDo not attempt live platform tests - unit tests with mocked connections are sufficient\nfor verifying SQL generation. The verification commands use only unit tests.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-19T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": ["ctas-sort-dataloader-connectivity"], "description": "With the centralized apply_ctas_sort() in PlatformAdapter (ctas-sort-dataloader-connectivity),\ncloud data warehouse platforms gain sorted-data loading by overriding only\n_build_ctas_sort_sql(table_name, sort_columns). This item implements those overrides.\n\nEach cloud platform has a distinct approach to physical data ordering:\n\nSnowflake\n Snowflake already applies SORT configuration via clustering keys in apply_unified_tuning().\n A post-load CTAS (CREATE OR REPLACE TABLE t AS SELECT * FROM t ORDER BY col) will\n re-materialize data sorted but without Automatic Clustering - this may be counterproductive.\n Research whether post-load CTAS sort is beneficial in addition to or instead of clustering.\n If clustering already covers the use case, _build_ctas_sort_sql should return None and the\n sort should remain in apply_unified_tuning() / apply_table_tunings().\n\nDatabricks / Delta Lake\n Delta Lake does not guarantee physical sort order on disk via CTAS ORDER BY.\n Physical ordering is achieved post-load via OPTIMIZE ... ZORDER BY (Z-ordering) or\n OPTIMIZE ... with liquid clustering. The _build_ctas_sort_sql hook is not the right\n mechanism; instead, an apply_post_load_optimize() pattern may be needed, or the existing\n Z_ORDERING TuningType support can be extended. Return None from _build_ctas_sort_sql\n and document the correct path.\n\nBigQuery\n BigQuery CTAS supports ORDER BY (CREATE OR REPLACE TABLE ... AS SELECT * FROM ... ORDER BY).\n However, BigQuery is distributed and row order is not preserved after DML, so a post-load\n CTAS sort does not produce persistent physical ordering. Clustering keys (via partition/\n cluster config) are the correct mechanism. Return None and document.\n\nRedshift\n Amazon Redshift supports SORT KEYS defined at table creation. A post-load CTAS\n (CREATE TABLE t_sorted AS SELECT * FROM t ORDER BY col) followed by DROP+RENAME is\n valid but loses distribution style and sort key metadata. The correct approach is to\n define the SORT KEY in CREATE TABLE DDL. Investigate whether a VACUUM SORT ONLY t\n command (which re-sorts in-place) is a better post-load mechanism than CTAS.\n\nClickHouse / ClickHouse Cloud\n ClickHouse table ORDER BY is defined immutably at CREATE TABLE time (ENGINE = MergeTree\n ORDER BY col). Post-load CTAS sort is not applicable. Return None from\n _build_ctas_sort_sql and document. The correct path is to include ORDER BY in CREATE\n TABLE DDL generation - this may require a separate tuning TODO for ClickHouse DDL.\n\nAthena\n Athena (Presto/Trino-based) queries S3 directly. CTAS (CREATE TABLE ... AS SELECT ...\n ORDER BY) writes new Parquet files to S3. This IS meaningful for query performance if\n the output is partitioned or sorted - but Athena's distributed nature means ORDER BY in\n CTAS only produces local file ordering, not global ordering across the dataset. Research\n whether this provides zone-map-equivalent benefits for Athena queries.\n\nAzure Synapse\n Azure Synapse Analytics supports CTAS (CREATE TABLE ... AS SELECT ... ORDER BY).\n Physical ordering depends on the distribution style (hash, round-robin, replicated).\n Investigate whether CTAS ORDER BY produces useful clustering for Synapse's columnar storage.", "id": "ctas-sort-cloud-platforms", "preserves": ["All existing cloud platform tests pass (no credential mocks broken)", "Databricks Z_ORDERING TuningType support in apply_unified_tuning() is not affected", "Platforms returning None from _build_ctas_sort_sql produce no SQL execution side effects", "Snowflake clustering key application in apply_unified_tuning() is not removed or bypassed"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/motherduck.py"}], "state": "done", "title": "Implement _build_ctas_sort_sql for cloud platforms (Snowflake, Databricks, BigQuery, Redshift, ClickHouse, Athena, Azure Synapse)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_cloud_platform_ctas_sort.py -v --tb=short", "description": "Cloud platform CTAS sort unit tests pass (no live credentials needed)", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-cloud-platforms", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing cloud platform tests unaffected", "expected": "0 failures", "item_id": "ctas-sort-cloud-platforms", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check . && uv run ty check", "description": "Lint and type check pass", "expected": "no errors", "item_id": "ctas-sort-cloud-platforms", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Snowflake apply_table_tunings() already handles TuningType.SORTING via clustering keys\n(benchbox/platforms/snowflake.py:~1570). Determine:\n- Does CTAS ORDER BY produce sorted micro-partitions independent of clustering?\n- Does it conflict with Automatic Clustering?\n- Is there a benchmark scenario where CTAS sort is better than ALTER TABLE CLUSTER BY?\nExpected outcome: return None (clustering already covers this) with doc comment.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Snowflake: does post-load CTAS ORDER BY add value beyond clustering keys?", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement AzureSynapseAdapter._build_ctas_sort_sql or document as unsupported", "wid": "w10"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "ClickHouse MergeTree ENGINE requires ORDER BY at CREATE TABLE time. There is no\npost-load CTAS that changes physical ordering. Override _build_ctas_sort_sql in\nClickHouseAdapter to return None with a doc comment. Create a follow-up note\n(or separate TODO) for ensuring the ORDER BY clause from TuningType.SORTING is\nincluded in ClickHouse CREATE TABLE DDL generation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document ClickHouse Cloud as intentionally unsupported via _build_ctas_sort_sql = None", "wid": "w11"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "BigQuery does not preserve row order. Override to return None with a comment directing\nusers to BigQuery clustering (CLUSTER BY) configuration which is separate from SORTING.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document BigQuery as intentionally unsupported via _build_ctas_sort_sql = None", "wid": "w12"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w10", "w11", "w12", "w2", "w4", "w6", "w8"], "notes": "Each platform test: mock connection, call apply_ctas_sort with a tuning_config\ncontaining sort columns, assert correct SQL generated or assert returns False with\nno SQL executed for unsupported platforms. Tests should not require live cloud credentials.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests: verify each cloud platform returns expected SQL or None", "wid": "w13"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w1"], "notes": "If research shows CTAS ORDER BY is beneficial: return\n CREATE OR REPLACE TABLE {t} AS SELECT * FROM {t} ORDER BY col;\nIf clustering already covers it: return None with log_debug message directing users\nto the clustering key configuration instead.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement SnowflakeAdapter._build_ctas_sort_sql based on w1 research", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Delta Lake physical ordering mechanisms:\n- OPTIMIZE t ZORDER BY col (multi-dimensional Z-order clustering)\n- OPTIMIZE t (liquid clustering if enabled)\n- CTAS ORDER BY (creates new unmanaged table, loses Delta metadata)\nConfirm that _build_ctas_sort_sql returning None is correct; document the right\npath for Databricks (likely a separate apply_post_load_optimize hook or extending\nthe existing Z_ORDERING TuningType support in apply_unified_tuning()).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Databricks/Delta Lake post-load sort mechanisms (OPTIMIZE ZORDER BY vs CTAS)", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w3"], "notes": "In DatabricksAdapter (or pyspark adapter), override _build_ctas_sort_sql to return None\nwith an explanatory comment. If the existing Z_ORDERING TuningType handling in\napply_unified_tuning() already covers physical sort, confirm and document the connection.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document Databricks unsupported CTAS sort; note correct Z-ordering path in code", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Redshift options:\n- VACUUM SORT ONLY table_name - re-sorts in-place using the table's defined SORT KEY.\n Requires SORT KEY to be defined in CREATE TABLE. No DROP/RENAME needed.\n- CTAS+DROP+RENAME - loses SORT KEY definition and distribution style.\nIf VACUUM SORT ONLY is viable: implement _build_ctas_sort_sql to return\n \"VACUUM SORT ONLY {table_name};\" (unconventional use of the hook, but works).\nIf not viable: return None and document.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Redshift: VACUUM SORT ONLY vs CTAS+DROP+RENAME for post-load sorting", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement RedshiftAdapter._build_ctas_sort_sql based on w5 research", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Athena CTAS writes Parquet files to S3. Key question: does ORDER BY in Athena CTAS\nproduce single sorted Parquet files, or distributed unsorted output across multiple\nfiles? Row group-level min/max statistics in Parquet enable Athena's predicate pushdown\n(equivalent to DuckDB zone maps). If ORDER BY produces sorted row groups within files,\nthis is beneficial.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Athena CTAS ORDER BY: does it produce zone-map-equivalent file sorting?", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement AthenaAdapter._build_ctas_sort_sql or document as unsupported", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sort-cloud-platforms", "needs": [], "notes": "Synapse columnar store uses columnstore indexes for compression and pushdown - physical\nrow order within a distribution affects rowgroup quality. CTAS ORDER BY may improve\nrowgroup delta store promotion. Verify against Synapse docs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research Azure Synapse CTAS ORDER BY effectiveness with hash/round-robin distribution", "wid": "w9"}], "worktree": "physical-tuning"} +{"anti_patterns": [{"dont": "make _build_ctas_sort_sql abstract", "instead": "return None by default (opt-in model)", "why": "not all platforms support CTAS sort and forcing every subclass to implement it would break the many adapters that don't need it"}, {"dont": "move the validate_sql_identifier call into _build_ctas_sort_sql", "instead": "keep it in the base apply_ctas_sort() before calling the hook", "why": "identifier validation is a shared safety concern, not platform-specific"}, {"dont": "pass DuckDBDDLGenerator into _build_ctas_sort_sql", "instead": "each platform builds its own SQL string directly", "why": "the hook must be self-contained per platform"}, {"dont": "rename generate_ctas_ddl parameter without grep-checking all callers first", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Start with w1 (add the hook method signature to base adapter) before touching any\nadapters, so the interface is settled before implementations follow.\n\nFor w2, keep the case-insensitive lookup and validate_sql_identifier call from the existing\nDuckDBAdapter.apply_ctas_sort() - both are correct and should move verbatim.\n\nFor w3, DuckDBAdapter._build_ctas_sort_sql builds the ORDER BY string from sort_columns\nwithout involving DuckDBDDLGenerator - just use f-string formatting directly since the\nbase class owns the logic that already extracted the columns. Only the SQL template string\nbelongs in the platform-specific hook.\n\nFor w4 (parameter rename), run grep first:\n grep -rn \"generate_ctas_ddl\" benchbox/ tests/\nto find all callers before renaming.\n\nFor w8 (MotherDuck), add the apply_ctas_sort call after each table's INSERT INTO completes\ninside MotherDuck's existing load_data() loop - mirror the DataLoader pattern.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-19T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": ["ctas-sorted-data-loading"], "description": "The initial CTAS sort implementation (ctas-sorted-data-loading) placed apply_ctas_sort()\ndirectly on DuckDBAdapter and used a duck-typed hasattr() check in DataLoader. This works\nfor DuckDB today, but extending to additional platforms would require copy-pasting the common\nlookup/validation/logging logic into each adapter.\n\nThis item centralizes apply_ctas_sort() in PlatformAdapter (base class) so that all common\nbehavior - case-insensitive table lookup, sort column extraction, validate_sql_identifier call,\ndry-run capture_sql, success logging, and debug logging on skipped paths - lives in exactly\none place. Each platform adapter overrides only the SQL generation hook\n_build_ctas_sort_sql(table_name, sort_columns) -> str | None, returning None if the platform\ndoes not support CTAS-based sorting.\n\nSecondary items in scope:\n- Wire tuning_config into the DataLoader calls for ClickHouse, StarRocks, SQLite, and\n MotherDuck (currently they construct DataLoader without tuning_config, so CTAS sort\n never fires even when tuning is enabled).\n- Rename the misleadingly named if_not_exists parameter in DuckDBDDLGenerator.generate_ctas_ddl()\n to or_replace (the parameter generates CREATE OR REPLACE, not IF NOT EXISTS).\n- Remove or simplify the hasattr(adapter, \"apply_ctas_sort\") guard in DataLoader now that\n the method is guaranteed by the base class; keep the tuning_config None check as the opt-in.\n- Add debug-level logging on all early-return paths in the base apply_ctas_sort() so\n operators can understand why sorting was skipped.", "id": "ctas-sort-dataloader-connectivity", "preserves": ["All existing test_duckdb_sorted_loading.py tests pass unchanged or are migrated 1-for-1", "DataLoader INSERT INTO path (tuning_config=None) is completely unaffected", "DuckDBAdapter.load_data() produces identical row counts after refactor", "DuckDBDDLGenerator.generate_ctas_ddl() behavioral output is identical after the parameter rename", "apply_ctas_sort returns False (not exception) when platform returns None from _build_ctas_sort_sql"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/duckdb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/workload.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/motherduck.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/sqlite.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/starrocks/workload.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter_ctas_sort.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_duckdb_sorted_loading.py"}], "state": "done", "title": "Centralize apply_ctas_sort in PlatformAdapter base class and wire all DataLoader callers", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_duckdb_sorted_loading.py -v --tb=short", "description": "All existing DuckDB sorted loading tests pass after refactor", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_base_adapter_ctas_sort.py -v --tb=short", "description": "New base adapter CTAS tests pass", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Full platform unit test suite passes", "expected": "0 failures", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run ruff check . && uv run ty check", "description": "Lint and type check pass", "expected": "no errors", "item_id": "ctas-sort-dataloader-connectivity", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": [], "notes": "Signature: def _build_ctas_sort_sql(self, table_name: str, sort_columns: list[TuningColumn]) -> str | None\nDefault implementation returns None (platform does not support CTAS sort).\nPlace near apply_unified_tuning() in benchbox/platforms/base/adapter.py.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add _build_ctas_sort_sql(table_name, sort_columns) hook to PlatformAdapter base class returning None", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w9"], "notes": "DataLoader docstring: explain that adapters control CTAS sort via _build_ctas_sort_sql().\nPlatformAdapter: document both apply_ctas_sort() (call from DataLoader/load_data) and\n_build_ctas_sort_sql() (override in subclasses to enable platform-specific sorting).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DataLoader and base adapter docstrings to document the hook protocol", "wid": "w10"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2", "w3", "w9"], "notes": "Move/adapt the TestApplyCtasSort and TestDataLoaderTuningConfig tests from\ntest_duckdb_sorted_loading.py so that base class behavior is tested through a\nminimal concrete subclass, and DuckDB-specific tests remain in the DuckDB file.\nAdd test: platform that returns None from _build_ctas_sort_sql \u2192 apply_ctas_sort returns False.\nAdd test: MotherDuck hook returns correct DuckDB SQL.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update / add tests for centralized apply_ctas_sort in base adapter", "wid": "w11"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w1"], "notes": "Move all logic from DuckDBAdapter.apply_ctas_sort() to PlatformAdapter:\n1. Case-insensitive lookup of table_name in tuning_config.table_tunings\n2. Extract sort columns via table_tuning.get_columns_by_type(TuningType.SORTING)\n3. If no sort columns: log_debug(\"No sort columns for {table} - skipping CTAS sort\") and return False\n4. Call self._build_ctas_sort_sql(validated_table, sort_columns)\n5. If None returned: log_debug(\"{platform} does not support CTAS sort - skipping\") and return False\n6. Dry-run path: self.capture_sql(sql, \"ctas_sort\", validated_table); return True\n7. Execute sql via connection\n8. self.log_verbose(\"Applied CTAS sorting to {table_name}: {col_names}\")\n9. Return True\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement centralized apply_ctas_sort() in PlatformAdapter with all common logic", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "DuckDB override returns:\n CREATE OR REPLACE TABLE {table_name} AS SELECT * FROM {table_name} ORDER BY col1, col2;\nBuild ORDER BY string from sort_columns sorted by col.order.\nRemove the now-redundant apply_ctas_sort() method from DuckDBAdapter entirely.\nMotherDuck can be added here as well since it shares DuckDB SQL syntax.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate DuckDBAdapter._build_ctas_sort_sql override; remove duplicated apply_ctas_sort", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w3"], "notes": "The parameter if_not_exists in benchbox/core/tuning/generators/duckdb.py:generate_ctas_ddl()\ngenerates CREATE OR REPLACE TABLE - the name is factually wrong and misleading.\nRename to or_replace. Update all remaining call sites that still use the generator\n(DuckDBAdapter._build_ctas_sort_sql should already be direct SQL by w3). Run grep to\nfind all usages before renaming.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rename generate_ctas_ddl(if_not_exists=...) parameter to or_replace in DuckDBDDLGenerator", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "benchbox/platforms/clickhouse/workload.py constructs DataLoader without tuning_config.\nAdd: tuning_config=self.unified_tuning_configuration if self.tuning_enabled else None\nClickHouse uses its own ORDER BY at table creation; _build_ctas_sort_sql returns None\n(deferred to ctas-sort-cloud-platforms) so this wiring is a no-op for now but enables\nfuture opt-in without another connectivity change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into ClickHouse DataLoader instantiation", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "benchbox/platforms/starrocks/workload.py - same pattern as ClickHouse.\nAdd tuning_config parameter so future _build_ctas_sort_sql can activate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into StarRocks DataLoader instantiation", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "benchbox/platforms/sqlite.py - same pattern.\nSQLite will likely return None from _build_ctas_sort_sql (no efficient CTAS sort path)\nbut the wiring must be present for the hook to have any effect.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into SQLite DataLoader instantiation", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w3"], "notes": "MotherDuck has its own load_data() using raw INSERT INTO SELECT * FROM read_parquet().\nAfter each table loads, add a call to self.apply_ctas_sort(table_name, effective_tuning, connection)\nwhen tuning_enabled. MotherDuck _build_ctas_sort_sql returns DuckDB-compatible SQL (same engine).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire tuning_config into MotherDuck load_data (custom loader, not DataLoader)", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sort-dataloader-connectivity", "needs": ["w2"], "notes": "In benchbox/platforms/base/data_loading.py the guard currently reads:\n if self.tuning_config and hasattr(self.adapter, \"apply_ctas_sort\"):\nSince apply_ctas_sort is now guaranteed on all PlatformAdapter subclasses, simplify to:\n if self.tuning_config:\nUpdate the docstring to remove duck-typing language; describe the base class hook protocol.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove hasattr(adapter, 'apply_ctas_sort') guard in DataLoader; keep tuning_config check", "wid": "w9"}], "worktree": "physical-tuning"} +{"anti_patterns": [{"dont": "assume DataFusion CTAS behavior matches DuckDB without verification", "instead": "verify that SELECT * FROM {table_name} is valid post-load before generating CTAS SQL", "why": "DataFusion's table registration model (external tables vs in-memory) differs"}, {"dont": "execute DROP TABLE without a preceding BEGIN transaction in PostgreSQL", "instead": "always wrap in a BEGIN/COMMIT block", "why": "a crash between DROP and RENAME would leave the table destroyed"}, {"dont": "skip the research work units (w1, w3, w5) to save time", "instead": "confirm physical sort behavior before coding", "why": "incorrect CTAS assumptions could silently produce unsorted data or corrupt the table"}], "approach": "Work research units (w1, w3, w5, w7) first in parallel - they are read-only and independent.\nThen implement based on findings (w2, w4, w6 depend on their research units).\nFor PostgreSQL, if the multi-statement list[str] return type is needed, extend apply_ctas_sort\nin the base class to handle it before implementing the PostgreSQL override.\n\nFor DataFusion, check the existing _load_table_parquet() and _load_table_csv() methods to\nunderstand how the table is registered - CTAS sort requires the table to exist as a\nnamed relation that can be self-joined in SELECT * FROM {table_name}.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-19T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": ["ctas-sort-dataloader-connectivity"], "description": "With the centralized apply_ctas_sort() in PlatformAdapter (ctas-sort-dataloader-connectivity),\neach local SQL platform needs only to override _build_ctas_sort_sql(table_name, sort_columns)\nto gain CTAS-based sorted loading. This item implements those overrides.\n\nPlatforms in scope (MotherDuck is handled in ctas-sort-dataloader-connectivity as it shares\nDuckDB's SQL syntax):\n\nDataFusion\n DataFusion's SQL dialect supports CREATE OR REPLACE TABLE ... AS SELECT ... ORDER BY ...\n (same pattern as DuckDB). Verify the exact syntax against DataFusion's SQL reference and\n implement the hook.\n\nPostgreSQL\n PostgreSQL does not support CREATE OR REPLACE TABLE ... AS SELECT. The closest equivalent is:\n CREATE TABLE {t}_sorted AS SELECT * FROM {t} ORDER BY col;\n DROP TABLE {t};\n ALTER TABLE {t}_sorted RENAME TO {t};\n This is safe within a single connection but requires the table to exist first (post-load).\n Wrap in a transaction to keep it atomic.\n\nStarRocks\n StarRocks supports CREATE TABLE ... AS SELECT (CTAS) with ORDER BY via its sort keys.\n However, StarRocks sorts are defined at table-creation time (via ORDER BY in CREATE TABLE),\n not on a post-load CTAS. Investigate whether a post-load CTAS reorders physical data or\n just creates an unsorted replica. If CTAS does not physically sort: return None and document\n as unsupported; the correct path for StarRocks is to define ORDER BY in CREATE TABLE DDL.\n\nSQLite\n SQLite has no efficient physical sort; INSERT ORDER determines storage order. A CTAS pattern\n (CREATE TABLE t2 AS SELECT * FROM t1 ORDER BY col; DROP TABLE t1; ALTER TABLE t2 RENAME TO t1)\n is syntactically valid but requires recreating indexes and constraints. Given SQLite's use\n as a development/testing platform (not production), assess whether the overhead is justified.\n If not: _build_ctas_sort_sql returns None with a logged debug message.", "id": "ctas-sort-local-sql-platforms", "preserves": ["All existing DataFusion, PostgreSQL, StarRocks, SQLite unit tests pass", "DataFusion load_data() produces identical row counts after adding _build_ctas_sort_sql", "PlatformAdapter.apply_ctas_sort remains backward-compatible for existing str/None hook returns while optionally supporting list[str]", "PostgreSQL COPY-based loading path is unmodified; only the post-load sort step is added", "SQLite DataLoader INSERT INTO path unchanged (CTAS sort returns None)", "StarRocks DataLoader INSERT INTO path unchanged if CTAS sort is determined unsupported"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/sqlite.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/starrocks/workload.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_local_platform_ctas_sort.py"}], "state": "done", "title": "Implement _build_ctas_sort_sql for local SQL platforms (DataFusion, PostgreSQL, StarRocks, SQLite)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_local_platform_ctas_sort.py -v --tb=short", "description": "Local SQL platform unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "ctas-sort-local-sql-platforms", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v --tb=short", "description": "Existing platform tests unaffected", "expected": "0 failures", "item_id": "ctas-sort-local-sql-platforms", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_local_platform_ctas_sort.py -k integration -v", "description": "DataFusion integration: sorted table row order matches sort key", "expected": "all tests pass", "item_id": "ctas-sort-local-sql-platforms", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "Check benchbox/platforms/datafusion.py and DataFusion SQL docs. Specifically verify:\n- Does DataFusion support CREATE OR REPLACE TABLE?\n- Does ORDER BY in CTAS physically sort the stored data?\n- What is the exact SQL for a registered in-memory or on-disk table?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research DataFusion CTAS sort syntax; verify CREATE OR REPLACE TABLE ... AS SELECT ... ORDER BY", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w1"], "notes": "If DataFusion supports CREATE OR REPLACE TABLE t AS SELECT * FROM t ORDER BY col:\n return f\"CREATE OR REPLACE TABLE {table_name} AS SELECT * FROM {table_name} ORDER BY {order_by};\"\nIf not, return None and document in code why DataFusion is unsupported.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DataFusionAdapter._build_ctas_sort_sql override", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "PostgreSQL post-load sort strategy:\n BEGIN;\n CREATE TABLE {t}_ctas_sort AS SELECT * FROM {t} ORDER BY col;\n DROP TABLE {t};\n ALTER TABLE {t}_ctas_sort RENAME TO {t};\n COMMIT;\nVerify: does this preserve indexes/constraints created by schema setup? Likely not -\nthe DROP removes the table and its constraints. The base class apply_ctas_sort runs\nafter data load but the schema (CREATE TABLE + indexes) was applied earlier in the\nrun. Determine whether constraints must be re-applied or if COPY-based loading makes\nthem unnecessary for benchmark purposes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research PostgreSQL DROP+CTAS sort pattern; assess transaction safety", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w3"], "notes": "Return a multi-statement SQL string (or list[str]) for the DROP+CTAS pattern.\nThe base class apply_ctas_sort must handle list[str] (execute each statement in order).\nIf apply_ctas_sort currently only handles str, extend it to handle list[str] as well.\nAlternatively, wrap all statements in a DO $$ ... $$ block if the connection supports it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement PostgreSQLAdapter._build_ctas_sort_sql", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "StarRocks uses a Duplicate Key / Sort Key model defined at table creation.\nKey questions:\n- Does CREATE TABLE t2 AS SELECT * FROM t1 ORDER BY col in StarRocks physically sort\n the output table's tablets?\n- Or is the ORDER BY hint advisory only?\nReference: StarRocks docs on CTAS, sort key behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research StarRocks CTAS sort - does post-load CTAS physically reorder data?", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w5"], "notes": "If CTAS physically sorts: implement override returning:\n CREATE TABLE {t}_sorted AS SELECT * FROM {t} ORDER BY col;\n (+ DROP+RENAME steps)\nIf CTAS does NOT physically sort, return None and add a log message explaining that\nStarRocks sort keys must be defined in CREATE TABLE DDL (pre-load, not post-load).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement StarRocksAdapter._build_ctas_sort_sql or document as unsupported", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": [], "notes": "SQLite CTAS sort requires DROP + RENAME which loses indexes/constraints, same as PG.\nGiven SQLite is a dev/testing platform (not production performance target), the cost\nlikely outweighs the benefit. Decision: return None with log_debug(\n\"SQLite CTAS sort skipped - physical sort not beneficial for testing platform\").\nDocument the decision in code comments so future maintainers understand it was deliberate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Determine SQLite stance: implement trivial CTAS sort or document as intentionally unsupported", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w2", "w4", "w6", "w7"], "notes": "Each platform test verifies:\n- Correct SQL returned when sort columns are present\n- None returned (or correct skip) when platform doesn't support it\n- SQL contains correct column ordering (sorted by TuningColumn.order)\nFor PostgreSQL multi-statement path: verify all statements are returned and in order.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for DataFusion, PostgreSQL, StarRocks, SQLite _build_ctas_sort_sql", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sort-local-sql-platforms", "needs": ["w2"], "notes": "Mirror tests/unit/platforms/test_duckdb_sorted_loading.py::TestCtasSortIntegration\nfor DataFusion: load a small table, apply CTAS sort, verify row order matches sort columns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration test: DataFusion actual sortedness after load with tuning config", "wid": "w9"}], "worktree": "physical-tuning"} +{"anti_patterns": [{"dont": "modify DataLoader.load() default behavior", "instead": "add CTAS as an opt-in path triggered by tuning config", "why": "existing platforms depend on INSERT INTO"}, {"dont": "remove INSERT INTO fallback", "instead": "keep both paths", "why": "not all platforms support CTAS"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-01-07T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox's tuning infrastructure generates sort clauses via DuckDBDDLGenerator\n(e.g., `ORDER BY L_SHIPDATE, L_ORDERKEY`), but the data loader uses INSERT INTO\nstatements that don't preserve sorting.\n\nDuckDB doesn't support inline ORDER BY in CREATE TABLE. Sorted tables must be\ncreated via CTAS patterns:\n CREATE TABLE sorted_table AS SELECT * FROM source ORDER BY col;\n\nThe current DataLoader uses INSERT INTO with VALUES, which doesn't apply sorting.\nThis gap was discovered during sorted data performance experiments showing 5-28x\nspeedups for selective queries when data is properly sorted (zone map pruning).\n\nThe fix requires modifying the data loading path to:\n1. Detect when tuning config specifies sorting for a table\n2. Use CTAS pattern instead of INSERT INTO for those tables\n3. Support both fresh load (CTAS) and append scenarios", "id": "ctas-sorted-data-loading", "preserves": ["DuckDB adapter backward compatible when no tuning config provided", "Existing INSERT INTO loading path unchanged for non-sorted tables", "benchbox run --platform duckdb --benchmark tpch --scale 0.01 produces identical row counts"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Implement CTAS-based sorted data loading for DuckDB", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_duckdb*.py -v --tb=short", "description": "Existing DuckDB tests still pass", "expected": "all tests pass, 0 failures", "item_id": "ctas-sorted-data-loading", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'sorted or ctas' -v --tb=short", "description": "Row counts match after CTAS vs INSERT INTO loading", "expected": "all tests pass", "item_id": "ctas-sorted-data-loading", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document current data loading flow in DataLoader class", "wid": "w1"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests for sorted CTAS loading", "wid": "w10"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration test: verify sortedness after load", "wid": "w11"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Performance validation: confirm zone map pruning benefit", "wid": "w12"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Identify integration point for tuning config in load path", "wid": "w2"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design CTAS-based loading strategy for DuckDB", "wid": "w3"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle fresh load scenario (CTAS)", "wid": "w4"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Handle append scenario (temp table + ordered insert)", "wid": "w5"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tuning_config parameter to DataLoader", "wid": "w6"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DuckDB CTAS handler for sorted tables", "wid": "w7"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Modify DuckDB adapter to pass tuning config to loader", "wid": "w8"}, {"evidence": null, "item_id": "ctas-sorted-data-loading", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add generate_ctas_ddl usage in data loading path", "wid": "w9"}], "worktree": "physical-tuning"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": "2025-11-28T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Trim unrelated worktree changes, craft a focused release branch, and produce a user-facing changelog summary.", "id": "curate-release-branch-and-changelog", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Curate Release Branch and Changelog", "verifications": [], "work": [], "worktree": "release-readiness"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-03T00:00:00Z", "deferrals": [], "deps": [], "description": "Dask runs fail on all 22 TPC-H queries due to API incompatibilities between\nDask and Pandas. The root cause is that pandas_impl query functions call\n.groupby(as_index=False) directly, which Dask doesn't support.\n\nErrors by category:\n- as_index=False not supported: Q1-Q5, Q10-Q13, Q15-Q18, Q20, Q22 (16 queries)\n- .dt.year on DatetimeIndex: Q7, Q8, Q9 (3 queries)\n- nunique aggregate: Q16, Q21 (2 queries)\n- Lazy scalar conversion: Q6, Q14, Q19 (3 queries)\n\nSolution: Created UnifiedPandasFrame wrapper (following Expression family pattern)\nthat intercepts DataFrame operations and routes to adapter for platform-specific\nhandling. Also made minor query adaptations to handle lazy scalar computations.", "id": "dask-platform-compatibility", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Fix Dask DataFrame Platform Compatibility", "verifications": [], "work": [], "worktree": "dataframe-expansion"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change Q10 semantics", "why": "the rework returns the identical result set."}, {"INSTEAD": "(not specified)", "dont": "remove the generic _run_with_resource_diagnostics worker-death catch", "why": "only the Q10 pre-emptive skip is obsolete."}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-30T00:00:00Z", "completed_pr": null, "created_at": "2026-05-30T00:00:00Z", "deferrals": [], "deps": [], "description": "dask-df pre-emptively SKIPPED TPC-H Q10 in the default local envelope via a\nresource guard (benchbox/platforms/dataframe/dask_df.py:\n_RESOURCE_ENVELOPE_SKIP_QUERY_IDS = {\"Q10\"}, _guarded_resource_query_diagnostic,\n_collect_skip_query_ids override). The guard's premise was that Q10's\nPandas-family Dask graph terminates the parent process with exit 137 (OOM) on\nlocal runs.\n\nInvestigation on 2026-05-29/30 (during uat-dask-stability-under-sweep) showed\nthe guard was OBSOLETE:\n\n- Dask 2026.3.0 makes the dask-expr query optimizer MANDATORY. Disabling it\n raises NotImplementedError (\"The legacy implementation is no longer\n supported\"). The optimizer auto-applies projection + filter pushdown, so the\n naive Q10 graph that OOMed on legacy Dask no longer materializes.\n- The unmodified q10_pandas_impl ran within the 2GB/1-worker/spill local\n envelope at SF1 in 2.6s (parquet) and 13.9s (CSV, the real benchbox IO),\n producing 37,967 groups \u2014 no OOM, no exit 137.\n- A projection-pushdown rework (select needed columns, push the\n l_returnflag='R' filter before the lineitem join, group by the narrow\n integer keys c_custkey/c_nationkey then rejoin detail columns) is byte-\n identical to the current result (same all_hash, same top-20) and also fits.", "id": "dask-q10-guard-staleness-and-projection-rework", "preserves": ["All other dask-df TPC-H cells that pass must not regress.", "Q10 result is identical (same rows) before and after the rework.", "The generic worker-death -> DaskResourceEnvelopeError -> FAIL safety net stays (only the Q10 pre-emptive skip is removed)."], "prior_art": [{"concept": "benchbox/core/tpch/dataframe_queries.py:q10_pandas_impl \u2014 reworked with projection pushdown; q10_expression_impl unchanged", "decision": "reuse", "path": "benchbox/core/tpch/dataframe_queries.py:q10_pandas_impl"}, {"concept": "benchbox/platforms/dataframe/dask_df.py: removed _RESOURCE_ENVELOPE_SKIP_QUERY_IDS / _guarded_resource_query_diagnostic / _collect_skip_query_ids override / _is_local_envelope / _tpch_q10_resource_envelope_diagnostic", "decision": "reuse", "path": "benchbox/platforms/dataframe/dask_df.py"}, {"concept": "tests/unit/platforms/dataframe/test_dask_df_coverage.py \u2014 replaced the Q10-guard tests with test_q10_not_preemptively_skipped_on_dask_local_envelope", "decision": "reuse", "path": "tests/unit/platforms/dataframe/test_dask_df_coverage.py"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpch/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Remove the obsolete dask TPC-H Q10 skip guard and apply a projection-pushdown rework", "verifications": [{"command": "uv run -- benchbox run --platform dask-df --benchmark tpch --scale 0.1 --queries Q10 --non-interactive --phases load,power", "description": "dask-df Q10 runs and returns rows (no pre-emptive skip)", "expected": "Q10 executes (not SKIPPED) and returns rows; no exit 137", "item_id": "dask-q10-guard-staleness-and-projection-rework", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/test_dask_df_coverage.py -q", "description": "dask coverage tests pass", "expected": "all pass", "item_id": "dask-q10-guard-staleness-and-projection-rework", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": "SF1 (= scale 1.0) validated via the real adapter IO paths: parquet 2.6s,\nCSV 13.9s, 37,967 groups, no exit 137. Live dask-df Q10 @0.1 after guard\nremoval: Q10 executed (not skipped), 1.3s, 100% success. 0.01 is trivially\nwithin envelope.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate: confirm unmodified Q10 runs within the default local envelope at 0.01/0.1/1 with the guard bypassed", "wid": "w0"}, {"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": "Byte-identical to the prior output (verified: same all_hash, same top-20,\n37,967 groups). 178 tests/unit/core/tpch dataframe/pandas tests pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the projection-pushdown rework to q10_pandas_impl", "wid": "w1"}, {"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": "Removed the pre-emptive guard: execute_query/execute_query_profiled\noverrides, _guarded_resource_query_diagnostic, _collect_skip_query_ids /\n_build_skipped_results overrides, _is_local_envelope,\n_has_explicit_local_resource_settings, _is_tpch_benchmark,\n_tpch_q10_resource_envelope_diagnostic, and the guard constants. KEPT the\ngeneric worker-death safety net (_run_with_resource_diagnostics,\n_resource_envelope_diagnostic, _RESOURCE_ENVELOPE_PATTERNS,\nDaskResourceEnvelopeError) and the memory/worker caps.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove/relax the dask Q10 skip guard so Q10 runs in the default local envelope", "wid": "w2"}, {"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests: drop the Q10-guard tests, add coverage that Q10 is no longer skipped", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "fix this only in joinorder", "why": "data_fetch is shared by future real-data benchmarks"}, {"INSTEAD": "(not specified)", "dont": "remove checksum verification to make concurrent downloads appear to pass", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "write directly to the shared final archive path from multiple processes", "why": "(unstated)"}], "approach": "First make the critical section explicit: only one process should decide and\npopulate a shared archive path at a time. Then switch the downloader to a\nprocess-unique temporary path plus atomic replace, so the final path is\neither absent or checksum-verified. Keep the implementation generic in\n`data_fetch`, not JoinOrder-specific.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Housekeeping for the JoinOrder canonical cutover audited\n`benchbox/core/data_fetch/` and found that archive downloads are not guarded\nagainst concurrent callers writing the same destination file.\n\n`fetch_data()` resolves the archive path under the benchmark output\ndirectory and calls `download(manifest.url, archive_path, ...)` when table\nfiles are missing. `download()` writes directly to `dest_path`, resumes from\nexisting bytes with a Range request, and verifies the final checksum. There is\nno per-archive lock, no unique `.part` path per process, and no atomic\n`os.replace()` from a completed temp file. Two processes that start with a\nmissing archive can therefore append/truncate the same file concurrently and\neither corrupt the archive or waste a large download before checksum failure.\n\nThis affects every benchmark that uses `benchbox/core/data_fetch/`, not just\nJoinOrder, so the housekeeping TODO deliberately records it as a separate\nfollow-up instead of fixing it inline.\n", "id": "data-fetch-concurrent-download-race", "preserves": ["Downloader test seams remain mockable without live network access", "Existing checksum mismatch errors keep surfacing expected and actual sha256 values", "Pre-populated verified table files still skip network access"], "prior_art": [{"concept": "benchbox/core/data_fetch/downloader.py:download \u2014 current direct-to-destination streaming implementation to harden", "decision": "reuse", "path": "benchbox/core/data_fetch/downloader.py:download"}, {"concept": "benchbox/core/data_fetch/manager.py:fetch_data \u2014 current archive path resolution and checksum verification path to extend", "decision": "extend", "path": "benchbox/core/data_fetch/manager.py:fetch_data"}, {"concept": "tests/conftest.py test lock \u2014 existing interprocess lock pattern to consider before adding a new dependency", "decision": "reuse", "path": "tests/conftest.py"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/data_fetch/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/data_fetch/"}], "state": "active", "title": "Make real-data archive downloads safe under concurrent fetch_data callers", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/data_fetch -q", "description": "Concurrent fetch_data callers cannot corrupt the final archive", "expected": "data_fetch concurrency tests pass", "item_id": "data-fetch-concurrent-download-race", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "data-fetch-concurrent-download-race", "needs": [], "notes": "Use an existing project locking pattern if one exists. The lock should\ncover the check-download-verify handoff so a second process reuses the\nverified archive after the first completes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a per-archive interprocess lock around download/extract verification", "wid": "w1"}, {"evidence": null, "item_id": "data-fetch-concurrent-download-race", "needs": ["w1"], "notes": "Avoid writing directly to the shared final path. Preserve the current\nchecksum failure diagnostics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Download to a process-unique temp file and atomically replace the archive only after checksum verification", "wid": "w2"}, {"evidence": null, "item_id": "data-fetch-concurrent-download-race", "needs": ["w1", "w2"], "notes": "Tests should prove one completed archive is reused and no caller observes\na partially written final archive.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add concurrency tests for two fetch_data callers targeting the same missing archive", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change Q10 semantics", "instead": "(not specified)", "why": "the rework returns the identical result set."}, {"dont": "remove the generic _run_with_resource_diagnostics worker-death catch", "instead": "(not specified)", "why": "only the Q10 pre-emptive skip is obsolete."}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-30T00:00:00Z", "completed_pr": null, "created_at": "2026-05-30T00:00:00Z", "deferrals": [], "deps": [], "description": "dask-df pre-emptively SKIPPED TPC-H Q10 in the default local envelope via a\nresource guard (benchbox/platforms/dataframe/dask_df.py:\n_RESOURCE_ENVELOPE_SKIP_QUERY_IDS = {\"Q10\"}, _guarded_resource_query_diagnostic,\n_collect_skip_query_ids override). The guard's premise was that Q10's\nPandas-family Dask graph terminates the parent process with exit 137 (OOM) on\nlocal runs.\n\nInvestigation on 2026-05-29/30 (during uat-dask-stability-under-sweep) showed\nthe guard was OBSOLETE:\n\n- Dask 2026.3.0 makes the dask-expr query optimizer MANDATORY. Disabling it\n raises NotImplementedError (\"The legacy implementation is no longer\n supported\"). The optimizer auto-applies projection + filter pushdown, so the\n naive Q10 graph that OOMed on legacy Dask no longer materializes.\n- The unmodified q10_pandas_impl ran within the 2GB/1-worker/spill local\n envelope at SF1 in 2.6s (parquet) and 13.9s (CSV, the real benchbox IO),\n producing 37,967 groups \u2014 no OOM, no exit 137.\n- A projection-pushdown rework (select needed columns, push the\n l_returnflag='R' filter before the lineitem join, group by the narrow\n integer keys c_custkey/c_nationkey then rejoin detail columns) is byte-\n identical to the current result (same all_hash, same top-20) and also fits.", "id": "dask-q10-guard-staleness-and-projection-rework", "preserves": ["All other dask-df TPC-H cells that pass must not regress.", "Q10 result is identical (same rows) before and after the rework.", "The generic worker-death -> DaskResourceEnvelopeError -> FAIL safety net stays (only the Q10 pre-emptive skip is removed)."], "prior_art": [{"concept": "benchbox/core/tpch/dataframe_queries.py:q10_pandas_impl \u2014 reworked with projection pushdown; q10_expression_impl unchanged", "decision": "reuse", "path": "benchbox/core/tpch/dataframe_queries.py:q10_pandas_impl"}, {"concept": "benchbox/platforms/dataframe/dask_df.py: removed _RESOURCE_ENVELOPE_SKIP_QUERY_IDS / _guarded_resource_query_diagnostic / _collect_skip_query_ids override / _is_local_envelope / _tpch_q10_resource_envelope_diagnostic", "decision": "reuse", "path": "benchbox/platforms/dataframe/dask_df.py"}, {"concept": "tests/unit/platforms/dataframe/test_dask_df_coverage.py \u2014 replaced the Q10-guard tests with test_q10_not_preemptively_skipped_on_dask_local_envelope", "decision": "reuse", "path": "tests/unit/platforms/dataframe/test_dask_df_coverage.py"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpch/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Remove the obsolete dask TPC-H Q10 skip guard and apply a projection-pushdown rework", "verifications": [{"command": "uv run -- benchbox run --platform dask-df --benchmark tpch --scale 0.1 --queries Q10 --non-interactive --phases load,power", "description": "dask-df Q10 runs and returns rows (no pre-emptive skip)", "expected": "Q10 executes (not SKIPPED) and returns rows; no exit 137", "item_id": "dask-q10-guard-staleness-and-projection-rework", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/dataframe/test_dask_df_coverage.py -q", "description": "dask coverage tests pass", "expected": "all pass", "item_id": "dask-q10-guard-staleness-and-projection-rework", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": "SF1 (= scale 1.0) validated via the real adapter IO paths: parquet 2.6s,\nCSV 13.9s, 37,967 groups, no exit 137. Live dask-df Q10 @0.1 after guard\nremoval: Q10 executed (not skipped), 1.3s, 100% success. 0.01 is trivially\nwithin envelope.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate: confirm unmodified Q10 runs within the default local envelope at 0.01/0.1/1 with the guard bypassed", "wid": "w0"}, {"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": "Byte-identical to the prior output (verified: same all_hash, same top-20,\n37,967 groups). 178 tests/unit/core/tpch dataframe/pandas tests pass.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the projection-pushdown rework to q10_pandas_impl", "wid": "w1"}, {"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": "Removed the pre-emptive guard: execute_query/execute_query_profiled\noverrides, _guarded_resource_query_diagnostic, _collect_skip_query_ids /\n_build_skipped_results overrides, _is_local_envelope,\n_has_explicit_local_resource_settings, _is_tpch_benchmark,\n_tpch_q10_resource_envelope_diagnostic, and the guard constants. KEPT the\ngeneric worker-death safety net (_run_with_resource_diagnostics,\n_resource_envelope_diagnostic, _RESOURCE_ENVELOPE_PATTERNS,\nDaskResourceEnvelopeError) and the memory/worker caps.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove/relax the dask Q10 skip guard so Q10 runs in the default local envelope", "wid": "w2"}, {"evidence": null, "item_id": "dask-q10-guard-staleness-and-projection-rework", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests: drop the Q10-guard tests, add coverage that Q10 is no longer skipped", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fix this only in joinorder", "instead": "(not specified)", "why": "data_fetch is shared by future real-data benchmarks"}, {"dont": "remove checksum verification to make concurrent downloads appear to pass", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "write directly to the shared final archive path from multiple processes", "instead": "(not specified)", "why": "(unstated)"}], "approach": "First make the critical section explicit: only one process should decide and\npopulate a shared archive path at a time. Then switch the downloader to a\nprocess-unique temporary path plus atomic replace, so the final path is\neither absent or checksum-verified. Keep the implementation generic in\n`data_fetch`, not JoinOrder-specific.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Housekeeping for the JoinOrder canonical cutover audited\n`benchbox/core/data_fetch/` and found that archive downloads are not guarded\nagainst concurrent callers writing the same destination file.\n\n`fetch_data()` resolves the archive path under the benchmark output\ndirectory and calls `download(manifest.url, archive_path, ...)` when table\nfiles are missing. `download()` writes directly to `dest_path`, resumes from\nexisting bytes with a Range request, and verifies the final checksum. There is\nno per-archive lock, no unique `.part` path per process, and no atomic\n`os.replace()` from a completed temp file. Two processes that start with a\nmissing archive can therefore append/truncate the same file concurrently and\neither corrupt the archive or waste a large download before checksum failure.\n\nThis affects every benchmark that uses `benchbox/core/data_fetch/`, not just\nJoinOrder, so the housekeeping TODO deliberately records it as a separate\nfollow-up instead of fixing it inline.\n", "id": "data-fetch-concurrent-download-race", "preserves": ["Downloader test seams remain mockable without live network access", "Existing checksum mismatch errors keep surfacing expected and actual sha256 values", "Pre-populated verified table files still skip network access"], "prior_art": [{"concept": "benchbox/core/data_fetch/downloader.py:download \u2014 current direct-to-destination streaming implementation to harden", "decision": "reuse", "path": "benchbox/core/data_fetch/downloader.py:download"}, {"concept": "benchbox/core/data_fetch/manager.py:fetch_data \u2014 current archive path resolution and checksum verification path to extend", "decision": "extend", "path": "benchbox/core/data_fetch/manager.py:fetch_data"}, {"concept": "tests/conftest.py test lock \u2014 existing interprocess lock pattern to consider before adding a new dependency", "decision": "reuse", "path": "tests/conftest.py"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/data_fetch/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/data_fetch/"}], "state": "active", "title": "Make real-data archive downloads safe under concurrent fetch_data callers", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/data_fetch -q", "description": "Concurrent fetch_data callers cannot corrupt the final archive", "expected": "data_fetch concurrency tests pass", "item_id": "data-fetch-concurrent-download-race", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "data-fetch-concurrent-download-race", "needs": [], "notes": "Use an existing project locking pattern if one exists. The lock should\ncover the check-download-verify handoff so a second process reuses the\nverified archive after the first completes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a per-archive interprocess lock around download/extract verification", "wid": "w1"}, {"evidence": null, "item_id": "data-fetch-concurrent-download-race", "needs": ["w1"], "notes": "Avoid writing directly to the shared final path. Preserve the current\nchecksum failure diagnostics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Download to a process-unique temp file and atomically replace the archive only after checksum verification", "wid": "w2"}, {"evidence": null, "item_id": "data-fetch-concurrent-download-race", "needs": ["w1", "w2"], "notes": "Tests should prove one completed archive is reused and no caller observes\na partially written final archive.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add concurrency tests for two fetch_data callers targeting the same missing archive", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:37:03Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #583 of pg-duckdb-platform-adapter.\nDeferred: Data lake query support (S3 Parquet/Iceberg reads via pg_duckdb)\nReason deferred: Requires cloud storage credentials and Parquet file management - separate feature", "id": "data-lake-query-support-s3-parquet-iceberg-reads-via-pg-duck", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Data lake query support (S3 Parquet/Iceberg reads via pg_duckdb)", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"Added `benchbox/core/databases/manager.py` exposing `test_connection` using platform adapter + core platform config.\"\n- \"Updated `benchbox/cli/database.py` to call the core utility.\"\n- \"Added `tests/unit/core/test_database_manager.py` (core), and CLI boundary tests covered through delegation.\"", "id": "database-management-boundaries", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Database Management Boundaries", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: Completed \u2705\"\n- \"**Description**: Update Primitives and TPCHavoc benchmarks to reuse existing TPC-H databases when configuration matches, implementing the same validation checks that TPC-H uses\"\n- \"**Tasks**:\"\n- \"**Implementation Details**:\"\n- \"**Files Affected**: `benchbox/core/primitives/benchmark.py`, `benchbox/core/tpchavoc/benchmark.py`, `tests/unit/test_database_reuse_optimization.py`\"\n- \"**Impact**: Significantly reduces setup time for Primitives and TPCHavoc benchmarks when TPC-H database already exists with compatible configuration. Leverages existing TPC-H data generation work while maintaining validation integrity and configuration consistency.\"", "id": "database-reuse-optimization-for-primitives-and-tpchavoc-benchmarks", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Database Reuse Optimization for Primitives and TPCHavoc Benchmarks", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "\"Technical Debt & Refactoring\"", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Allow users to specify tuning/notuning mode for benchmark execution\"\n- \"**Tasks**:\"\n- \"**Files Affected**: CLI module, platform adapters, benchmark result classes\"\n- \"**Impact**: \u2705 Enables fair performance comparisons and allows users to test optimized vs baseline configurations\"", "id": "database-tuning-configuration-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Database Tuning Configuration \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "\"Technical Debt & Refactoring\"", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Comprehensive database validation and anonymized results collection\"\n- \"**Tasks**:\"\n- \"**Files Affected**: Platform adapters, result export classes, database validation logic, schema creation\"\n- \"**Impact**: \u2705 Ensures benchmark validity and enables anonymous performance data collection for analysis\"", "id": "database-validation-and-enhanced-results-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Database Validation and Enhanced Results \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change the default deployment_mode from 'server'", "why": "existing users who rely on Docker + MinIO must not be affected by any implicit mode change"}, {"INSTEAD": "use a version-pinned constant in process_manager.py with a clear upgrade path", "dont": "hardcode binary download URLs in the adapter", "why": "Databend releases frequently"}, {"INSTEAD": "(not specified)", "dont": "implement standalone mode for Linux in this TODO", "why": "the scope is macOS (the user-identified gap); Linux users already have Docker"}, {"INSTEAD": "if w2 reveals flakiness, park the remaining work", "dont": "skip the validation gate (w2)", "why": "the process lifecycle complexity is only worth adding if standalone mode is actually reliable"}, {"INSTEAD": "start only in create_connection()", "dont": "start Databend processes in __init__", "why": "the adapter may be constructed for introspection/dry-run without a real benchmark"}], "approach": "Follow the ClickHouse 3-mode pattern precisely:\n - deployment_mode.py for normalization/resolution (clickhouse/deployment_mode.py)\n - Thin first-class adapters that force their mode (clickhouse_local.py)\n - Mode-specific setup methods inside the base adapter (clickhouse/setup.py)\n - No eager process startup - start the standalone binary only inside\n create_connection(), not in __init__\nUnlike ClickHouse local mode (chDB, in-process), standalone requires external\nprocess lifecycle via process_manager.py. Treat this as the most complex part:\nwrite it defensively with timeouts, port conflict detection, and clean failure\nmodes. Do not attempt to make process management \"clever\" - sequential start\nwith polling is correct and sufficient.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Databend currently supports two implicit modes - remote self-hosted (Docker +\nMinIO) and Databend Cloud - but they aren't modelled as distinct deployment\nmodes. This TODO adds a ClickHouse-style 3-mode architecture:\n\n standalone Native macOS binary, local disk storage, no Docker or object\n store required. Manages databend-meta and databend-query\n processes in-process during the benchmark run.\n server Current Docker-based mode (MinIO + databend-query). Unchanged.\n cloud Databend Cloud managed service (HTTPS, existing credential\n flow). Promoted to a first-class named platform.\n\n--- COMPLEXITY NOTICE ---\nThis is significantly harder than the ClickHouse equivalent. ClickHouse local\nmode uses chDB, an embedded in-process Python library with zero binary\nmanagement. Databend standalone requires managing TWO separate external OS\nprocesses with ordered startup and teardown:\n\n 1. databend-meta (Raft consensus / metadata service, gRPC port 9191)\n 2. databend-query (compute / SQL layer, HTTP port 8000)\n\nThe meta service must be healthy before the query service starts. Shutdown\nmust be reversed. Both binaries must be downloaded and cached per architecture\n(arm64 vs x86_64). Two TOML config files must be generated per run. This is\ncloser in complexity to an embedded-PostgreSQL lifecycle manager than to\nDuckDB or chDB. The work should not begin until the validation gate (w2)\nconfirms the standalone mode is reliable enough to be worth this investment.\n\n--- KEY RESEARCH FINDINGS (2026-04-20) ---\nFrom prior conversation analysis:\n - Databend releases macOS arm64 + x86_64 binaries on GitHub releases\n - supports storage_type = \"disk\" (no MinIO) for development/testing\n - DSN format unchanged: databend+http://user:pass@localhost:8000/db\n - adapter.py create_connection(), create_schema(), load_data() all survive\n unchanged - they already use the DSN, no mode-specific logic needed\n - deployment_mode.py, process_manager.py, config_generator.py are all new\n files (none exist today); pattern follows clickhouse/deployment_mode.py\n - Raft data dir, ports 9191 (meta gRPC), 8000 (query HTTP), 3307 (MySQL)\n all need to be configurable and default to localhost with no port\n conflicts against other platforms\n", "id": "databend-3-mode-local-server-cloud", "preserves": ["All existing databend unit tests continue to pass after deployment_mode changes", "Databend Cloud credential flow (DATABEND_HOST, DATABEND_USER, DATABEND_PASSWORD, DATABEND_SSL env vars) continues to work for existing users", "Docker docker-compose.yml for databend is not modified or removed", "Existing 'databend' platform behavior (server/Docker mode) is completely unchanged - default deployment_mode must remain 'server'"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/platform_defaults.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend_cloud.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend_standalone.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/databend-standalone.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/databend.md"}, {"kind": "only_modify", "path_glob": "examples/getting_started/local/"}, {"kind": "only_modify", "path_glob": "tests/integration/platforms/test_databend_smoke.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databend_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databend_deployment_mode.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databend_standalone.py"}], "state": "planning", "title": "Add 3-mode support to Databend platform (standalone / server / cloud)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_databend_adapter.py -q", "description": "Existing Databend unit tests unaffected by deployment_mode changes", "expected": "All tests pass with 0 failures", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_databend_deployment_mode.py -q", "description": "New deployment_mode unit tests pass", "expected": "All tests pass with 0 failures", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.platforms import get_platform_adapter; a = get_platform_adapter('databend-standalone'); print(a.deployment_mode)\"", "description": "databend-standalone platform is registered and listed", "expected": "standalone", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.platforms import get_platform_adapter; a = get_platform_adapter('databend-cloud'); print(a.deployment_mode)\"", "description": "databend-cloud platform is registered and listed", "expected": "cloud", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/integration/platforms/test_databend_smoke.py::test_standalone_mode -q", "description": "TPC-H Q1 via standalone mode completes correctly (requires w2 environment)", "expected": "1 passed", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": [], "notes": "This is a mandatory gate. Do not proceed to w2 or later without completing\nthis step. The findings here update the KNOWN UNKNOWNS below.\n\nSteps:\n 1. Find the latest stable Databend release with macOS binaries on GitHub\n (datafuselabs/databend). Record the exact binary names, URLs, and sizes\n for arm64 and x86_64.\n 2. Confirm the standalone deployment model: does it require both\n databend-meta and databend-query, or is there a single \"standalone\"\n binary that bundles both? (Check the latest docs/releases carefully -\n some versions ship databend-standalone or an all-in-one binary.)\n 3. Confirm whether storage_type = \"disk\" is still supported in the\n current release and what the exact TOML config key is.\n 4. Read the current BenchBox databend adapter fully to check if anything\n has changed since the prior research (adapter.py, __init__.py,\n docker-compose.yml).\n 5. Update this TODO's KNOWN UNKNOWNS section with your findings. If any\n finding materially changes the implementation plan, update w3-w8\n accordingly before proceeding.\n\nKNOWN UNKNOWNS (to be resolved in w1):\n - Whether a single \"databend-standalone\" binary exists (simplifies\n lifecycle management to 1 process instead of 2)\n - Exact TOML config keys for disk storage in the current release\n - Whether Raft state is required even for single-node dev mode or if\n there is a simpler embedded meta option\n - Minimum macOS version required for the binaries\n - Whether the databend-driver Python package version pinned in BenchBox\n (>=0.28.0) is compatible with the latest databend binary protocol\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Research gate: re-validate Databend Mac binary capabilities before any code changes", "wid": "w1"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w1"], "notes": "This is a go/no-go gate. If Databend standalone cannot complete TPC-H Q1\nreliably, the remaining work items (w3-w8) should be parked and this TODO\nupdated with the blocker. Do not write implementation code until w2 passes.\n\nSteps:\n 1. Download the Databend macOS binary identified in w1 to a temp dir.\n 2. Write minimal databend-meta.toml and databend-query.toml using local\n disk storage (path: /tmp/benchbox_databend_validation/).\n 3. Start databend-meta; poll until port 9191 responds.\n 4. Start databend-query; poll until port 8000 responds.\n 5. Run a minimal TPC-H SF 0.01 benchmark:\n benchbox run --platform databend \\\n --benchmark tpch --scale 0.01 \\\n --queries Q1\n (using DATABEND_HOST=localhost, DATABEND_PORT=8000,\n DATABEND_USER=benchbox, DATABEND_PASSWORD=benchbox,\n DATABEND_SSL=false)\n 6. Confirm Q1 returns correct results (use --validation loose).\n 7. Shut down query then meta; confirm clean exit.\n 8. Record: startup time, Q1 execution time, any errors or warnings.\n 9. Update this TODO with validation results before marking w2 done.\n\nGO criteria: Q1 completes with correct results, clean startup/shutdown.\nNO-GO criteria: startup fails, Q1 crashes, results are incorrect, or\nprocess cleanup leaves zombie processes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Validation gate: manually run TPC-H Q1 against Databend standalone on macOS before writing code", "wid": "w2"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w2"], "notes": "New file. Model directly on benchbox/platforms/clickhouse/deployment_mode.py.\n\nContents:\n - DATABEND_DEPLOYMENT_MODES = frozenset({\"standalone\", \"server\", \"cloud\"})\n - DEFAULT_DATABEND_DEPLOYMENT_MODE = \"server\" (preserve existing default)\n - normalize_databend_deployment_mode(value) \u2192 canonical string or raises\n - resolve_databend_deployment_mode(config) \u2192 reads config keys in priority:\n 1. \"deployment_mode\" (canonical)\n 2. \"mode\" (legacy alias)\n default: \"server\"\n - Unit tests in tests/unit/platforms/test_databend_deployment_mode.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add deployment_mode.py to benchbox/platforms/databend/", "wid": "w3"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w3"], "notes": "New file: benchbox/platforms/databend/process_manager.py\n\nResponsibilities:\n - _detect_arch() \u2192 \"aarch64\" or \"x86_64\" for macOS\n - _binary_cache_dir() \u2192 ~/.benchbox/binaries/databend/{version}/\n - download_databend_binaries(version) \u2192 downloads meta + query (or standalone\n binary if w1 confirms a single-binary option exists), verifies checksums\n - generate_meta_config(data_dir, ports) \u2192 writes databend-meta.toml\n - generate_query_config(data_dir, meta_port, query_port) \u2192 writes databend-query.toml\n - start_standalone(data_dir, ports) \u2192 starts meta, polls 9191, starts query,\n polls 8000; returns (meta_proc, query_proc) handles\n - stop_standalone(meta_proc, query_proc) \u2192 SIGTERM query, wait, SIGTERM meta, wait;\n raises if either exits non-zero\n - Context manager: DatabendStandaloneProcess for use in adapter lifecycle\n\nPort defaults: meta_grpc=9191, query_http=8000, query_mysql=3307\nConfig must be parameterizable to avoid conflicts with other running platforms.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Implement process_manager.py: download, start, health-check, stop databend processes", "wid": "w4"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w4"], "notes": "Modify benchbox/platforms/databend/adapter.py:\n - Import resolve_databend_deployment_mode from deployment_mode\n - In __init__: resolve deployment_mode early, store as self.deployment_mode\n - Add _setup_standalone_mode(config): reads database_path, configures\n local ports, invokes process_manager to prepare (but not start -\n start happens in create_connection or a lifecycle hook)\n - create_connection() dispatch: standalone \u2192 start process, build DSN to\n localhost; server/cloud \u2192 existing behavior unchanged\n - disconnect() / cleanup: standalone \u2192 stop_standalone(); others \u2192 no change\n - database_path handling: use benchbox.utils.path_utils pattern like\n ClickHouse local mode; generate per-benchmark path from benchmark name\n + scale factor\n\nModify benchbox/platforms/databend/__init__.py:\n - Pass deployment_mode and database_path through _build_databend_config()\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Integrate standalone mode into DatabendAdapter (adapter.py + __init__.py)", "wid": "w5"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w5"], "notes": "New files (thin wrappers, mirror clickhouse_local.py and clickhouse_cloud.py):\n\nbenchbox/platforms/databend_standalone.py:\n - class DatabendStandaloneAdapter(DatabendAdapter)\n - __init__ forces config[\"deployment_mode\"] = \"standalone\"\n - Exposes database_path option\n\nbenchbox/platforms/databend_cloud.py:\n - class DatabendCloudAdapter(DatabendAdapter)\n - __init__ forces config[\"deployment_mode\"] = \"cloud\", ssl=True, port=443\n - Required credentials: DATABEND_HOST, DATABEND_PASSWORD (env vars)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add databend_standalone.py and databend_cloud.py first-class adapters", "wid": "w6"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w6"], "notes": "benchbox/platforms/__init__.py:\n - Add \"databend-standalone\" \u2192 DatabendStandaloneAdapter\n - Add \"databend-cloud\" \u2192 DatabendCloudAdapter\n - Keep \"databend\" \u2192 DatabendAdapter (unchanged, defaults to server mode)\n\nbenchbox/cli/platform_defaults.py (or equivalent registration point):\n - Add option specs for \"databend\": deployment_mode (standalone|server|cloud)\n - Add option specs for \"databend-standalone\": database_path\n - Add option specs for \"databend-cloud\": host, warehouse, ssl\n\nbenchbox/core/platform_registry.py:\n - Register databend-standalone and databend-cloud metadata entries\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Register databend-standalone and databend-cloud in platform registry and CLI", "wid": "w7"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w7"], "notes": "Unit tests:\n - tests/unit/platforms/test_databend_deployment_mode.py (new)\n - tests/unit/platforms/test_databend_standalone.py (new)\n - Extend tests/unit/platforms/test_databend_adapter.py: mode resolution,\n standalone setup, cloud setup\n\nIntegration tests:\n - tests/integration/platforms/test_databend_smoke.py: add standalone mode\n smoke (requires binaries; mark @pytest.mark.live_integration)\n\nDocs:\n - docs/platforms/databend.md: update to describe all 3 modes\n - docs/platforms/databend-standalone.md (new): quick start, binary\n download, ports, limitations vs server mode\n\nExamples:\n - examples/getting_started/local/databend_standalone_tpch.py (new)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Tests, docs, and examples for 3-mode Databend", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the default deployment_mode from 'server'", "instead": "(not specified)", "why": "existing users who rely on Docker + MinIO must not be affected by any implicit mode change"}, {"dont": "hardcode binary download URLs in the adapter", "instead": "use a version-pinned constant in process_manager.py with a clear upgrade path", "why": "Databend releases frequently"}, {"dont": "implement standalone mode for Linux in this TODO", "instead": "(not specified)", "why": "the scope is macOS (the user-identified gap); Linux users already have Docker"}, {"dont": "skip the validation gate (w2)", "instead": "if w2 reveals flakiness, park the remaining work", "why": "the process lifecycle complexity is only worth adding if standalone mode is actually reliable"}, {"dont": "start Databend processes in __init__", "instead": "start only in create_connection()", "why": "the adapter may be constructed for introspection/dry-run without a real benchmark"}], "approach": "Follow the ClickHouse 3-mode pattern precisely:\n - deployment_mode.py for normalization/resolution (clickhouse/deployment_mode.py)\n - Thin first-class adapters that force their mode (clickhouse_local.py)\n - Mode-specific setup methods inside the base adapter (clickhouse/setup.py)\n - No eager process startup - start the standalone binary only inside\n create_connection(), not in __init__\nUnlike ClickHouse local mode (chDB, in-process), standalone requires external\nprocess lifecycle via process_manager.py. Treat this as the most complex part:\nwrite it defensively with timeouts, port conflict detection, and clean failure\nmodes. Do not attempt to make process management \"clever\" - sequential start\nwith polling is correct and sufficient.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Databend currently supports two implicit modes - remote self-hosted (Docker +\nMinIO) and Databend Cloud - but they aren't modelled as distinct deployment\nmodes. This TODO adds a ClickHouse-style 3-mode architecture:\n\n standalone Native macOS binary, local disk storage, no Docker or object\n store required. Manages databend-meta and databend-query\n processes in-process during the benchmark run.\n server Current Docker-based mode (MinIO + databend-query). Unchanged.\n cloud Databend Cloud managed service (HTTPS, existing credential\n flow). Promoted to a first-class named platform.\n\n--- COMPLEXITY NOTICE ---\nThis is significantly harder than the ClickHouse equivalent. ClickHouse local\nmode uses chDB, an embedded in-process Python library with zero binary\nmanagement. Databend standalone requires managing TWO separate external OS\nprocesses with ordered startup and teardown:\n\n 1. databend-meta (Raft consensus / metadata service, gRPC port 9191)\n 2. databend-query (compute / SQL layer, HTTP port 8000)\n\nThe meta service must be healthy before the query service starts. Shutdown\nmust be reversed. Both binaries must be downloaded and cached per architecture\n(arm64 vs x86_64). Two TOML config files must be generated per run. This is\ncloser in complexity to an embedded-PostgreSQL lifecycle manager than to\nDuckDB or chDB. The work should not begin until the validation gate (w2)\nconfirms the standalone mode is reliable enough to be worth this investment.\n\n--- KEY RESEARCH FINDINGS (2026-04-20) ---\nFrom prior conversation analysis:\n - Databend releases macOS arm64 + x86_64 binaries on GitHub releases\n - supports storage_type = \"disk\" (no MinIO) for development/testing\n - DSN format unchanged: databend+http://user:pass@localhost:8000/db\n - adapter.py create_connection(), create_schema(), load_data() all survive\n unchanged - they already use the DSN, no mode-specific logic needed\n - deployment_mode.py, process_manager.py, config_generator.py are all new\n files (none exist today); pattern follows clickhouse/deployment_mode.py\n - Raft data dir, ports 9191 (meta gRPC), 8000 (query HTTP), 3307 (MySQL)\n all need to be configurable and default to localhost with no port\n conflicts against other platforms\n", "id": "databend-3-mode-local-server-cloud", "preserves": ["All existing databend unit tests continue to pass after deployment_mode changes", "Databend Cloud credential flow (DATABEND_HOST, DATABEND_USER, DATABEND_PASSWORD, DATABEND_SSL env vars) continues to work for existing users", "Docker docker-compose.yml for databend is not modified or removed", "Existing 'databend' platform behavior (server/Docker mode) is completely unchanged - default deployment_mode must remain 'server'"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/platform_defaults.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend_cloud.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databend_standalone.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/databend-standalone.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/databend.md"}, {"kind": "only_modify", "path_glob": "examples/getting_started/local/"}, {"kind": "only_modify", "path_glob": "tests/integration/platforms/test_databend_smoke.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databend_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databend_deployment_mode.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databend_standalone.py"}], "state": "planning", "title": "Add 3-mode support to Databend platform (standalone / server / cloud)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_databend_adapter.py -q", "description": "Existing Databend unit tests unaffected by deployment_mode changes", "expected": "All tests pass with 0 failures", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_databend_deployment_mode.py -q", "description": "New deployment_mode unit tests pass", "expected": "All tests pass with 0 failures", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.platforms import get_platform_adapter; a = get_platform_adapter('databend-standalone'); print(a.deployment_mode)\"", "description": "databend-standalone platform is registered and listed", "expected": "standalone", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.platforms import get_platform_adapter; a = get_platform_adapter('databend-cloud'); print(a.deployment_mode)\"", "description": "databend-cloud platform is registered and listed", "expected": "cloud", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/integration/platforms/test_databend_smoke.py::test_standalone_mode -q", "description": "TPC-H Q1 via standalone mode completes correctly (requires w2 environment)", "expected": "1 passed", "item_id": "databend-3-mode-local-server-cloud", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": [], "notes": "This is a mandatory gate. Do not proceed to w2 or later without completing\nthis step. The findings here update the KNOWN UNKNOWNS below.\n\nSteps:\n 1. Find the latest stable Databend release with macOS binaries on GitHub\n (datafuselabs/databend). Record the exact binary names, URLs, and sizes\n for arm64 and x86_64.\n 2. Confirm the standalone deployment model: does it require both\n databend-meta and databend-query, or is there a single \"standalone\"\n binary that bundles both? (Check the latest docs/releases carefully -\n some versions ship databend-standalone or an all-in-one binary.)\n 3. Confirm whether storage_type = \"disk\" is still supported in the\n current release and what the exact TOML config key is.\n 4. Read the current BenchBox databend adapter fully to check if anything\n has changed since the prior research (adapter.py, __init__.py,\n docker-compose.yml).\n 5. Update this TODO's KNOWN UNKNOWNS section with your findings. If any\n finding materially changes the implementation plan, update w3-w8\n accordingly before proceeding.\n\nKNOWN UNKNOWNS (to be resolved in w1):\n - Whether a single \"databend-standalone\" binary exists (simplifies\n lifecycle management to 1 process instead of 2)\n - Exact TOML config keys for disk storage in the current release\n - Whether Raft state is required even for single-node dev mode or if\n there is a simpler embedded meta option\n - Minimum macOS version required for the binaries\n - Whether the databend-driver Python package version pinned in BenchBox\n (>=0.28.0) is compatible with the latest databend binary protocol\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Research gate: re-validate Databend Mac binary capabilities before any code changes", "wid": "w1"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w1"], "notes": "This is a go/no-go gate. If Databend standalone cannot complete TPC-H Q1\nreliably, the remaining work items (w3-w8) should be parked and this TODO\nupdated with the blocker. Do not write implementation code until w2 passes.\n\nSteps:\n 1. Download the Databend macOS binary identified in w1 to a temp dir.\n 2. Write minimal databend-meta.toml and databend-query.toml using local\n disk storage (path: /tmp/benchbox_databend_validation/).\n 3. Start databend-meta; poll until port 9191 responds.\n 4. Start databend-query; poll until port 8000 responds.\n 5. Run a minimal TPC-H SF 0.01 benchmark:\n benchbox run --platform databend \\\n --benchmark tpch --scale 0.01 \\\n --queries Q1\n (using DATABEND_HOST=localhost, DATABEND_PORT=8000,\n DATABEND_USER=benchbox, DATABEND_PASSWORD=benchbox,\n DATABEND_SSL=false)\n 6. Confirm Q1 returns correct results (use --validation loose).\n 7. Shut down query then meta; confirm clean exit.\n 8. Record: startup time, Q1 execution time, any errors or warnings.\n 9. Update this TODO with validation results before marking w2 done.\n\nGO criteria: Q1 completes with correct results, clean startup/shutdown.\nNO-GO criteria: startup fails, Q1 crashes, results are incorrect, or\nprocess cleanup leaves zombie processes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Validation gate: manually run TPC-H Q1 against Databend standalone on macOS before writing code", "wid": "w2"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w2"], "notes": "New file. Model directly on benchbox/platforms/clickhouse/deployment_mode.py.\n\nContents:\n - DATABEND_DEPLOYMENT_MODES = frozenset({\"standalone\", \"server\", \"cloud\"})\n - DEFAULT_DATABEND_DEPLOYMENT_MODE = \"server\" (preserve existing default)\n - normalize_databend_deployment_mode(value) \u2192 canonical string or raises\n - resolve_databend_deployment_mode(config) \u2192 reads config keys in priority:\n 1. \"deployment_mode\" (canonical)\n 2. \"mode\" (legacy alias)\n default: \"server\"\n - Unit tests in tests/unit/platforms/test_databend_deployment_mode.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add deployment_mode.py to benchbox/platforms/databend/", "wid": "w3"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w3"], "notes": "New file: benchbox/platforms/databend/process_manager.py\n\nResponsibilities:\n - _detect_arch() \u2192 \"aarch64\" or \"x86_64\" for macOS\n - _binary_cache_dir() \u2192 ~/.benchbox/binaries/databend/{version}/\n - download_databend_binaries(version) \u2192 downloads meta + query (or standalone\n binary if w1 confirms a single-binary option exists), verifies checksums\n - generate_meta_config(data_dir, ports) \u2192 writes databend-meta.toml\n - generate_query_config(data_dir, meta_port, query_port) \u2192 writes databend-query.toml\n - start_standalone(data_dir, ports) \u2192 starts meta, polls 9191, starts query,\n polls 8000; returns (meta_proc, query_proc) handles\n - stop_standalone(meta_proc, query_proc) \u2192 SIGTERM query, wait, SIGTERM meta, wait;\n raises if either exits non-zero\n - Context manager: DatabendStandaloneProcess for use in adapter lifecycle\n\nPort defaults: meta_grpc=9191, query_http=8000, query_mysql=3307\nConfig must be parameterizable to avoid conflicts with other running platforms.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Implement process_manager.py: download, start, health-check, stop databend processes", "wid": "w4"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w4"], "notes": "Modify benchbox/platforms/databend/adapter.py:\n - Import resolve_databend_deployment_mode from deployment_mode\n - In __init__: resolve deployment_mode early, store as self.deployment_mode\n - Add _setup_standalone_mode(config): reads database_path, configures\n local ports, invokes process_manager to prepare (but not start -\n start happens in create_connection or a lifecycle hook)\n - create_connection() dispatch: standalone \u2192 start process, build DSN to\n localhost; server/cloud \u2192 existing behavior unchanged\n - disconnect() / cleanup: standalone \u2192 stop_standalone(); others \u2192 no change\n - database_path handling: use benchbox.utils.path_utils pattern like\n ClickHouse local mode; generate per-benchmark path from benchmark name\n + scale factor\n\nModify benchbox/platforms/databend/__init__.py:\n - Pass deployment_mode and database_path through _build_databend_config()\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Integrate standalone mode into DatabendAdapter (adapter.py + __init__.py)", "wid": "w5"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w5"], "notes": "New files (thin wrappers, mirror clickhouse_local.py and clickhouse_cloud.py):\n\nbenchbox/platforms/databend_standalone.py:\n - class DatabendStandaloneAdapter(DatabendAdapter)\n - __init__ forces config[\"deployment_mode\"] = \"standalone\"\n - Exposes database_path option\n\nbenchbox/platforms/databend_cloud.py:\n - class DatabendCloudAdapter(DatabendAdapter)\n - __init__ forces config[\"deployment_mode\"] = \"cloud\", ssl=True, port=443\n - Required credentials: DATABEND_HOST, DATABEND_PASSWORD (env vars)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add databend_standalone.py and databend_cloud.py first-class adapters", "wid": "w6"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w6"], "notes": "benchbox/platforms/__init__.py:\n - Add \"databend-standalone\" \u2192 DatabendStandaloneAdapter\n - Add \"databend-cloud\" \u2192 DatabendCloudAdapter\n - Keep \"databend\" \u2192 DatabendAdapter (unchanged, defaults to server mode)\n\nbenchbox/cli/platform_defaults.py (or equivalent registration point):\n - Add option specs for \"databend\": deployment_mode (standalone|server|cloud)\n - Add option specs for \"databend-standalone\": database_path\n - Add option specs for \"databend-cloud\": host, warehouse, ssl\n\nbenchbox/core/platform_registry.py:\n - Register databend-standalone and databend-cloud metadata entries\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Register databend-standalone and databend-cloud in platform registry and CLI", "wid": "w7"}, {"evidence": null, "item_id": "databend-3-mode-local-server-cloud", "needs": ["w7"], "notes": "Unit tests:\n - tests/unit/platforms/test_databend_deployment_mode.py (new)\n - tests/unit/platforms/test_databend_standalone.py (new)\n - Extend tests/unit/platforms/test_databend_adapter.py: mode resolution,\n standalone setup, cloud setup\n\nIntegration tests:\n - tests/integration/platforms/test_databend_smoke.py: add standalone mode\n smoke (requires binaries; mark @pytest.mark.live_integration)\n\nDocs:\n - docs/platforms/databend.md: update to describe all 3 modes\n - docs/platforms/databend-standalone.md (new): quick start, binary\n download, ports, limitations vs server mode\n\nExamples:\n - examples/getting_started/local/databend_standalone_tpch.py (new)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Tests, docs, and examples for 3-mode Databend", "wid": "w8"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-16T00:00:00Z", "completed_pr": null, "created_at": "2025-01-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement Databend (https://github.com/databendlabs/databend) as a supported platform\nin BenchBox. Databend is a cloud-native, Rust-based data warehouse with Snowflake-\ncompatible SQL, compute/storage separation, and object storage backend.\n\n**Key Technical Approach:**\n- Use Snowflake SQLGlot dialect as translation proxy (Databend claims ~100% compatibility)\n- Leverage `databend-driver` (v0.28.2+) for DB-API 2.0 connectivity\n- Implement lightweight QueryTransformer for edge cases if needed\n- Deploy Databend + MinIO via Docker Compose for CI testing\n\n**Rationale for Low Priority:**\nBased on comprehensive analysis (Jan 2025), integration was deferred because:\n1. SQLGlot does NOT support Databend as a native dialect\n2. Significant platform overlap with ClickHouse, Snowflake, and Firebolt\n3. Lower enterprise adoption compared to alternatives\n4. Medium-high integration complexity due to custom translation layer\n\n**Conditions for Prioritization:**\n- SQLGlot adds native Databend dialect support\n- User demand accumulates (3+ explicit requests)\n- Databend Cloud enterprise adoption grows\n- TPC-certified benchmark results published", "id": "databend-cloud-olap-platform", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Databend Cloud-Native OLAP Platform Integration", "verifications": [], "work": [{"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create databend/ platform module structure", "wid": "w1"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add query plan capture support (EXPLAIN)", "wid": "w10"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement tuning support (clustering keys)", "wid": "w11"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test all 22 TPC-H queries for compatibility", "wid": "w12"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test all 99 TPC-DS queries for compatibility", "wid": "w13"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement QueryTransformer for edge cases if needed", "wid": "w14"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document any query modifications required", "wid": "w15"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create unit tests (test_databend_adapter.py)", "wid": "w16"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create Docker Compose for Databend + MinIO", "wid": "w17"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add integration smoke tests", "wid": "w18"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add CI workflow for Databend testing", "wid": "w19"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement base DatabendAdapter class", "wid": "w2"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create platform documentation (docs/platforms/databend.md)", "wid": "w20"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add example script (examples/getting_started/cloud/)", "wid": "w21"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update platform comparison matrix", "wid": "w22"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement connection handling with databend-driver", "wid": "w3"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Configure Snowflake dialect as translation proxy", "wid": "w4"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement CLI argument registration", "wid": "w5"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add databend dependency group to pyproject.toml", "wid": "w6"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement schema creation with Snowflake SQL translation", "wid": "w7"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement data loading (INSERT batching)", "wid": "w8"}, {"evidence": null, "item_id": "databend-cloud-olap-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement query execution with result handling", "wid": "w9"}], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Archived DONE item for: \"\u2705 Databricks Adapter: Correct Data Loading Approach (COPY INTO + Unity Catalog Volumes)\"", "id": "databricks-adapter-correct-data-loading-approach-copy-into-unity-catalog-volumes", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"\u2705 Databricks Adapter: Correct Data Loading Approach (COPY INTO + Unity Catalog Volumes)\"", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"benchbox/platforms/databricks.py: load_data now uses COPY INTO for each table, infers delimiter (tbl \u2192 `|`, csv \u2192 `,`), counts rows, logs timings, and optionally OPTIMIZEs Delta tables. Supports staging via:\"\n- Enforces staging location presence (raises ValueError if missing) and eliminates temporary views/INSERT SELECT.\n- \"Tests: tests/unit/platforms/test_databricks_copy_into.py validates COPY INTO SQL, absence of temp views/INSERT, UC Volume staging, and error on missing staging.\"\n- Path normalization for remote output roots added separately for consistency across CLI/unified runner.", "id": "databricks-adapter-correct-data-loading-approach-copy-into-unity-catalog-volumes-2", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\"\u2705 Databricks Adapter: Correct Data Loading Approach (COPY INTO + Unity Catalog Volumes)\"", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-24T00:00:00Z", "completed_pr": null, "created_at": "2025-12-24T00:00:00Z", "deferrals": [], "deps": [], "description": "Add DataFrame execution mode to the existing Databricks adapter. This is the\nCRITICAL PRIORITY item for Week 1 of the managed cloud Spark integration project.\n\nDatabricks is the market leader for managed Spark with the largest user base.\nThis implementation leverages 100% of existing infrastructure (session management,\nUC Volumes, Delta Lake, authentication) with minimal new code.", "id": "databricks-dataframe-mode", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "Databricks DataFrame Mode - CRITICAL PRIORITY", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Support and Deployment", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Update all Databricks examples to use modern 3-line adapter pattern with Databricks-specific tuning flags similar to updated DuckDB TPC-H and TPC-DS examples\"\n- \"**Tasks**:\"\n- \"**Files Affected**: 5 key `examples/databricks_*.py` files modernized (tpch, tpcds, tpchavoc, ssb, primitives)\"\n- \"**Impact**: Successfully eliminated manual Databricks SDK anti-patterns, implemented consistent 3-line adapter pattern across all modernized examples, added comprehensive Databricks-specific features including Unity Catalog integration and Delta Lake optimizations, enhanced with statistical analysis and concurrent query capabilities, providing production-ready examples that demonstrate correct BenchBox usage with modern CLI interfaces\"", "id": "databricks-example-modernization-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Databricks Example Modernization \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": "imported from YAML status: Blocked", "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-26T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up validation for commit `b56457e17` (\"feat(databricks): add liquid TPC rendering profiles\").\nGoal is to keep a concrete evidence ledger for remaining Databricks liquid TPC coverage gaps, including\nruntime compatibility, conflict validation, metadata integrity, and comparison-path behavior.\n", "id": "databricks-liquid-clustering-tuning-review-20260526", "preserves": ["Do not claim Databricks live certification without successful real run(s) proving live acceptance and metadata fields.", "Do not commit raw Databricks logs or benchmark output directories; only reference artifact paths and summarize in evidence.", "Keep clear comparison boundary across zorder, liquid auto/manual, and duckdb tuned results (physical rendering / profile identity).", "No silent migration of legacy ZORDER templates to Liquid semantics."], "prior_art": [{"concept": "benchbox/core/tuning/interface.py: databricks effective layout validation", "decision": "reuse", "path": "benchbox/core/tuning/interface.py"}, {"concept": "benchbox/platforms/databricks/adapter.py: metadata and strategy application paths", "decision": "reuse", "path": "benchbox/platforms/databricks/adapter.py"}, {"concept": "tests/unit/platforms/test_databricks_liquid_clustering.py: metadata assertions for requested/resolved/operations", "decision": "reuse", "path": "tests/unit/platforms/test_databricks_liquid_clustering.py"}], "priority": "high", "scope": [], "state": "active", "title": "Databricks liquid clustering review follow-up: runtime compatibility, validation gaps, and live certification", "verifications": [{"command": "uv run -- python _project/scripts/tuning_profile_check.py --benchmarks tpch,tpcds --platforms databricks,duckdb --strict", "description": "Re-run baseline profile and unit checks", "expected": "All tuned/liquid templates map and render as expected", "item_id": "databricks-liquid-clustering-tuning-review-20260526", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- benchbox run --dry-run /tmp/review_mixed ...; uv run -- benchbox run --dry-run /tmp/review_none_invalid ...; uv run -- benchbox run --dry-run /tmp/review_liquid_partition ...; uv run -- benchbox run --dry-run /tmp/review_liquid_distribution ...", "description": "Re-run conflict + rendering dry-runs and confirm PASS/FAIL matrix", "expected": "Expected failures for all four conflict cases", "item_id": "databricks-liquid-clustering-tuning-review-20260526", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "databricks-liquid-clustering-tuning-review-20260526", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Preserve all evidence and rerun the listed baseline checks after any future changes", "wid": "w1"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-review-20260526", "needs": ["w1"], "notes": "Current behavior passes 5-key liquid_clustering_columns in /tmp/benchbox_tuning_manual_5cols.yaml without rejection.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add explicit validation for `liquid_clustering_columns` over 4 keys in manual mode and add regression test", "wid": "w2"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-review-20260526", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add runtime compatibility checks (DBR/channel/warehouse/UC/optimize-full eligibility) before emitting liquid-clustering SQL and add unit/integration assertions for each failure mode.", "wid": "w3"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-review-20260526", "needs": ["w3"], "notes": "Blocked until valid environment credentials and approved Databricks target are available.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Run live Databricks SF 0.01 executions for tpch/tpcds liquid templates with valid credentials and verify all requested Databricks metadata fields.", "wid": "w4"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-review-20260526", "needs": ["w4"], "notes": "Verify result metadata includes requested/resolved strategy, physical rendering id, applied/skipped operations, logical tuning profile id, and platform physical tuning mechanisms.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "On successful live run, verify result metadata contains all liquid-specific fields", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep logic adapter-scoped where possible", "dont": "implement Databricks-only behavior in platform-agnostic base classes without guards", "why": "it risks cross-platform regressions"}, {"INSTEAD": "fail fast with clear remediation", "dont": "ship strategy parsing without conflict validation", "why": "mixed config states become non-reproducible"}, {"INSTEAD": "require explicit strategy selection", "dont": "silently reinterpret existing Z-ORDER configs as liquid clustering", "why": "it changes benchmark semantics"}], "approach": "Extend the unified tuning model with an explicit Databricks clustering strategy\nrepresentation rather than inferring from generic sorting fields. Apply the strategy\nin Databricks adapter lifecycle points where table format/state is known.\n\nKeep precedence rules explicit: liquid clustering, z-order, and none must be mutually\nunderstandable with clear conflict handling and warnings.\n\nAdd result metadata so downstream analysis can segment runs by applied strategy.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [], "deps": ["cloud-dw-sorted-data-ingestion", "ctas-sort-cloud-platforms"], "description": "Expand Databricks tuning support so Liquid Clustering is a first-class,\nconfigurable strategy in BenchBox, not a secondary/implicit path behind\nZ-ORDER/OPTIMIZE workflows.\n\nRationale:\n- Databricks guidance is increasingly steering users toward Liquid Clustering.\n- Current BenchBox tuning emphasizes Z-ORDER and OPTIMIZE behaviors.\n- Users need direct control and comparability for Liquid Clustering in the\n unified tuning model and CLI/platform-option workflows.\n\nScope:\n- Add explicit Liquid Clustering representation in tuning configuration.\n- Implement Databricks adapter execution paths for create-time and post-load\n Liquid Clustering operations.\n- Preserve backwards compatibility for existing Z-ORDER-based configs.\n- Expose strategy metadata in results to support fair comparisons.", "id": "databricks-liquid-clustering-tuning-support", "preserves": ["Databricks adapter error messages remain actionable when strategy prerequisites are missing", "Dry-run output remains deterministic and includes all emitted Databricks tuning actions", "Existing Databricks runs using Z-ORDER tuning continue to execute without config migration", "Non-Databricks platforms ignore Databricks-specific liquid clustering settings cleanly"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/snowflake.py"}], "state": "done", "title": "Add first-class Databricks Liquid Clustering support in BenchBox tuning configurations", "verifications": [{"command": "uv run _project/scripts/validate_todo.py _project/DONE/physical-tuning/active/databricks-liquid-clustering-tuning-support.yaml", "description": "New TODO validates against schema", "expected": "Validation passed", "item_id": "databricks-liquid-clustering-tuning-support", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_databricks_liquid_clustering.py -v --tb=short", "description": "Databricks liquid clustering tests pass", "expected": "all tests pass, 0 failures", "item_id": "databricks-liquid-clustering-tuning-support", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -k databricks -v --tb=short", "description": "Existing Databricks adapter tests remain green", "expected": "all tests pass, 0 failures", "item_id": "databricks-liquid-clustering-tuning-support", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define tuning schema updates for Databricks liquid clustering strategy and validation rules", "wid": "w1"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add config parsing/normalization for liquid clustering in CLI and tuning loader paths", "wid": "w2"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Databricks adapter support for liquid clustering DDL/DML lifecycle operations", "wid": "w3"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Maintain backward compatibility path for existing Z-ORDER configs with explicit precedence rules", "wid": "w4"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add result metadata fields for applied Databricks clustering strategy and maintenance operations", "wid": "w5"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w3", "w4", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit/integration tests for config validation and adapter SQL/action generation", "wid": "w6"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs/examples for liquid clustering usage, migration guidance, and A/B comparison with Z-ORDER", "wid": "w7"}], "worktree": "physical-tuning"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add broad automatic template rewriting before validation and golden tests can prove every emitted field.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "allow Liquid Clustering and Z-ORDER fields in the same effective config without a hard validation error or a documented explicit waiver.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "claim cross-platform performance equivalence from logical profile coverage alone.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "keep hardcoding Databricks profile mechanisms to `z_order`; the mapping must depend on the selected physical rendering.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "report Databricks `distribution` as a real physical distribution key; Databricks distribution candidates are logical locality evidence.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "silently rename or reinterpret existing Databricks TPC tuned YAMLs as Liquid Clustering; that would change historical comparison semantics.", "why": "(unstated)"}], "approach": "Treat Databricks Liquid Clustering as a strategy-aware physical rendering of\nthe existing logical TPC profile. First revalidate current Databricks docs and\nBenchBox execution order, then model rendering ids and mechanism semantics in\nthe profile validation layer. Add Liquid-specific templates or generated\nartifacts only after conflict validation is explicit. Finally, wire result\nmetadata and tests so comparisons cannot collapse Z-ORDER, Liquid, and\nfallback tuning into one misleading `tuned` bucket.\n\nPrefer small, reviewable layers:\n 1. strategy-aware mapping data model;\n 2. Databricks Liquid template/rendering artifacts;\n 3. config and adapter conflict validation;\n 4. lifecycle/operation metadata correctness;\n 5. docs and comparison semantics.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-26T00:00:00Z", "completed_pr": null, "created_at": "2026-05-26T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "databricks-tpc-liquid-clustering-profile-rendering", "id": 139, "reason": "Requires cloud credentials and explicit live-test approval.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Live Databricks warehouse validation for Liquid TPC is already covered by open item databricks-liquid-clustering-tuning-review-20260526.", "summary": "Run live Databricks warehouse validation for Liquid TPC-H/TPC-DS"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "databricks-tpc-liquid-clustering-profile-rendering", "id": 140, "reason": "This TODO must keep the Databricks Liquid rendering gap focused while preserving generalized profile semantics.", "resolution": "promoted", "resolved_item": "generate-and-certify-bigquery-redshift-and-snowflake-tpc-tun", "resolved_reason": null, "summary": "Generate and certify BigQuery, Redshift, and Snowflake TPC tuned templates from the logical profile"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "databricks-tpc-liquid-clustering-profile-rendering", "id": 141, "reason": "Metadata must land first so the UI warning has a stable contract to consume.", "resolution": "promoted", "resolved_item": "build-a-comparison-ui-cohort-warning-for-cross-mechanism-tun", "resolved_reason": null, "summary": "Build a comparison-UI cohort warning for cross-mechanism tuned results"}], "deps": ["generalize-tpc-tuning-base"], "description": "Follow-up to generalize-tpc-tuning-base. The generalized TPC profile work\ncorrectly moved Databricks TPC-H/TPC-DS tuning columns into a reusable\nlogical profile, but review exposed a Databricks-specific blind spot that can\ncorrupt tuned-run comparisons: the current checked-in Databricks TPC examples\nstill render the logical profile through the legacy Z-ORDER path, while\nmodern Databricks guidance treats Liquid Clustering as the preferred table\nlayout for new Delta tables.\n\nThe goal is not to make Databricks special again. The goal is to separate\nthree concerns cleanly so the TPC tuning base stays reusable across platforms:\nlogical workload candidates, platform physical rendering strategy, and\nexecution/result metadata. Databricks must support at least Z-ORDER and\nLiquid Clustering as distinct renderings of the same logical TPC profile, and\nresult comparisons must expose which rendering was actually requested and\napplied.\n\nCurrent external evidence checked on 2026-05-26:\n- Databricks liquid clustering docs describe Liquid Clustering as a data\n layout optimization that replaces table partitioning and ZORDER, supports\n CREATE/ALTER TABLE CLUSTER BY, and supports CLUSTER BY AUTO for automatic\n key selection.\n Source: https://docs.databricks.com/gcp/en/delta/clustering\n- Databricks optimize-layout docs recommend Liquid Clustering instead of\n partitions, ZORDER, or other layout approaches; OPTIMIZE groups data by\n Liquid Clustering keys for clustered tables.\n Source: https://docs.databricks.com/aws/en/delta/optimize\n- Databricks data-skipping docs state that ZORDER cannot be used with Liquid\n Clustering and recommend Liquid Clustering for new tables.\n Source: https://docs.databricks.com/gcp/en/delta/data-skipping\n\nCurrent BenchBox facts to revalidate before implementation:\n- `TuningType.LIQUID_CLUSTERING` and Databricks liquid configuration fields\n already exist in `benchbox/core/tuning/interface.py`.\n- `DatabricksAdapter` can resolve `databricks_clustering_strategy` and emit\n `ALTER TABLE ... CLUSTER BY (...)`, but the TPC profile capability mapping\n currently reports Databricks logical locality as `z_order`.\n- The checked-in Databricks TPC YAMLs set `z_ordering_enabled: true` and keep\n per-table `partitioning` fields, which is not a valid Liquid Clustering\n rendering without a deliberate conversion rule.\n- Unified tuning application currently runs before data loading for fresh\n native tables, while Databricks post-load layout operations and metadata\n need to distinguish create-time `CLUSTER BY`, post-load `OPTIMIZE`, and\n Z-ORDER statements.", "id": "databricks-tpc-liquid-clustering-profile-rendering", "preserves": ["DuckDB tuned templates and mappings remain free of Databricks vocabulary such as Z-ORDER or Liquid Clustering.", "Existing Databricks Z-ORDER tuned templates remain available for legacy strategy comparisons unless a migration is explicitly approved.", "Liquid Clustering, Z-ORDER, and basic-constraints fallback are distinguishable in result metadata and comparison semantics.", "New routine tests carry explicit `unit` and `fast` pytest markers.", "No live Databricks/cloud tests run without explicit user approval.", "No result metadata claims a tuning mechanism was applied unless BenchBox attempted and recorded that operation.", "The logical TPC profile remains platform-neutral; Databricks is one renderer, not the source of truth."], "prior_art": [{"concept": "_project/DONE/physical-tuning/active/databricks-liquid-clustering-tuning-support.yaml: prior Databricks Liquid implementation TODO - supersede its broad completion claim with profile/rendering validation", "decision": "supersede", "path": "_project/DONE/physical-tuning/active/databricks-liquid-clustering-tuning-support.yaml"}, {"concept": "benchbox/cli/commands/run.py: current Databricks strategy and liquid-column platform-option normalization - extend conflict handling", "decision": "extend", "path": "benchbox/cli/commands/run.py"}, {"concept": "benchbox/cli/config.py: current Databricks sample tuning defaults - update only with an explicit legacy-vs-Liquid default decision", "decision": "reuse", "path": "benchbox/cli/config.py"}, {"concept": "benchbox/cli/dryrun.py: existing dry-run command rendering for sorted ingestion and platform options - keep preview truthful", "decision": "reuse", "path": "benchbox/cli/dryrun.py"}, {"concept": "benchbox/core/tuning/interface.py: existing Liquid Clustering config fields and TuningType - reuse and tighten validation", "decision": "reuse", "path": "benchbox/core/tuning/interface.py"}, {"concept": "benchbox/core/tuning/platform_capabilities.py: current logical-role to platform-mechanism mapping - make strategy-aware", "decision": "reuse", "path": "benchbox/core/tuning/platform_capabilities.py"}, {"concept": "benchbox/core/tuning/profile_validation.py: existing template/profile validation and metadata projection - extend with physical rendering ids", "decision": "extend", "path": "benchbox/core/tuning/profile_validation.py"}, {"concept": "benchbox/core/tuning/profiles/tpc.yaml: existing logical TPC tuning profile data - extend rather than replace", "decision": "extend", "path": "benchbox/core/tuning/profiles/tpc.yaml"}, {"concept": "benchbox/platforms/base/sorted_ingestion.py: existing strategy/capability matrix pattern - reuse for method capability semantics where applicable", "decision": "reuse", "path": "benchbox/platforms/base/sorted_ingestion.py"}, {"concept": "benchbox/platforms/databricks/adapter.py: existing Z-ORDER and Liquid Clustering operation hooks - make lifecycle and metadata behavior truthful", "decision": "reuse", "path": "benchbox/platforms/databricks/adapter.py"}, {"concept": "docs/usage/tpc-tuning-profiles.md: current logical profile comparison semantics - expand with Databricks strategy distinctions", "decision": "reuse", "path": "docs/usage/tpc-tuning-profiles.md"}, {"concept": "tests/unit/cli/test_run_command_branches.py and tests/unit/cli/test_cli_dryrun.py: current CLI override and dry-run coverage - extend for Liquid conflict validation", "decision": "extend", "path": "tests/unit/cli/test_run_command_branches.py"}, {"concept": "tests/unit/platforms/test_databricks_adapter.py and tests/unit/platforms/test_databricks_adapter_coverage.py: current Databricks strategy precedence coverage - update expected conflicts", "decision": "reuse", "path": "tests/unit/platforms/test_databricks_adapter.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/_binaries/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "docs/_build/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/databricks-tpc-liquid-clustering-profile-rendering.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/databricks-tpc-liquid-clustering-profile-rendering.yaml"}, {"kind": "only_modify", "path_glob": "_project/scripts/tuning_profile_check.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/config.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/interface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/platform_capabilities.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/profile_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/profiles/tpc.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/workload_profiles.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/result_capture.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/sorted_ingestion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "docs/reference/cli/run.md"}, {"kind": "only_modify", "path_glob": "docs/usage/cloud-sorted-ingestion.md"}, {"kind": "only_modify", "path_glob": "docs/usage/databricks-liquid-clustering.md"}, {"kind": "only_modify", "path_glob": "docs/usage/tpc-tuning-profiles.md"}, {"kind": "only_modify", "path_glob": "examples/tunings/README.md"}, {"kind": "only_modify", "path_glob": "examples/tunings/databricks/"}, {"kind": "only_modify", "path_glob": "examples/tunings/duckdb/"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_databricks_tuning_examples.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_interactive_dry_run.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_platform_hooks.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_run_command_branches.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tuning/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tuning/generators/test_spark_family_ddl.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Model Databricks TPC Liquid Clustering as a first-class reusable profile rendering", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/DONE/main/active/databricks-tpc-liquid-clustering-profile-rendering.yaml", "description": "New TODO validates against schema", "expected": "Validation passed", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after adding the dependency edge", "expected": "passes", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python _project/scripts/tuning_profile_check.py --benchmarks tpch,tpcds --platforms databricks,duckdb --strict", "description": "TPC profile validation covers both legacy Z-ORDER and Databricks Liquid renderings", "expected": "passes, with Databricks mechanisms reflecting the selected rendering", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -n 0 tests/unit/core/tuning/test_tpc_workload_profiles.py tests/unit/core/tuning/test_tpc_tuning_template_parity.py tests/unit/cli/test_databricks_tuning_examples.py -q", "description": "Strategy-aware tuning profile unit tests pass", "expected": "all tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -n 0 tests/unit/platforms -k 'databricks and (liquid or tuning or zorder)' -q", "description": "Databricks tuning strategy unit tests pass", "expected": "all relevant tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest -n 0 tests/unit/cli/test_run_command_branches.py tests/unit/cli/test_cli_dryrun.py tests/unit/cli/test_interactive_dry_run.py -q", "description": "CLI and dry-run strategy override tests pass", "expected": "all tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest -n 0 tests/unit/core/tuning/test_platform_optimization_config.py tests/unit/core/tuning/generators/test_spark_family_ddl.py -q", "description": "Core tuning config and Spark-family DDL tests pass", "expected": "all tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 7}, {"command": "make pr-preflight", "description": "Final PR gate before publication when implementing this TODO", "expected": "passes", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 8}], "work": [{"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": [], "notes": "Re-open the three Databricks documentation sources in `description`\nand record a compact dated summary in docs or TODO completion notes:\n - minimum Databricks Runtime/version constraints for manual Liquid\n Clustering, automatic Liquid Clustering, and OPTIMIZE FULL;\n - compatibility rules for Liquid Clustering with legacy partitioning\n and ZORDER;\n - recommended migration from Hive-style partitioning and ZORDER to\n Liquid Clustering keys;\n - current syntax for CREATE TABLE, ALTER TABLE, CLUSTER BY AUTO, and\n post-load optimization.\n\nThis is a hard freshness gate. Do not proceed with implementation from\nthe cited 2026-05-26 evidence alone; if Databricks guidance changed,\nupdate the TODO completion notes or docs with the checked date, URLs,\nand the changed behavior before touching runtime code.\n\nRe-read the current branch code and summarize:\n - how `platform_optimizations.databricks_clustering_strategy`,\n `liquid_clustering_enabled`, and `liquid_clustering_columns` are\n parsed and normalized;\n - where schema creation, data loading, unified tuning application,\n sorted ingestion, OPTIMIZE, and result metadata are ordered for\n native Databricks runs;\n - whether the current adapter has any path that would emit invalid\n Liquid + ZORDER or Liquid + partitioning combinations.\n\nDo not commit raw stdout logs. If command output is useful, store the raw\ncapture under `/tmp` and commit only the durable summary.\n\nCompleted 2026-05-26. Current Databricks documentation still says Liquid\nClustering replaces table partitioning and ZORDER for Delta layout; Delta\nsupport is GA on Databricks Runtime 15.2 and above; automatic Liquid\nClustering uses `CLUSTER BY AUTO`, requires Unity Catalog managed Delta\ntables and Databricks Runtime 15.4 LTS or above for intelligent key\nselection; manual Liquid supports up to four clustering keys; `OPTIMIZE`\nclusters Liquid tables; and `OPTIMIZE FULL` is available in Databricks\nRuntime 16.0 and above to force reclustering. Data-skipping docs still\nstate that ZORDER cannot be combined with Liquid Clustering.\n\nBranch-code trace: unified Databricks tuning is validated before schema\ncreation. For fresh native runs BenchBox creates schema, applies unified\ntuning/table hooks before data load, then COPY INTO loads data and\npost-load Delta OPTIMIZE is attempted per table. The old adapter behavior\ncould let Liquid fields override ZORDER during strategy resolution and\nhide invalid mixed effective configs. This implementation changes that to\nhard validation, records requested/resolved strategy, and records applied\nor skipped layout operations for Liquid AUTO/manual, ZORDER, OPTIMIZE, and\nANALYZE paths.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate Databricks Liquid Clustering guidance and current BenchBox execution semantics", "wid": "w0"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w0"], "notes": "Extend the profile/capability model so Databricks mapping is strategy\naware instead of hardcoded to Z-ORDER.\n\nRequired concepts:\n - logical profile: reusable TPC candidate source, currently `tpc-v1`;\n - physical rendering: platform-specific representation of the logical\n profile;\n - Databricks rendering ids:\n databricks_z_order\n databricks_liquid_manual\n databricks_liquid_auto\n - physical mechanisms:\n z_order\n liquid_clustering\n liquid_clustering_auto\n partitioning\n optimize\n optimize_full\n - unsupported/waived decisions with explicit reasons.\n\nThe mapping must answer:\n - for Z-ORDER rendering, which logical roles map to partitioning,\n clustering, distribution hints, and ZORDER operations;\n - for manual Liquid rendering, which logical roles become clustering\n keys, which legacy partition/ZORDER columns are folded into those\n keys, and which candidates exceed Databricks key limits;\n - for automatic Liquid rendering, how BenchBox reports logical intent\n when Databricks chooses keys asynchronously from workload history;\n - how distribution candidates should be represented when Databricks has\n no user-managed physical distribution key.\n\nResolve the default-selection decision before w2 starts. Unless the\nimplementation records an explicit migration decision with tests, docs,\nand release-note guidance, preserve the current `--tuning tuned`\nDatabricks behavior as the legacy Z-ORDER rendering and expose Liquid\nthrough an explicit rendering/template/strategy path.\n\nDo not claim that Databricks Liquid, Databricks Z-ORDER, DuckDB sorting,\nBigQuery clustering, Redshift sort/dist keys, and Snowflake clustering are\nperformance-equivalent. Report logical coverage and physical mechanism\ndifferences separately.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define Databricks physical rendering variants for the logical TPC profile", "wid": "w1"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1"], "notes": "Choose one of these implementation shapes and document the tradeoff:\n - generated Liquid variants from the logical profile; or\n - checked-in Databricks Liquid YAML variants, e.g.\n `examples/tunings/databricks/tpch_liquid_tuned.yaml` and\n `examples/tunings/databricks/tpcds_liquid_tuned.yaml`.\n\nMinimum requirements for Liquid variants:\n - `z_ordering_enabled: false`;\n - `liquid_clustering_enabled: true` or\n `databricks_clustering_strategy: liquid_clustering`;\n - no per-table legacy `partitioning` fields in Liquid-rendered Delta\n templates; logical partition candidates must be converted into Liquid\n clustering-key candidates or moved into a separate unsupported/waived\n decision list with rationale;\n - logical ZORDER candidates are explicitly converted into Liquid\n clustering-key candidates or moved into a separate unsupported/waived\n decision list with rationale;\n - key-count limits are enforced or explicitly handled;\n - template metadata records both `logical_tuning_profile_id` and\n `physical_rendering_id`.\n\nPreserve existing Z-ORDER YAML behavior for users who intentionally want\nlegacy comparisons. Do not silently reinterpret existing\n`tpch_tuned.yaml` or `tpcds_tuned.yaml` as Liquid Clustering unless a\nmigration decision is made explicit in docs, tests, and release notes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add explicit Databricks Liquid TPC template/rendering artifacts without breaking Z-ORDER templates", "wid": "w2"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1", "w2"], "notes": "Add validation that hard-fails invalid or misleading Databricks\neffective configurations. Do not allow waivers for runtime config\nconflicts that Databricks treats as mutually exclusive. At minimum,\nreject:\n - `liquid_clustering_enabled: true` with `z_ordering_enabled: true`;\n - Liquid rendering with non-empty `z_ordering_columns`;\n - Liquid rendering with per-table partitioning on Delta tables where\n legacy partitioning would conflict with Liquid Clustering;\n - `databricks_clustering_strategy: none` with Liquid fields;\n - generic `distribution` fields being reported as a real Databricks\n physical distribution mechanism.\n\nWaivers are only acceptable for logical profile candidates that cannot be\nrepresented in a selected rendering, not for contradictory effective\nDatabricks configuration fields.\n\nAdd this validation in the lowest shared layer that can see the full\nunified tuning configuration, then add adapter-level guardrails where\nruntime table format or Databricks-specific constraints are only known by\nthe adapter. Error messages must name the conflicting fields and the\nmigration path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enforce Databricks layout conflict validation across config, template, and adapter paths", "wid": "w3"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1", "w2", "w3"], "notes": "Trace and, if needed, adjust the native Databricks tuned lifecycle:\n - schema creation: table format, create-time CLUSTER BY, legacy\n partitioning, and table properties;\n - data load: COPY INTO and sorted-ingestion hooks;\n - post-load layout: ZORDER, ALTER TABLE CLUSTER BY, OPTIMIZE,\n OPTIMIZE FULL where applicable, and ANALYZE;\n - metadata capture: actual statements attempted and succeeded.\n\nThe implementation must prevent result metadata from saying a physical\nmechanism was applied when BenchBox only stored a config field or skipped\nthe operation. Capture skipped/failed layout operations separately from\nsuccessful operations so comparison tooling can treat them differently.\n\nThis work unit should include dry-run coverage where possible. Live\nDatabricks testing remains out of scope unless explicitly approved.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct Databricks tuning lifecycle so metadata matches operations actually applied", "wid": "w4"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1", "w4"], "notes": "Extend result/tuning metadata so `tuned` comparisons can distinguish\nlogical profile coverage from physical rendering.\n\nRequired fields or equivalent payload:\n - `logical_tuning_profile_id`\n - `logical_tuning_profile_version`\n - `physical_rendering_id`\n - `platform_physical_tuning_mechanisms`\n - `requested_databricks_clustering_strategy`\n - `resolved_databricks_clustering_strategy`\n - `applied_layout_operations`\n - `skipped_layout_operations`\n - `logical_profile_coverage`\n - `unmapped_logical_candidates`\n - `invalid_or_conflicting_tuning_fields`\n\nComparison semantics:\n - same logical profile + different physical rendering is comparable\n only as a cross-mechanism comparison, not as identical tuning;\n - Z-ORDER vs Liquid Clustering should segment cohorts unless the user\n explicitly asks to compare physical strategies;\n - basic-constraints fallback remains a coverage gap, not a tuned\n equivalent;\n - automatic Liquid Clustering must be reported as automatic/adaptive,\n not as proof that BenchBox selected a particular set of keys.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expose strategy-aware tuning metadata for fair cross-platform comparisons", "wid": "w5"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w2", "w3", "w4", "w5"], "notes": "Add focused fast/unit coverage before broad gates.\n\nRequired tests:\n - Databricks Liquid template validation passes for TPC-H and TPC-DS and\n reports `liquid_clustering` or `liquid_clustering_auto`, not\n `z_order`;\n - Databricks Z-ORDER templates still pass and continue reporting\n `z_order`;\n - mixed Liquid + ZORDER config fails with a clear error;\n - mixed Liquid + legacy partitioning fails with a clear error; logical\n partition candidates may be waived only outside the effective\n Liquid template/config, with rationale;\n - profile validation detects when Databricks distribution candidates\n are reported as real physical distribution keys;\n - result metadata distinguishes requested strategy, resolved strategy,\n and applied operations;\n - DuckDB, BigQuery, Redshift, and Snowflake mapping tests still prove\n the logical profile is platform-neutral and that physical mechanism\n differences are surfaced rather than hidden.\n\nNew routine unit tests must include top-level:\n `pytestmark = [pytest.mark.unit, pytest.mark.fast]`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests that fail on Liquid/Z-ORDER confusion and cross-platform parity regressions", "wid": "w6"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w5", "w6"], "notes": "Update user-facing docs so a benchmark consumer can answer:\n - whether a Databricks run used Z-ORDER, manual Liquid Clustering, or\n automatic Liquid Clustering;\n - how legacy TPC Z-ORDER/partition candidates map into Liquid keys;\n - why Databricks distribution candidates are logical locality\n candidates, not user-managed physical distribution keys;\n - when Z-ORDER and Liquid runs should be compared as different physical\n strategies rather than equivalent tuned runs;\n - which Databricks Runtime or Unity Catalog assumptions apply.\n\nUpdate:\n - `docs/usage/tpc-tuning-profiles.md`\n - `docs/usage/cloud-sorted-ingestion.md`\n - `examples/tunings/README.md`\n - Databricks TPC YAML metadata or adjacent docs for both legacy Z-ORDER\n and Liquid variants.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs and examples with Databricks migration and comparison guidance", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "implement Databricks-only behavior in platform-agnostic base classes without guards", "instead": "keep logic adapter-scoped where possible", "why": "it risks cross-platform regressions"}, {"dont": "ship strategy parsing without conflict validation", "instead": "fail fast with clear remediation", "why": "mixed config states become non-reproducible"}, {"dont": "silently reinterpret existing Z-ORDER configs as liquid clustering", "instead": "require explicit strategy selection", "why": "it changes benchmark semantics"}], "approach": "Extend the unified tuning model with an explicit Databricks clustering strategy\nrepresentation rather than inferring from generic sorting fields. Apply the strategy\nin Databricks adapter lifecycle points where table format/state is known.\n\nKeep precedence rules explicit: liquid clustering, z-order, and none must be mutually\nunderstandable with clear conflict handling and warnings.\n\nAdd result metadata so downstream analysis can segment runs by applied strategy.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [], "deps": ["cloud-dw-sorted-data-ingestion", "ctas-sort-cloud-platforms"], "description": "Expand Databricks tuning support so Liquid Clustering is a first-class,\nconfigurable strategy in BenchBox, not a secondary/implicit path behind\nZ-ORDER/OPTIMIZE workflows.\n\nRationale:\n- Databricks guidance is increasingly steering users toward Liquid Clustering.\n- Current BenchBox tuning emphasizes Z-ORDER and OPTIMIZE behaviors.\n- Users need direct control and comparability for Liquid Clustering in the\n unified tuning model and CLI/platform-option workflows.\n\nScope:\n- Add explicit Liquid Clustering representation in tuning configuration.\n- Implement Databricks adapter execution paths for create-time and post-load\n Liquid Clustering operations.\n- Preserve backwards compatibility for existing Z-ORDER-based configs.\n- Expose strategy metadata in results to support fair comparisons.", "id": "databricks-liquid-clustering-tuning-support", "preserves": ["Databricks adapter error messages remain actionable when strategy prerequisites are missing", "Dry-run output remains deterministic and includes all emitted Databricks tuning actions", "Existing Databricks runs using Z-ORDER tuning continue to execute without config migration", "Non-Databricks platforms ignore Databricks-specific liquid clustering settings cleanly"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/snowflake.py"}], "state": "done", "title": "Add first-class Databricks Liquid Clustering support in BenchBox tuning configurations", "verifications": [{"command": "uv run _project/scripts/validate_todo.py _project/DONE/physical-tuning/active/databricks-liquid-clustering-tuning-support.yaml", "description": "New TODO validates against schema", "expected": "Validation passed", "item_id": "databricks-liquid-clustering-tuning-support", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_databricks_liquid_clustering.py -v --tb=short", "description": "Databricks liquid clustering tests pass", "expected": "all tests pass, 0 failures", "item_id": "databricks-liquid-clustering-tuning-support", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -k databricks -v --tb=short", "description": "Existing Databricks adapter tests remain green", "expected": "all tests pass, 0 failures", "item_id": "databricks-liquid-clustering-tuning-support", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define tuning schema updates for Databricks liquid clustering strategy and validation rules", "wid": "w1"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add config parsing/normalization for liquid clustering in CLI and tuning loader paths", "wid": "w2"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement Databricks adapter support for liquid clustering DDL/DML lifecycle operations", "wid": "w3"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Maintain backward compatibility path for existing Z-ORDER configs with explicit precedence rules", "wid": "w4"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add result metadata fields for applied Databricks clustering strategy and maintenance operations", "wid": "w5"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w3", "w4", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit/integration tests for config validation and adapter SQL/action generation", "wid": "w6"}, {"evidence": null, "item_id": "databricks-liquid-clustering-tuning-support", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs/examples for liquid clustering usage, migration guidance, and A/B comparison with Z-ORDER", "wid": "w7"}], "worktree": "physical-tuning"} +{"anti_patterns": [{"dont": "add broad automatic template rewriting before validation and golden tests can prove every emitted field.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "allow Liquid Clustering and Z-ORDER fields in the same effective config without a hard validation error or a documented explicit waiver.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "claim cross-platform performance equivalence from logical profile coverage alone.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "keep hardcoding Databricks profile mechanisms to `z_order`; the mapping must depend on the selected physical rendering.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "report Databricks `distribution` as a real physical distribution key; Databricks distribution candidates are logical locality evidence.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "silently rename or reinterpret existing Databricks TPC tuned YAMLs as Liquid Clustering; that would change historical comparison semantics.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Treat Databricks Liquid Clustering as a strategy-aware physical rendering of\nthe existing logical TPC profile. First revalidate current Databricks docs and\nBenchBox execution order, then model rendering ids and mechanism semantics in\nthe profile validation layer. Add Liquid-specific templates or generated\nartifacts only after conflict validation is explicit. Finally, wire result\nmetadata and tests so comparisons cannot collapse Z-ORDER, Liquid, and\nfallback tuning into one misleading `tuned` bucket.\n\nPrefer small, reviewable layers:\n 1. strategy-aware mapping data model;\n 2. Databricks Liquid template/rendering artifacts;\n 3. config and adapter conflict validation;\n 4. lifecycle/operation metadata correctness;\n 5. docs and comparison semantics.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-26T00:00:00Z", "completed_pr": null, "created_at": "2026-05-26T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "databricks-tpc-liquid-clustering-profile-rendering", "id": 139, "reason": "Requires cloud credentials and explicit live-test approval.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Live Databricks warehouse validation for Liquid TPC is already covered by open item databricks-liquid-clustering-tuning-review-20260526.", "summary": "Run live Databricks warehouse validation for Liquid TPC-H/TPC-DS"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "databricks-tpc-liquid-clustering-profile-rendering", "id": 140, "reason": "This TODO must keep the Databricks Liquid rendering gap focused while preserving generalized profile semantics.", "resolution": "promoted", "resolved_item": "generate-and-certify-bigquery-redshift-and-snowflake-tpc-tun", "resolved_reason": null, "summary": "Generate and certify BigQuery, Redshift, and Snowflake TPC tuned templates from the logical profile"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "databricks-tpc-liquid-clustering-profile-rendering", "id": 141, "reason": "Metadata must land first so the UI warning has a stable contract to consume.", "resolution": "promoted", "resolved_item": "build-a-comparison-ui-cohort-warning-for-cross-mechanism-tun", "resolved_reason": null, "summary": "Build a comparison-UI cohort warning for cross-mechanism tuned results"}], "deps": ["generalize-tpc-tuning-base"], "description": "Follow-up to generalize-tpc-tuning-base. The generalized TPC profile work\ncorrectly moved Databricks TPC-H/TPC-DS tuning columns into a reusable\nlogical profile, but review exposed a Databricks-specific blind spot that can\ncorrupt tuned-run comparisons: the current checked-in Databricks TPC examples\nstill render the logical profile through the legacy Z-ORDER path, while\nmodern Databricks guidance treats Liquid Clustering as the preferred table\nlayout for new Delta tables.\n\nThe goal is not to make Databricks special again. The goal is to separate\nthree concerns cleanly so the TPC tuning base stays reusable across platforms:\nlogical workload candidates, platform physical rendering strategy, and\nexecution/result metadata. Databricks must support at least Z-ORDER and\nLiquid Clustering as distinct renderings of the same logical TPC profile, and\nresult comparisons must expose which rendering was actually requested and\napplied.\n\nCurrent external evidence checked on 2026-05-26:\n- Databricks liquid clustering docs describe Liquid Clustering as a data\n layout optimization that replaces table partitioning and ZORDER, supports\n CREATE/ALTER TABLE CLUSTER BY, and supports CLUSTER BY AUTO for automatic\n key selection.\n Source: https://docs.databricks.com/gcp/en/delta/clustering\n- Databricks optimize-layout docs recommend Liquid Clustering instead of\n partitions, ZORDER, or other layout approaches; OPTIMIZE groups data by\n Liquid Clustering keys for clustered tables.\n Source: https://docs.databricks.com/aws/en/delta/optimize\n- Databricks data-skipping docs state that ZORDER cannot be used with Liquid\n Clustering and recommend Liquid Clustering for new tables.\n Source: https://docs.databricks.com/gcp/en/delta/data-skipping\n\nCurrent BenchBox facts to revalidate before implementation:\n- `TuningType.LIQUID_CLUSTERING` and Databricks liquid configuration fields\n already exist in `benchbox/core/tuning/interface.py`.\n- `DatabricksAdapter` can resolve `databricks_clustering_strategy` and emit\n `ALTER TABLE ... CLUSTER BY (...)`, but the TPC profile capability mapping\n currently reports Databricks logical locality as `z_order`.\n- The checked-in Databricks TPC YAMLs set `z_ordering_enabled: true` and keep\n per-table `partitioning` fields, which is not a valid Liquid Clustering\n rendering without a deliberate conversion rule.\n- Unified tuning application currently runs before data loading for fresh\n native tables, while Databricks post-load layout operations and metadata\n need to distinguish create-time `CLUSTER BY`, post-load `OPTIMIZE`, and\n Z-ORDER statements.", "id": "databricks-tpc-liquid-clustering-profile-rendering", "preserves": ["DuckDB tuned templates and mappings remain free of Databricks vocabulary such as Z-ORDER or Liquid Clustering.", "Existing Databricks Z-ORDER tuned templates remain available for legacy strategy comparisons unless a migration is explicitly approved.", "Liquid Clustering, Z-ORDER, and basic-constraints fallback are distinguishable in result metadata and comparison semantics.", "New routine tests carry explicit `unit` and `fast` pytest markers.", "No live Databricks/cloud tests run without explicit user approval.", "No result metadata claims a tuning mechanism was applied unless BenchBox attempted and recorded that operation.", "The logical TPC profile remains platform-neutral; Databricks is one renderer, not the source of truth."], "prior_art": [{"concept": "_project/DONE/physical-tuning/active/databricks-liquid-clustering-tuning-support.yaml: prior Databricks Liquid implementation TODO - supersede its broad completion claim with profile/rendering validation", "decision": "supersede", "path": "_project/DONE/physical-tuning/active/databricks-liquid-clustering-tuning-support.yaml"}, {"concept": "benchbox/cli/commands/run.py: current Databricks strategy and liquid-column platform-option normalization - extend conflict handling", "decision": "extend", "path": "benchbox/cli/commands/run.py"}, {"concept": "benchbox/cli/config.py: current Databricks sample tuning defaults - update only with an explicit legacy-vs-Liquid default decision", "decision": "reuse", "path": "benchbox/cli/config.py"}, {"concept": "benchbox/cli/dryrun.py: existing dry-run command rendering for sorted ingestion and platform options - keep preview truthful", "decision": "reuse", "path": "benchbox/cli/dryrun.py"}, {"concept": "benchbox/core/tuning/interface.py: existing Liquid Clustering config fields and TuningType - reuse and tighten validation", "decision": "reuse", "path": "benchbox/core/tuning/interface.py"}, {"concept": "benchbox/core/tuning/platform_capabilities.py: current logical-role to platform-mechanism mapping - make strategy-aware", "decision": "reuse", "path": "benchbox/core/tuning/platform_capabilities.py"}, {"concept": "benchbox/core/tuning/profile_validation.py: existing template/profile validation and metadata projection - extend with physical rendering ids", "decision": "extend", "path": "benchbox/core/tuning/profile_validation.py"}, {"concept": "benchbox/core/tuning/profiles/tpc.yaml: existing logical TPC tuning profile data - extend rather than replace", "decision": "extend", "path": "benchbox/core/tuning/profiles/tpc.yaml"}, {"concept": "benchbox/platforms/base/sorted_ingestion.py: existing strategy/capability matrix pattern - reuse for method capability semantics where applicable", "decision": "reuse", "path": "benchbox/platforms/base/sorted_ingestion.py"}, {"concept": "benchbox/platforms/databricks/adapter.py: existing Z-ORDER and Liquid Clustering operation hooks - make lifecycle and metadata behavior truthful", "decision": "reuse", "path": "benchbox/platforms/databricks/adapter.py"}, {"concept": "docs/usage/tpc-tuning-profiles.md: current logical profile comparison semantics - expand with Databricks strategy distinctions", "decision": "reuse", "path": "docs/usage/tpc-tuning-profiles.md"}, {"concept": "tests/unit/cli/test_run_command_branches.py and tests/unit/cli/test_cli_dryrun.py: current CLI override and dry-run coverage - extend for Liquid conflict validation", "decision": "extend", "path": "tests/unit/cli/test_run_command_branches.py"}, {"concept": "tests/unit/platforms/test_databricks_adapter.py and tests/unit/platforms/test_databricks_adapter_coverage.py: current Databricks strategy precedence coverage - update expected conflicts", "decision": "reuse", "path": "tests/unit/platforms/test_databricks_adapter.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/_binaries/"}, {"kind": "do_not_modify", "path_glob": "benchmark_runs/"}, {"kind": "do_not_modify", "path_glob": "docs/_build/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/main/active/databricks-tpc-liquid-clustering-profile-rendering.yaml"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/databricks-tpc-liquid-clustering-profile-rendering.yaml"}, {"kind": "only_modify", "path_glob": "_project/scripts/tuning_profile_check.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/config.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/tuning.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/interface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/platform_capabilities.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/profile_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/profiles/tpc.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/workload_profiles.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/result_capture.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/sorted_ingestion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/databricks/adapter.py"}, {"kind": "only_modify", "path_glob": "docs/reference/cli/run.md"}, {"kind": "only_modify", "path_glob": "docs/usage/cloud-sorted-ingestion.md"}, {"kind": "only_modify", "path_glob": "docs/usage/databricks-liquid-clustering.md"}, {"kind": "only_modify", "path_glob": "docs/usage/tpc-tuning-profiles.md"}, {"kind": "only_modify", "path_glob": "examples/tunings/README.md"}, {"kind": "only_modify", "path_glob": "examples/tunings/databricks/"}, {"kind": "only_modify", "path_glob": "examples/tunings/duckdb/"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_databricks_tuning_examples.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_interactive_dry_run.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_platform_hooks.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_run_command_branches.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tuning/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tuning/generators/test_spark_family_ddl.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Model Databricks TPC Liquid Clustering as a first-class reusable profile rendering", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/DONE/main/active/databricks-tpc-liquid-clustering-profile-rendering.yaml", "description": "New TODO validates against schema", "expected": "Validation passed", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after adding the dependency edge", "expected": "passes", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python _project/scripts/tuning_profile_check.py --benchmarks tpch,tpcds --platforms databricks,duckdb --strict", "description": "TPC profile validation covers both legacy Z-ORDER and Databricks Liquid renderings", "expected": "passes, with Databricks mechanisms reflecting the selected rendering", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -n 0 tests/unit/core/tuning/test_tpc_workload_profiles.py tests/unit/core/tuning/test_tpc_tuning_template_parity.py tests/unit/cli/test_databricks_tuning_examples.py -q", "description": "Strategy-aware tuning profile unit tests pass", "expected": "all tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -n 0 tests/unit/platforms -k 'databricks and (liquid or tuning or zorder)' -q", "description": "Databricks tuning strategy unit tests pass", "expected": "all relevant tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest -n 0 tests/unit/cli/test_run_command_branches.py tests/unit/cli/test_cli_dryrun.py tests/unit/cli/test_interactive_dry_run.py -q", "description": "CLI and dry-run strategy override tests pass", "expected": "all tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest -n 0 tests/unit/core/tuning/test_platform_optimization_config.py tests/unit/core/tuning/generators/test_spark_family_ddl.py -q", "description": "Core tuning config and Spark-family DDL tests pass", "expected": "all tests pass", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 7}, {"command": "make pr-preflight", "description": "Final PR gate before publication when implementing this TODO", "expected": "passes", "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "last_result": null, "last_run": null, "seq": 8}], "work": [{"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": [], "notes": "Re-open the three Databricks documentation sources in `description`\nand record a compact dated summary in docs or TODO completion notes:\n - minimum Databricks Runtime/version constraints for manual Liquid\n Clustering, automatic Liquid Clustering, and OPTIMIZE FULL;\n - compatibility rules for Liquid Clustering with legacy partitioning\n and ZORDER;\n - recommended migration from Hive-style partitioning and ZORDER to\n Liquid Clustering keys;\n - current syntax for CREATE TABLE, ALTER TABLE, CLUSTER BY AUTO, and\n post-load optimization.\n\nThis is a hard freshness gate. Do not proceed with implementation from\nthe cited 2026-05-26 evidence alone; if Databricks guidance changed,\nupdate the TODO completion notes or docs with the checked date, URLs,\nand the changed behavior before touching runtime code.\n\nRe-read the current branch code and summarize:\n - how `platform_optimizations.databricks_clustering_strategy`,\n `liquid_clustering_enabled`, and `liquid_clustering_columns` are\n parsed and normalized;\n - where schema creation, data loading, unified tuning application,\n sorted ingestion, OPTIMIZE, and result metadata are ordered for\n native Databricks runs;\n - whether the current adapter has any path that would emit invalid\n Liquid + ZORDER or Liquid + partitioning combinations.\n\nDo not commit raw stdout logs. If command output is useful, store the raw\ncapture under `/tmp` and commit only the durable summary.\n\nCompleted 2026-05-26. Current Databricks documentation still says Liquid\nClustering replaces table partitioning and ZORDER for Delta layout; Delta\nsupport is GA on Databricks Runtime 15.2 and above; automatic Liquid\nClustering uses `CLUSTER BY AUTO`, requires Unity Catalog managed Delta\ntables and Databricks Runtime 15.4 LTS or above for intelligent key\nselection; manual Liquid supports up to four clustering keys; `OPTIMIZE`\nclusters Liquid tables; and `OPTIMIZE FULL` is available in Databricks\nRuntime 16.0 and above to force reclustering. Data-skipping docs still\nstate that ZORDER cannot be combined with Liquid Clustering.\n\nBranch-code trace: unified Databricks tuning is validated before schema\ncreation. For fresh native runs BenchBox creates schema, applies unified\ntuning/table hooks before data load, then COPY INTO loads data and\npost-load Delta OPTIMIZE is attempted per table. The old adapter behavior\ncould let Liquid fields override ZORDER during strategy resolution and\nhide invalid mixed effective configs. This implementation changes that to\nhard validation, records requested/resolved strategy, and records applied\nor skipped layout operations for Liquid AUTO/manual, ZORDER, OPTIMIZE, and\nANALYZE paths.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate Databricks Liquid Clustering guidance and current BenchBox execution semantics", "wid": "w0"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w0"], "notes": "Extend the profile/capability model so Databricks mapping is strategy\naware instead of hardcoded to Z-ORDER.\n\nRequired concepts:\n - logical profile: reusable TPC candidate source, currently `tpc-v1`;\n - physical rendering: platform-specific representation of the logical\n profile;\n - Databricks rendering ids:\n databricks_z_order\n databricks_liquid_manual\n databricks_liquid_auto\n - physical mechanisms:\n z_order\n liquid_clustering\n liquid_clustering_auto\n partitioning\n optimize\n optimize_full\n - unsupported/waived decisions with explicit reasons.\n\nThe mapping must answer:\n - for Z-ORDER rendering, which logical roles map to partitioning,\n clustering, distribution hints, and ZORDER operations;\n - for manual Liquid rendering, which logical roles become clustering\n keys, which legacy partition/ZORDER columns are folded into those\n keys, and which candidates exceed Databricks key limits;\n - for automatic Liquid rendering, how BenchBox reports logical intent\n when Databricks chooses keys asynchronously from workload history;\n - how distribution candidates should be represented when Databricks has\n no user-managed physical distribution key.\n\nResolve the default-selection decision before w2 starts. Unless the\nimplementation records an explicit migration decision with tests, docs,\nand release-note guidance, preserve the current `--tuning tuned`\nDatabricks behavior as the legacy Z-ORDER rendering and expose Liquid\nthrough an explicit rendering/template/strategy path.\n\nDo not claim that Databricks Liquid, Databricks Z-ORDER, DuckDB sorting,\nBigQuery clustering, Redshift sort/dist keys, and Snowflake clustering are\nperformance-equivalent. Report logical coverage and physical mechanism\ndifferences separately.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define Databricks physical rendering variants for the logical TPC profile", "wid": "w1"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1"], "notes": "Choose one of these implementation shapes and document the tradeoff:\n - generated Liquid variants from the logical profile; or\n - checked-in Databricks Liquid YAML variants, e.g.\n `examples/tunings/databricks/tpch_liquid_tuned.yaml` and\n `examples/tunings/databricks/tpcds_liquid_tuned.yaml`.\n\nMinimum requirements for Liquid variants:\n - `z_ordering_enabled: false`;\n - `liquid_clustering_enabled: true` or\n `databricks_clustering_strategy: liquid_clustering`;\n - no per-table legacy `partitioning` fields in Liquid-rendered Delta\n templates; logical partition candidates must be converted into Liquid\n clustering-key candidates or moved into a separate unsupported/waived\n decision list with rationale;\n - logical ZORDER candidates are explicitly converted into Liquid\n clustering-key candidates or moved into a separate unsupported/waived\n decision list with rationale;\n - key-count limits are enforced or explicitly handled;\n - template metadata records both `logical_tuning_profile_id` and\n `physical_rendering_id`.\n\nPreserve existing Z-ORDER YAML behavior for users who intentionally want\nlegacy comparisons. Do not silently reinterpret existing\n`tpch_tuned.yaml` or `tpcds_tuned.yaml` as Liquid Clustering unless a\nmigration decision is made explicit in docs, tests, and release notes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add explicit Databricks Liquid TPC template/rendering artifacts without breaking Z-ORDER templates", "wid": "w2"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1", "w2"], "notes": "Add validation that hard-fails invalid or misleading Databricks\neffective configurations. Do not allow waivers for runtime config\nconflicts that Databricks treats as mutually exclusive. At minimum,\nreject:\n - `liquid_clustering_enabled: true` with `z_ordering_enabled: true`;\n - Liquid rendering with non-empty `z_ordering_columns`;\n - Liquid rendering with per-table partitioning on Delta tables where\n legacy partitioning would conflict with Liquid Clustering;\n - `databricks_clustering_strategy: none` with Liquid fields;\n - generic `distribution` fields being reported as a real Databricks\n physical distribution mechanism.\n\nWaivers are only acceptable for logical profile candidates that cannot be\nrepresented in a selected rendering, not for contradictory effective\nDatabricks configuration fields.\n\nAdd this validation in the lowest shared layer that can see the full\nunified tuning configuration, then add adapter-level guardrails where\nruntime table format or Databricks-specific constraints are only known by\nthe adapter. Error messages must name the conflicting fields and the\nmigration path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enforce Databricks layout conflict validation across config, template, and adapter paths", "wid": "w3"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1", "w2", "w3"], "notes": "Trace and, if needed, adjust the native Databricks tuned lifecycle:\n - schema creation: table format, create-time CLUSTER BY, legacy\n partitioning, and table properties;\n - data load: COPY INTO and sorted-ingestion hooks;\n - post-load layout: ZORDER, ALTER TABLE CLUSTER BY, OPTIMIZE,\n OPTIMIZE FULL where applicable, and ANALYZE;\n - metadata capture: actual statements attempted and succeeded.\n\nThe implementation must prevent result metadata from saying a physical\nmechanism was applied when BenchBox only stored a config field or skipped\nthe operation. Capture skipped/failed layout operations separately from\nsuccessful operations so comparison tooling can treat them differently.\n\nThis work unit should include dry-run coverage where possible. Live\nDatabricks testing remains out of scope unless explicitly approved.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct Databricks tuning lifecycle so metadata matches operations actually applied", "wid": "w4"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w1", "w4"], "notes": "Extend result/tuning metadata so `tuned` comparisons can distinguish\nlogical profile coverage from physical rendering.\n\nRequired fields or equivalent payload:\n - `logical_tuning_profile_id`\n - `logical_tuning_profile_version`\n - `physical_rendering_id`\n - `platform_physical_tuning_mechanisms`\n - `requested_databricks_clustering_strategy`\n - `resolved_databricks_clustering_strategy`\n - `applied_layout_operations`\n - `skipped_layout_operations`\n - `logical_profile_coverage`\n - `unmapped_logical_candidates`\n - `invalid_or_conflicting_tuning_fields`\n\nComparison semantics:\n - same logical profile + different physical rendering is comparable\n only as a cross-mechanism comparison, not as identical tuning;\n - Z-ORDER vs Liquid Clustering should segment cohorts unless the user\n explicitly asks to compare physical strategies;\n - basic-constraints fallback remains a coverage gap, not a tuned\n equivalent;\n - automatic Liquid Clustering must be reported as automatic/adaptive,\n not as proof that BenchBox selected a particular set of keys.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expose strategy-aware tuning metadata for fair cross-platform comparisons", "wid": "w5"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w2", "w3", "w4", "w5"], "notes": "Add focused fast/unit coverage before broad gates.\n\nRequired tests:\n - Databricks Liquid template validation passes for TPC-H and TPC-DS and\n reports `liquid_clustering` or `liquid_clustering_auto`, not\n `z_order`;\n - Databricks Z-ORDER templates still pass and continue reporting\n `z_order`;\n - mixed Liquid + ZORDER config fails with a clear error;\n - mixed Liquid + legacy partitioning fails with a clear error; logical\n partition candidates may be waived only outside the effective\n Liquid template/config, with rationale;\n - profile validation detects when Databricks distribution candidates\n are reported as real physical distribution keys;\n - result metadata distinguishes requested strategy, resolved strategy,\n and applied operations;\n - DuckDB, BigQuery, Redshift, and Snowflake mapping tests still prove\n the logical profile is platform-neutral and that physical mechanism\n differences are surfaced rather than hidden.\n\nNew routine unit tests must include top-level:\n `pytestmark = [pytest.mark.unit, pytest.mark.fast]`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests that fail on Liquid/Z-ORDER confusion and cross-platform parity regressions", "wid": "w6"}, {"evidence": null, "item_id": "databricks-tpc-liquid-clustering-profile-rendering", "needs": ["w5", "w6"], "notes": "Update user-facing docs so a benchmark consumer can answer:\n - whether a Databricks run used Z-ORDER, manual Liquid Clustering, or\n automatic Liquid Clustering;\n - how legacy TPC Z-ORDER/partition candidates map into Liquid keys;\n - why Databricks distribution candidates are logical locality\n candidates, not user-managed physical distribution keys;\n - when Z-ORDER and Liquid runs should be compared as different physical\n strategies rather than equivalent tuned runs;\n - which Databricks Runtime or Unity Catalog assumptions apply.\n\nUpdate:\n - `docs/usage/tpc-tuning-profiles.md`\n - `docs/usage/cloud-sorted-ingestion.md`\n - `examples/tunings/README.md`\n - Databricks TPC YAML metadata or adjacent docs for both legacy Z-ORDER\n and Liquid variants.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs and examples with Databricks migration and comparison guidance", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-31T00:00:00Z", "completed_pr": null, "created_at": "2026-01-25T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement DataFrame versions of AI Primitives benchmark operations. The current\nbenchmark has **16 SQL queries** across 4 categories testing AI/ML functions:\n\n**Categories (16 queries total):**\n- Generative (4): text completion, customer profile summary, Q&A, SQL generation\n- NLP (5): sentiment analysis (single/batch), classification (priority/segment), entity extraction\n- Transform (4): summarization (short/long), translation, grammar correction\n- Embedding (3): single embedding, batch embeddings, large-dimension embeddings\n\n**DataFrame Implementation Approach:**\nThe SQL benchmark uses platform-specific AI functions (Snowflake Cortex, BigQuery ML,\nDatabricks AI). DataFrame implementations would use:\n\n- **PySpark MLlib**: Native ML functions, model inference pipelines\n- **Sentence Transformers**: For embeddings via Hugging Face integration\n- **Polars + NumPy**: Vector operations, embedding similarity\n- **Pandas + scikit-learn**: Classification, NLP pipelines\n\nNot all 16 SQL queries translate directly - generative queries require LLM APIs\nwhile embedding/vector operations map well to DataFrame native operations.", "id": "dataframe-ai-primitives", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Implement DataFrame Version of AI Primitives Benchmark", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-02T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Ensure DataFrame mode coexists with existing SQL mode. Define interaction\nbetween modes, migration path, and CLI interface for mode selection.", "id": "dataframe-backwards-compatibility-and-mode-coexistence", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "DataFrame Backwards Compatibility and Mode Coexistence", "verifications": [], "work": [], "worktree": "dataframe-phase0-compat"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change any adapter's empty-string/null outcome while consolidating", "why": "this is a structural move only."}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-17T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up to dataframe-csv-empty-string-null-uniform (#904) and its coverage\nfollow-up dataframe-csv-empty-string-parity-coverage (#936). #904 landed the\nempty-string-vs-null CSV coercion PER ADAPTER rather than as the single shared\nstep the original YAML specified. It is functionally equivalent today, but it\nmeans each DataFrame adapter carries its own copy of the contract \u2014 each is an\nindependent risk surface that can drift.\n\nConsolidate the coercion into one shared step in the common DataFrame CSV load\npath so the empty-string/null contract has a single source of truth. This is now\na safe refactor because #936 pins the contract across every installed adapter\n(datafusion/pyspark/pandas/polars/dask, plus the SQL<->DataFrame parity case), so\nthe parity tests are the behavior-preservation net.\n", "id": "dataframe-csv-coercion-single-shared-step", "preserves": ["Behavior-preserving refactor: no adapter's observable CSV-loading semantics change.", "The empty-string vs null CSV contract is unchanged for every adapter \u2014 proven by the #936 parity tests staying green."], "prior_art": [{"concept": "{'ref': 'benchbox/platforms/dataframe/benchmark_mixin.py (DataFrame CSV load path)', 'note': 'The common load path where the single shared coercion step should live.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/dataframe/polars_df.py (missing_utf8_is_empty_string) and the sibling per-adapter coercion sites', 'note': 'The per-adapter coercion this item consolidates; enumerate every site before extracting.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_dataframe_csv_empty_string_null_semantics.py (#936)', 'note': 'The cross-adapter parity net that must stay byte-identical-green through the refactor.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "planning", "title": "Consolidate per-adapter empty-string/null CSV coercion into one shared step", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_dataframe_csv_empty_string_null_semantics.py -q", "description": "Cross-adapter parity holds through the refactor", "expected": "all per-adapter + cross-surface parity cells pass unchanged; coercion now lives in one shared step", "item_id": "dataframe-csv-coercion-single-shared-step", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dataframe-csv-coercion-single-shared-step", "needs": [], "notes": "Enumerate where each adapter implements the empty-field -> \"\" vs null rule\n(null_marker semantics). Record file:line for each.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Map every per-adapter empty-string/null CSV coercion site", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-csv-coercion-single-shared-step", "needs": ["w1"], "notes": "Move the contract into a single shared step each adapter defers to; remove the\nper-adapter special-casing. Behavior-preserving move only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extract the coercion to one shared step in the common DataFrame CSV load path", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-csv-coercion-single-shared-step", "needs": ["w2"], "notes": "The #936 parametrized parity tests (per-adapter + SQL<->DataFrame CSV-path)\nmust stay green unchanged; add a test asserting no per-adapter coercion path\nremains if useful.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Confirm byte-identical behavior via the #936 parity tests", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "hardcode empty->'' regardless of null_marker", "why": "that would break null_marker='' benchmarks."}, {"INSTEAD": "(not specified)", "dont": "patch only PySpark and DataFusion in isolation", "why": "centralize so future adapters inherit the contract."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The empty-string-vs-null contract is centralized at the Parquet conversion layer\n(data_loader.py, strings_can_be_null = null_marker is not None), and\nprefer_parquet=True is the default - so the DEFAULT path is consistent. But on the\nprefer_parquet=False CSV-direct path the contract is NOT uniform:\n - pandas_df.py:378 and polars_df.py:283 restore '' for empty fields when\n null_marker is None (dask/modin/cudf do likewise).\n - pyspark_df.py:461 returns directly after CSV load (Spark emits null for empty\n fields by default).\n - datafusion_df.py:511 documents null_marker as \"Unused\".\nSo for a null-marker-less benchmark (e.g. ClickBench), a prefer_parquet=False run\non PySpark or DataFusion emits NULL where the SQL reference (and the other\nadapters) emit \"\" - the exact Q6/Q17/Q18 divergence class the polars adapter's own\ncomment warns about. It is non-default, hence latent, but it is a real\ncross-surface fidelity gap and will silently mis-compare if that path is exercised.\n\nPrefer a SINGLE shared post-coercion (so future adapters inherit it) over\nper-adapter patches.", "id": "dataframe-csv-empty-string-null-uniform", "preserves": ["JoinOrder and other null_marker='' benchmarks still map empty -> NULL; the fix is null_marker-driven, not a blanket empty->'' for all.", "The production DataFrame run path is reused, not forked; the change is additive.", "prefer_parquet=True (default) load behavior is byte-unchanged for every adapter."], "prior_art": [{"concept": "{'ref': '_project/analysis/joinorder-synthetic-cross-surface-divergences.md and clickbench-cross-surface-divergences.md', 'note': 'Documents the null_marker root cause (resolve_csv_dialect) and the Q17/Q24/Q6/Q17/Q18 empty-string class.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/dataframe/data_loader.py:423', 'note': 'The Parquet-layer contract (strings_can_be_null = null_marker is not None) - the canonical semantics the CSV-direct path must match.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/dataframe/benchmark_mixin.py / shared_loading.py', 'note': 'Where a shared post-load coercion should live so PySpark/DataFusion inherit it without per-adapter code.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/dataframe/pandas_df.py:378, polars_df.py:283', 'note': \"The correct empty-string restoration (fillna('')/missing_utf8_is_empty_string) to generalize into one shared step.\"}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Normalize empty-string-vs-null semantics across all DataFrame adapters on the CSV-direct load path", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_dataframe_csv_empty_string_null_semantics.py -q", "description": "CSV-direct empty-string parity on the two lagging adapters", "expected": "0 empty-string-vs-null divergences; JoinOrder still nulls empties", "item_id": "dataframe-csv-empty-string-null-uniform", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dataframe-csv-empty-string-null-uniform", "needs": [], "notes": "Add one shared post-load step (in benchmark_mixin/shared_loading) that, when\nnull_marker is None, restores '' for empty string-typed fields - applied\nuniformly across adapters, including PySpark (emptyValue/post-read restore) and\nDataFusion. Resolve null_marker via resolve_csv_dialect (the benchmark\ninstance's marker), not a file-extension heuristic. Keep prefer_parquet=True\nbehavior byte-unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a shared null_marker-aware empty-string post-coercion on the CSV-direct path", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-null-uniform", "needs": ["w1"], "notes": "Extend coverage to run a null-marker-less benchmark (ClickBench) cross-surface\non PySpark and DataFusion with prefer_parquet=False and assert zero\nempty-string-vs-null divergences. Add a focused regression naming the Q6/Q17/\nQ18 pattern. Confirm JoinOrder (null_marker=\"\") still nulls empties (must not\nregress the opposite direction).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate the fix: ClickBench cross-surface on PySpark/DataFusion with prefer_parquet=False", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "fix an adapter parity gap inside this test-only item", "why": "surface it as a defect TODO."}, {"INSTEAD": "(not specified)", "dont": "introduce a new benchmark or heavyweight fixture for the cross-surface case; reuse a small existing one.", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-csv-empty-string-parity-coverage", "id": 367, "reason": "Functionally equivalent today; consolidation is a separate maintainability item, not coverage.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Exact match: open item dataframe-csv-coercion-single-shared-step already tracks this consolidation.", "summary": "Refactor the per-adapter coercion into the single shared step #904's YAML originally specified."}], "deps": [], "description": "Follow-up to dataframe-csv-empty-string-null-uniform (PR #904, DONE). The fix\nmade empty-string-vs-null CSV loading uniform, but the regression test covers\nonly DataFusion and PySpark. The success metric claims uniformity across ALL CSV\nadapters; pandas/polars/dask/modin/cudf empty-string -> \"\" (not null) parity is\nasserted nowhere new \u2014 it rests on untouched legacy behavior with no pin. #904\nalso landed the coercion per-adapter rather than as the single shared step the\nYAML specified (functionally equivalent, but it means each adapter is its own\nrisk surface). This item pins the contract for the remaining adapters and adds a\nCSV-path cross-surface case.\n\nTest-only by intent: if a parity gap surfaces for any adapter, STOP and file a\ndefect TODO \u2014 do not fix adapter behavior inside this coverage item.", "id": "dataframe-csv-empty-string-parity-coverage", "preserves": ["Fast lane stays green on a standard dev environment (availability skips for absent adapters).", "No change to adapter CSV-loading behavior \u2014 this is coverage only."], "prior_art": [{"concept": "{'ref': 'tests/integration/test_dataframe_csv_empty_string_null_semantics.py', 'note': 'The #904 regression test covering DataFusion+PySpark; parametrize its existing assertions across the remaining adapters rather than authoring a new harness.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "tests/integration/test_dataframe_csv_empty_string_null_semantics.py"}], "state": "done", "title": "Pin empty-string/null CSV parity across all DataFrame adapters, not just DataFusion+PySpark", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_dataframe_csv_empty_string_null_semantics.py -q", "description": "Empty-string/null parity holds across installed adapters", "expected": "per-adapter parametrized cells pass for pandas/polars/dask/modin; cudf skipped if absent", "item_id": "dataframe-csv-empty-string-parity-coverage", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": [], "notes": "grep tests/integration/test_dataframe_csv_empty_string_null_semantics.py for\nadapter imports/cases (review-time state: only DATAFUSION_DF_AVAILABLE and\nPYSPARK_AVAILABLE). If a later commit already added adapters, narrow w1 to the\nremaining ones. Record the live coverage in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm on fresh develop that the #904 test still covers only DataFusion+PySpark", "wid": "w0"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": ["w0"], "notes": "Extend the existing test to assert, per adapter, the null-marker-dependent\ncontract: a bare empty field loads as \"\" when csv_null_marker is None, and\nas null when csv_null_marker == \"\". This is not a single uniform outcome \u2014\n\"uniform\" means the same null-marker-conditioned rule holds identically\nacross adapters, not that a bare empty field always loads as null. Guard\ncudf and any non-installed adapter with availability skips so the fast lane\nstays green on a standard dev env.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Parametrize empty-string/null CSV semantics across pandas, polars, dask, modin (cudf availability-skipped)", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": [], "notes": "Add at least one cross-surface equivalence case that forces the CSV load path\n(prefer_parquet=False) so SQL<->DataFrame empty-string/null parity is exercised\nend-to-end, not only per-adapter loading. Reuse an existing small cross-surface\nfixture rather than a new benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a cross-surface CSV-path case (prefer_parquet=False) exercising SQL<->DataFrame parity", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": ["w0", "w1", "w2"], "notes": "Ensure the parametrized cells are fast-marked and collected by the required\ncode-test fast lane (or an explicit node if a marker excludes them). Record\nthe lane membership in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm the new cells run in the required fast lane", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change any adapter's empty-string/null outcome while consolidating", "instead": "(not specified)", "why": "this is a structural move only."}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-17T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up to dataframe-csv-empty-string-null-uniform (#904) and its coverage\nfollow-up dataframe-csv-empty-string-parity-coverage (#936). #904 landed the\nempty-string-vs-null CSV coercion PER ADAPTER rather than as the single shared\nstep the original YAML specified. It is functionally equivalent today, but it\nmeans each DataFrame adapter carries its own copy of the contract \u2014 each is an\nindependent risk surface that can drift.\n\nConsolidate the coercion into one shared step in the common DataFrame CSV load\npath so the empty-string/null contract has a single source of truth. This is now\na safe refactor because #936 pins the contract across every installed adapter\n(datafusion/pyspark/pandas/polars/dask, plus the SQL<->DataFrame parity case), so\nthe parity tests are the behavior-preservation net.\n", "id": "dataframe-csv-coercion-single-shared-step", "preserves": ["Behavior-preserving refactor: no adapter's observable CSV-loading semantics change.", "The empty-string vs null CSV contract is unchanged for every adapter \u2014 proven by the #936 parity tests staying green."], "prior_art": [{"concept": "{'ref': 'benchbox/platforms/dataframe/benchmark_mixin.py (DataFrame CSV load path)', 'note': 'The common load path where the single shared coercion step should live.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/dataframe/polars_df.py (missing_utf8_is_empty_string) and the sibling per-adapter coercion sites', 'note': 'The per-adapter coercion this item consolidates; enumerate every site before extracting.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/integration/test_dataframe_csv_empty_string_null_semantics.py (#936)', 'note': 'The cross-adapter parity net that must stay byte-identical-green through the refactor.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "planning", "title": "Consolidate per-adapter empty-string/null CSV coercion into one shared step", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_dataframe_csv_empty_string_null_semantics.py -q", "description": "Cross-adapter parity holds through the refactor", "expected": "all per-adapter + cross-surface parity cells pass unchanged; coercion now lives in one shared step", "item_id": "dataframe-csv-coercion-single-shared-step", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dataframe-csv-coercion-single-shared-step", "needs": [], "notes": "Enumerate where each adapter implements the empty-field -> \"\" vs null rule\n(null_marker semantics). Record file:line for each.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Map every per-adapter empty-string/null CSV coercion site", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-csv-coercion-single-shared-step", "needs": ["w1"], "notes": "Move the contract into a single shared step each adapter defers to; remove the\nper-adapter special-casing. Behavior-preserving move only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extract the coercion to one shared step in the common DataFrame CSV load path", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-csv-coercion-single-shared-step", "needs": ["w2"], "notes": "The #936 parametrized parity tests (per-adapter + SQL<->DataFrame CSV-path)\nmust stay green unchanged; add a test asserting no per-adapter coercion path\nremains if useful.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Confirm byte-identical behavior via the #936 parity tests", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "hardcode empty->'' regardless of null_marker", "instead": "(not specified)", "why": "that would break null_marker='' benchmarks."}, {"dont": "patch only PySpark and DataFusion in isolation", "instead": "(not specified)", "why": "centralize so future adapters inherit the contract."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The empty-string-vs-null contract is centralized at the Parquet conversion layer\n(data_loader.py, strings_can_be_null = null_marker is not None), and\nprefer_parquet=True is the default - so the DEFAULT path is consistent. But on the\nprefer_parquet=False CSV-direct path the contract is NOT uniform:\n - pandas_df.py:378 and polars_df.py:283 restore '' for empty fields when\n null_marker is None (dask/modin/cudf do likewise).\n - pyspark_df.py:461 returns directly after CSV load (Spark emits null for empty\n fields by default).\n - datafusion_df.py:511 documents null_marker as \"Unused\".\nSo for a null-marker-less benchmark (e.g. ClickBench), a prefer_parquet=False run\non PySpark or DataFusion emits NULL where the SQL reference (and the other\nadapters) emit \"\" - the exact Q6/Q17/Q18 divergence class the polars adapter's own\ncomment warns about. It is non-default, hence latent, but it is a real\ncross-surface fidelity gap and will silently mis-compare if that path is exercised.\n\nPrefer a SINGLE shared post-coercion (so future adapters inherit it) over\nper-adapter patches.", "id": "dataframe-csv-empty-string-null-uniform", "preserves": ["JoinOrder and other null_marker='' benchmarks still map empty -> NULL; the fix is null_marker-driven, not a blanket empty->'' for all.", "The production DataFrame run path is reused, not forked; the change is additive.", "prefer_parquet=True (default) load behavior is byte-unchanged for every adapter."], "prior_art": [{"concept": "{'ref': '_project/analysis/joinorder-synthetic-cross-surface-divergences.md and clickbench-cross-surface-divergences.md', 'note': 'Documents the null_marker root cause (resolve_csv_dialect) and the Q17/Q24/Q6/Q17/Q18 empty-string class.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/dataframe/data_loader.py:423', 'note': 'The Parquet-layer contract (strings_can_be_null = null_marker is not None) - the canonical semantics the CSV-direct path must match.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/dataframe/benchmark_mixin.py / shared_loading.py', 'note': 'Where a shared post-load coercion should live so PySpark/DataFusion inherit it without per-adapter code.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/dataframe/pandas_df.py:378, polars_df.py:283', 'note': \"The correct empty-string restoration (fillna('')/missing_utf8_is_empty_string) to generalize into one shared step.\"}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Normalize empty-string-vs-null semantics across all DataFrame adapters on the CSV-direct load path", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_dataframe_csv_empty_string_null_semantics.py -q", "description": "CSV-direct empty-string parity on the two lagging adapters", "expected": "0 empty-string-vs-null divergences; JoinOrder still nulls empties", "item_id": "dataframe-csv-empty-string-null-uniform", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dataframe-csv-empty-string-null-uniform", "needs": [], "notes": "Add one shared post-load step (in benchmark_mixin/shared_loading) that, when\nnull_marker is None, restores '' for empty string-typed fields - applied\nuniformly across adapters, including PySpark (emptyValue/post-read restore) and\nDataFusion. Resolve null_marker via resolve_csv_dialect (the benchmark\ninstance's marker), not a file-extension heuristic. Keep prefer_parquet=True\nbehavior byte-unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a shared null_marker-aware empty-string post-coercion on the CSV-direct path", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-null-uniform", "needs": ["w1"], "notes": "Extend coverage to run a null-marker-less benchmark (ClickBench) cross-surface\non PySpark and DataFusion with prefer_parquet=False and assert zero\nempty-string-vs-null divergences. Add a focused regression naming the Q6/Q17/\nQ18 pattern. Confirm JoinOrder (null_marker=\"\") still nulls empties (must not\nregress the opposite direction).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate the fix: ClickBench cross-surface on PySpark/DataFusion with prefer_parquet=False", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fix an adapter parity gap inside this test-only item", "instead": "(not specified)", "why": "surface it as a defect TODO."}, {"dont": "introduce a new benchmark or heavyweight fixture for the cross-surface case; reuse a small existing one.", "instead": "(not specified)", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-csv-empty-string-parity-coverage", "id": 367, "reason": "Functionally equivalent today; consolidation is a separate maintainability item, not coverage.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Exact match: open item dataframe-csv-coercion-single-shared-step already tracks this consolidation.", "summary": "Refactor the per-adapter coercion into the single shared step #904's YAML originally specified."}], "deps": [], "description": "Follow-up to dataframe-csv-empty-string-null-uniform (PR #904, DONE). The fix\nmade empty-string-vs-null CSV loading uniform, but the regression test covers\nonly DataFusion and PySpark. The success metric claims uniformity across ALL CSV\nadapters; pandas/polars/dask/modin/cudf empty-string -> \"\" (not null) parity is\nasserted nowhere new \u2014 it rests on untouched legacy behavior with no pin. #904\nalso landed the coercion per-adapter rather than as the single shared step the\nYAML specified (functionally equivalent, but it means each adapter is its own\nrisk surface). This item pins the contract for the remaining adapters and adds a\nCSV-path cross-surface case.\n\nTest-only by intent: if a parity gap surfaces for any adapter, STOP and file a\ndefect TODO \u2014 do not fix adapter behavior inside this coverage item.", "id": "dataframe-csv-empty-string-parity-coverage", "preserves": ["Fast lane stays green on a standard dev environment (availability skips for absent adapters).", "No change to adapter CSV-loading behavior \u2014 this is coverage only."], "prior_art": [{"concept": "{'ref': 'tests/integration/test_dataframe_csv_empty_string_null_semantics.py', 'note': 'The #904 regression test covering DataFusion+PySpark; parametrize its existing assertions across the remaining adapters rather than authoring a new harness.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "tests/integration/test_dataframe_csv_empty_string_null_semantics.py"}], "state": "done", "title": "Pin empty-string/null CSV parity across all DataFrame adapters, not just DataFusion+PySpark", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_dataframe_csv_empty_string_null_semantics.py -q", "description": "Empty-string/null parity holds across installed adapters", "expected": "per-adapter parametrized cells pass for pandas/polars/dask/modin; cudf skipped if absent", "item_id": "dataframe-csv-empty-string-parity-coverage", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": [], "notes": "grep tests/integration/test_dataframe_csv_empty_string_null_semantics.py for\nadapter imports/cases (review-time state: only DATAFUSION_DF_AVAILABLE and\nPYSPARK_AVAILABLE). If a later commit already added adapters, narrow w1 to the\nremaining ones. Record the live coverage in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm on fresh develop that the #904 test still covers only DataFusion+PySpark", "wid": "w0"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": ["w0"], "notes": "Extend the existing test to assert, per adapter, the null-marker-dependent\ncontract: a bare empty field loads as \"\" when csv_null_marker is None, and\nas null when csv_null_marker == \"\". This is not a single uniform outcome \u2014\n\"uniform\" means the same null-marker-conditioned rule holds identically\nacross adapters, not that a bare empty field always loads as null. Guard\ncudf and any non-installed adapter with availability skips so the fast lane\nstays green on a standard dev env.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Parametrize empty-string/null CSV semantics across pandas, polars, dask, modin (cudf availability-skipped)", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": [], "notes": "Add at least one cross-surface equivalence case that forces the CSV load path\n(prefer_parquet=False) so SQL<->DataFrame empty-string/null parity is exercised\nend-to-end, not only per-adapter loading. Reuse an existing small cross-surface\nfixture rather than a new benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a cross-surface CSV-path case (prefer_parquet=False) exercising SQL<->DataFrame parity", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-csv-empty-string-parity-coverage", "needs": ["w0", "w1", "w2"], "notes": "Ensure the parametrized cells are fast-marked and collected by the required\ncode-test fast lane (or an explicit node if a marker excludes them). Record\nthe lane membership in the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm the new cells run in the required fast lane", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-02T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Define and implement strategy for loading TPC-H/TPC-DS data into DataFrames.\n\nDifferent platforms prefer different formats (CSV vs Parquet vs Arrow) and\nhave different loading mechanisms. Must integrate with existing DataSourceResolver.", "id": "dataframe-data-loading-strategy-and-format-management", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "DataFrame Data Loading Strategy and Format Management", "verifications": [], "work": [], "worktree": "dataframe-phase0-data-loading"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Establish dependency management strategy for 8 dataframe platforms with\noverlapping dependencies (numpy, pyarrow, pandas). Some platforms have\nincompatible version requirements that must be resolved.", "id": "dataframe-dependency-resolution-and-optional-groups-strategy", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "DataFrame Dependency Resolution and Optional Groups Strategy", "verifications": [], "work": [], "worktree": "dataframe-phase0-deps"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Evaluate whether JoinOrder should be implemented for DataFrame platforms.\n\nJoinOrder primarily measures SQL optimizer join-order decisions. DataFrame\nworkloads typically specify join order directly, so benchmark intent may not\ntransfer meaningfully.", "id": "dataframe-joinorder-feasibility", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Evaluate DataFrame Relevance for JoinOrder Benchmark", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-25T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement maintenance operations (INSERT/UPDATE/DELETE equivalents) for DataFrame\nplatforms, enabling full TPC-H and TPC-DS compliance in DataFrame mode.\n\nCurrently, `get_maintenance_operations_sql()` only supports SQL mode. When users\nrun `--platform polars-df --benchmark tpch --phases maintenance`, no error is raised\nand the maintenance phase silently does nothing useful.\n\nDataFrame platforms like Spark (Delta Lake, Iceberg), Polars, and others support\nwrite operations that can implement TPC maintenance semantics:\n- Delta Lake: MERGE, UPDATE, DELETE operations\n- Iceberg: Row-level deletes, upserts\n- Polars: write_parquet() with append/overwrite modes\n- Parquet files: Partition-based updates\n\nThis work involves:\n1. Adding validation to error when maintenance+dataframe is attempted (short-term fix)\n2. Designing a unified maintenance abstraction that works for both SQL and DataFrame\n3. Implementing platform-specific maintenance operations", "id": "dataframe-maintenance-operations", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Implement DataFrame Maintenance Operations for TPC-H/TPC-DS", "verifications": [], "work": [], "worktree": "dataframe-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-02T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Implement memory management strategy and per-platform scale factor limits\nto prevent OOM crashes. Different platforms have different memory constraints\n(Pandas loads all in RAM, cuDF limited by GPU VRAM, etc.).", "id": "dataframe-memory-management-and-scale-factor-limits", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "DataFrame Memory Management and Scale Factor Limits", "verifications": [], "work": [], "worktree": "dataframe-phase0-memory"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-31T00:00:00Z", "completed_pr": null, "created_at": "2026-01-25T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement DataFrame versions of Metadata Primitives benchmark operations. This\nbenchmark tests metadata introspection operations: schema discovery, column\nintrospection, table statistics, and query introspection.\n\nDataFrame platforms have metadata capabilities through various APIs:\n\n**PySpark:**\n- spark.catalog.listDatabases(), listTables(), listColumns()\n- spark.sql(\"SHOW TABLES\"), spark.sql(\"DESCRIBE TABLE\")\n- Delta Lake: DESCRIBE HISTORY, DESCRIBE DETAIL\n- Iceberg: metadata tables (snapshots, manifests, files)\n\n**Polars:**\n- df.schema, df.dtypes, df.describe()\n- Limited catalog capabilities (file-based, no central catalog)\n\n**Pandas:**\n- df.dtypes, df.info(), df.describe()\n- No catalog capabilities\n\nThe benchmark should focus on operations that are meaningful across platforms,\nwith platform-specific extensions for lakehouse metadata.", "id": "dataframe-metadata-primitives", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Implement DataFrame Version of Metadata Primitives Benchmark", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} -{"anti_patterns": [{"INSTEAD": "keep extraction targeted to known parameter slots", "dont": "build a full SQL parser for parameter extraction", "why": "regex on well-known TPC query templates is sufficient and far simpler"}, {"INSTEAD": "parameterize for consistency but preserve exact defaults", "dont": "change Q7/Q8 default date values from 1995-01-01/1996-12-31", "why": "these are TPC-H spec-fixed values that do not vary with seed"}, {"INSTEAD": "use module-level override dicts instead", "dont": "change the DataFrameContext Protocol or any of its 5 implementations", "why": "that forces changes to PolarsDataFrameContext, ExpressionFamilyContext, PandasFamilyContext, and all platform adapters"}, {"INSTEAD": "inject params through the parameter functions instead", "dont": "change the function signature of expression_impl/pandas_impl callables", "why": "every query registration binds Callable[[DataFrameContext], Any] and changing this signature requires updating 200+ registrations"}, {"INSTEAD": "this TODO is parameter-only", "dont": "modify the permutation/ordering logic", "why": "ordering is already correct for both TPC-H and TPC-DS in DataFrame mode"}, {"INSTEAD": "use regex patterns anchored to SQL keywords instead", "dont": "parse dsqgen output by positional character offset", "why": "SQL formatting may vary by dialect flag"}, {"INSTEAD": "extract from binary output instead", "dont": "reimplement qgen/dsqgen parameter generation algorithms in Python", "why": "these algorithms are complex, version-specific, and maintained by the TPC"}, {"INSTEAD": "keep them as defaults", "dont": "remove the static default parameter values", "why": "they serve as documentation and fallback when binaries are unavailable"}], "approach": "MINIMUM-CHANGE STRATEGY - inject parameters through centralized parameter functions,\nnot through DataFrameContext Protocol changes.\n\nKey insight: TPC-DS already has a single choke point - every query calls\nget_parameters(query_id) from parameters.py. Adding an optional `overrides` parameter\nto this function is the minimum fix for all 99 TPC-DS queries: zero query function\nmodifications needed.\n\nTPC-H is harder: queries inline values directly (TPCH_PARAMS dict is dead code).\nThe minimum fix requires: (1) consolidate inline values into a TPCH_DEFAULT_PARAMS dict,\n(2) create get_tpch_parameters(query_id, overrides=None), (3) refactor 44 query functions\n(22 expression + 22 pandas) to call it. This is unavoidable scope.\n\nParameter extraction from binaries:\n1. Run qgen/dsqgen for each query with a given seed/SF/stream\n2. Parse substitution values from generated SQL using regex anchored to SQL keywords\n3. Return dict structure matching the existing parameter formats\n4. Cache results per seed/SF/stream combination\n\nOverride injection mechanism: Use a module-level _parameter_overrides dict in each\nparameters module. The dataframe_runner sets it before query execution and clears it\nafter. The get_*_parameters() functions check overrides first, then fall back to static\ndefaults. This avoids touching DataFrameContext, DataFrameQuery, or any platform adapter.\n\nPhase 6 approach: Fix the 3 identified gaps with minimal scope:\n- Q7/Q8: Add start_date/end_date to TPCH_DEFAULT_PARAMS, update 4 functions to read from params\n- Q96: This is the sole TPC-DS query that bypasses get_parameters() - add the call to both impls\n- Tests: Add integration-level tests that verify query functions actually consume overrides\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": ["fix-tpch-reference-seed-and-parameter-parity"], "description": "DataFrame query implementations for TPC-H (22 queries) and TPC-DS (99 queries) use\nhardcoded static parameter values instead of deriving parameters from the same\nseed-based generation that SQL queries use via the qgen/dsqgen binaries. This means:\n\n1. SQL queries produce different parameter values for each stream/seed combination\n (as the TPC spec requires), while DataFrame queries always use the same fixed values\n regardless of stream_id or seed.\n2. SQL and DataFrame modes cannot be meaningfully compared because they execute\n different workloads when using any non-default seed.\n3. Multi-stream throughput tests in DataFrame mode repeat identical parameters across\n streams instead of varying them per the TPC specification.\n\nThe ordering permutation logic is already correct - both SQL and DataFrame modes use\nTPCHStreams.PERMUTATION_MATRIX and TPCDSPermutationGenerator respectively. The gap\nis purely in parameter substitution.\n\nIMPORTANT: The two benchmarks have different parameter access patterns:\n- TPC-DS: All 99 queries call get_parameters(query_id) from parameters.py - a single\n centralized choke point. Minimum fix: modify get_parameters() to accept overrides.\n- TPC-H: All 22 queries INLINE parameter values directly (e.g., cutoff_date = date(1998, 9, 2)).\n The TPCH_PARAMS dict at module top is dead documentation - never referenced by any\n query function. Minimum fix: create a get_tpch_parameters(query_id) function, then\n refactor each query to call it instead of using inline literals.\n\nPOST-IMPLEMENTATION REVIEW (2026-02-06):\nPhases 1-5 are complete and the core mechanism works. An adversarial code review\nidentified 3 defects that require a follow-up Phase 6:\n- BUG: TPC-H Q7/Q8 have hardcoded date parameters that bypass the override system\n- BUG: TPC-DS Q96 bypasses get_parameters() and uses hardcoded literals directly\n- GAP: Missing integration-level tests that verify query functions consume overrides\n- STYLE: Misplaced import in parameter_extractor.py", "id": "dataframe-parameter-substitution-parity", "preserves": ["All 22 TPC-H query functions produce identical results when no overrides are set (static defaults match current inline values exactly)", "All 99 TPC-DS query functions continue to call get_parameters(N) with no signature change - overrides parameter is optional with default None", "DataFrameContext Protocol and all 5 implementations unchanged: DataFrameContext (Protocol), DataFrameContextImpl (ABC), PolarsDataFrameContext, ExpressionFamilyContext, PandasFamilyContext", "Existing DataFrame benchmark results at default seed remain reproducible", "Q7/Q8 dates are spec-fixed at 1995-01-01 to 1996-12-31 regardless of seed - Phase 6 parameterizes them for architectural consistency but the default values MUST remain these exact dates", "TPCDSPermutationGenerator ordering logic is unchanged - permutations are already correct", "TPCHStreams.PERMUTATION_MATRIX ordering logic is unchanged - permutations are already correct", "The params = get_parameters(N) call pattern in all TPC-DS query functions continues to work without modification", "qgen/dsqgen binary invocation patterns in QGenBinary and DSQGenBinary remain untouched"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/query.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/c_tools.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/streams.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/streams.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/dataframe_queries/parameters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/dataframe_queries/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/parameter_extractor.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/parameter_extractor.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_parameter_parity.py"}], "state": "done", "title": "Ensure DataFrame variants use identical parameter substitution and ordering permutations as SQL variants for TPC-H and TPC-DS", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -v", "description": "TPC-H DataFrame parameters match qgen output for seed=1 SF=1", "expected": "All parameter parity tests pass", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'dataframe' -v --timeout=120", "description": "Existing DataFrame tests still pass (backward compatibility)", "expected": "All existing DataFrame tests pass, 0 regressions", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v", "description": "Platform adapter tests still pass (no Protocol breakage)", "expected": "All platform tests pass, 0 failures", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m benchbox run --platform polars-df --benchmark tpch --scale 0.01 --queries Q1,Q3,Q6 --phases power", "description": "Smoke test: DataFrame benchmark runs end-to-end", "expected": "Benchmark completes successfully with 3 query results", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -k 'q7 or q8' -v", "description": "Phase 6: Q7/Q8 date parameterization", "expected": "Q7/Q8 date override tests pass", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -k 'q96' -v", "description": "Phase 6: Q96 parameterization", "expected": "Q96 parameter tests pass", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -k 'integration' -v", "description": "Phase 6: Integration-level override propagation", "expected": "Integration tests verify overrides reach query functions", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 7}], "work": [], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "build a full SQL parser for parameter extraction", "instead": "keep extraction targeted to known parameter slots", "why": "regex on well-known TPC query templates is sufficient and far simpler"}, {"dont": "change Q7/Q8 default date values from 1995-01-01/1996-12-31", "instead": "parameterize for consistency but preserve exact defaults", "why": "these are TPC-H spec-fixed values that do not vary with seed"}, {"dont": "change the DataFrameContext Protocol or any of its 5 implementations", "instead": "use module-level override dicts instead", "why": "that forces changes to PolarsDataFrameContext, ExpressionFamilyContext, PandasFamilyContext, and all platform adapters"}, {"dont": "change the function signature of expression_impl/pandas_impl callables", "instead": "inject params through the parameter functions instead", "why": "every query registration binds Callable[[DataFrameContext], Any] and changing this signature requires updating 200+ registrations"}, {"dont": "modify the permutation/ordering logic", "instead": "this TODO is parameter-only", "why": "ordering is already correct for both TPC-H and TPC-DS in DataFrame mode"}, {"dont": "parse dsqgen output by positional character offset", "instead": "use regex patterns anchored to SQL keywords instead", "why": "SQL formatting may vary by dialect flag"}, {"dont": "reimplement qgen/dsqgen parameter generation algorithms in Python", "instead": "extract from binary output instead", "why": "these algorithms are complex, version-specific, and maintained by the TPC"}, {"dont": "remove the static default parameter values", "instead": "keep them as defaults", "why": "they serve as documentation and fallback when binaries are unavailable"}], "approach": "MINIMUM-CHANGE STRATEGY - inject parameters through centralized parameter functions,\nnot through DataFrameContext Protocol changes.\n\nKey insight: TPC-DS already has a single choke point - every query calls\nget_parameters(query_id) from parameters.py. Adding an optional `overrides` parameter\nto this function is the minimum fix for all 99 TPC-DS queries: zero query function\nmodifications needed.\n\nTPC-H is harder: queries inline values directly (TPCH_PARAMS dict is dead code).\nThe minimum fix requires: (1) consolidate inline values into a TPCH_DEFAULT_PARAMS dict,\n(2) create get_tpch_parameters(query_id, overrides=None), (3) refactor 44 query functions\n(22 expression + 22 pandas) to call it. This is unavoidable scope.\n\nParameter extraction from binaries:\n1. Run qgen/dsqgen for each query with a given seed/SF/stream\n2. Parse substitution values from generated SQL using regex anchored to SQL keywords\n3. Return dict structure matching the existing parameter formats\n4. Cache results per seed/SF/stream combination\n\nOverride injection mechanism: Use a module-level _parameter_overrides dict in each\nparameters module. The dataframe_runner sets it before query execution and clears it\nafter. The get_*_parameters() functions check overrides first, then fall back to static\ndefaults. This avoids touching DataFrameContext, DataFrameQuery, or any platform adapter.\n\nPhase 6 approach: Fix the 3 identified gaps with minimal scope:\n- Q7/Q8: Add start_date/end_date to TPCH_DEFAULT_PARAMS, update 4 functions to read from params\n- Q96: This is the sole TPC-DS query that bypasses get_parameters() - add the call to both impls\n- Tests: Add integration-level tests that verify query functions actually consume overrides\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": ["fix-tpch-reference-seed-and-parameter-parity"], "description": "DataFrame query implementations for TPC-H (22 queries) and TPC-DS (99 queries) use\nhardcoded static parameter values instead of deriving parameters from the same\nseed-based generation that SQL queries use via the qgen/dsqgen binaries. This means:\n\n1. SQL queries produce different parameter values for each stream/seed combination\n (as the TPC spec requires), while DataFrame queries always use the same fixed values\n regardless of stream_id or seed.\n2. SQL and DataFrame modes cannot be meaningfully compared because they execute\n different workloads when using any non-default seed.\n3. Multi-stream throughput tests in DataFrame mode repeat identical parameters across\n streams instead of varying them per the TPC specification.\n\nThe ordering permutation logic is already correct - both SQL and DataFrame modes use\nTPCHStreams.PERMUTATION_MATRIX and TPCDSPermutationGenerator respectively. The gap\nis purely in parameter substitution.\n\nIMPORTANT: The two benchmarks have different parameter access patterns:\n- TPC-DS: All 99 queries call get_parameters(query_id) from parameters.py - a single\n centralized choke point. Minimum fix: modify get_parameters() to accept overrides.\n- TPC-H: All 22 queries INLINE parameter values directly (e.g., cutoff_date = date(1998, 9, 2)).\n The TPCH_PARAMS dict at module top is dead documentation - never referenced by any\n query function. Minimum fix: create a get_tpch_parameters(query_id) function, then\n refactor each query to call it instead of using inline literals.\n\nPOST-IMPLEMENTATION REVIEW (2026-02-06):\nPhases 1-5 are complete and the core mechanism works. An adversarial code review\nidentified 3 defects that require a follow-up Phase 6:\n- BUG: TPC-H Q7/Q8 have hardcoded date parameters that bypass the override system\n- BUG: TPC-DS Q96 bypasses get_parameters() and uses hardcoded literals directly\n- GAP: Missing integration-level tests that verify query functions consume overrides\n- STYLE: Misplaced import in parameter_extractor.py", "id": "dataframe-parameter-substitution-parity", "preserves": ["All 22 TPC-H query functions produce identical results when no overrides are set (static defaults match current inline values exactly)", "All 99 TPC-DS query functions continue to call get_parameters(N) with no signature change - overrides parameter is optional with default None", "DataFrameContext Protocol and all 5 implementations unchanged: DataFrameContext (Protocol), DataFrameContextImpl (ABC), PolarsDataFrameContext, ExpressionFamilyContext, PandasFamilyContext", "Existing DataFrame benchmark results at default seed remain reproducible", "Q7/Q8 dates are spec-fixed at 1995-01-01 to 1996-12-31 regardless of seed - Phase 6 parameterizes them for architectural consistency but the default values MUST remain these exact dates", "TPCDSPermutationGenerator ordering logic is unchanged - permutations are already correct", "TPCHStreams.PERMUTATION_MATRIX ordering logic is unchanged - permutations are already correct", "The params = get_parameters(N) call pattern in all TPC-DS query functions continues to work without modification", "qgen/dsqgen binary invocation patterns in QGenBinary and DSQGenBinary remain untouched"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/query.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/c_tools.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/streams.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/streams.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/dataframe_queries/parameters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/dataframe_queries/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/parameter_extractor.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/parameter_extractor.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_parameter_parity.py"}], "state": "done", "title": "Ensure DataFrame variants use identical parameter substitution and ordering permutations as SQL variants for TPC-H and TPC-DS", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -v", "description": "TPC-H DataFrame parameters match qgen output for seed=1 SF=1", "expected": "All parameter parity tests pass", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'dataframe' -v --timeout=120", "description": "Existing DataFrame tests still pass (backward compatibility)", "expected": "All existing DataFrame tests pass, 0 regressions", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/ -v", "description": "Platform adapter tests still pass (no Protocol breakage)", "expected": "All platform tests pass, 0 failures", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m benchbox run --platform polars-df --benchmark tpch --scale 0.01 --queries Q1,Q3,Q6 --phases power", "description": "Smoke test: DataFrame benchmark runs end-to-end", "expected": "Benchmark completes successfully with 3 query results", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -k 'q7 or q8' -v", "description": "Phase 6: Q7/Q8 date parameterization", "expected": "Q7/Q8 date override tests pass", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -k 'q96' -v", "description": "Phase 6: Q96 parameterization", "expected": "Q96 parameter tests pass", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest tests/unit/core/test_parameter_parity.py -k 'integration' -v", "description": "Phase 6: Integration-level override propagation", "expected": "Integration tests verify overrides reach query functions", "item_id": "dataframe-parameter-substitution-parity", "last_result": null, "last_run": null, "seq": 7}], "work": [], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Performance and Efficiency", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Optimize DataFrame query performance and create comparison benchmarks across platforms and vs SQL mode.", "id": "dataframe-performance-optimization-and-analysis", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "DataFrame Performance Optimization and Analysis", "verifications": [], "work": [], "worktree": "dataframe-performance"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Implement runtime platform availability detection with graceful error handling\nwhen platforms are not installed. Must provide helpful error messages with\ninstallation instructions.", "id": "dataframe-platform-availability-detection-and-error-handling", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "DataFrame Platform Availability Detection and Error Handling", "verifications": [], "work": [], "worktree": "dataframe-phase0-detection"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation and User Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-13T00:00:00Z", "deferrals": [], "deps": [], "description": "Create example Python scripts for DataFrame platforms.\n\nDataFrame mode is a key differentiator for BenchBox but lacks examples.\nNeed examples for:\n- Polars (fastest DataFrame platform)\n- CuDF (GPU-accelerated)\n\nExamples should demonstrate:\n- DataFrame adapter initialization\n- TPC-H benchmark execution\n- Result interpretation", "id": "dataframe-platform-examples", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "DataFrame Platform Example Scripts", "verifications": [], "work": [], "worktree": "docs-experience"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-25T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement DataFrame (Polars, Pandas, PySpark, DataFusion) versions of all 136 Read\nPrimitives queries. This benchmark tests fundamental database read operations and is\nthe ideal candidate for DataFrame expansion - it uses the TPC-H schema (already\nsupported by DataFrame loaders) and covers operations that map well to DataFrame APIs.\n\nThe Read Primitives catalog includes queries across 29 categories:\n- Aggregation (13 queries): COUNT, SUM, AVG, COUNT DISTINCT, GROUP BY\n- Filter (10 queries): WHERE clauses, predicate pushdown\n- Optimizer (13 queries): Subquery decorrelation, join reordering, constant folding\n- Array (9 queries): ARRAY operations, UNNEST, array aggregations\n- String (11 queries): LIKE, CONCAT, SUBSTRING, string functions\n- Order By (8 queries): Sorting, TOP-N, ORDER BY ALL\n- Window (6 queries): ROW_NUMBER, RANK, LAG/LEAD, aggregate windows\n- Qualify (6 queries): QUALIFY clause for window filtering\n- Shuffle (5 queries): Data redistribution patterns\n- Group By (5 queries): GROUP BY ALL, GROUP BY ROLLUP/CUBE\n- And 19 more categories with 1-4 queries each\n\nImplementation follows the established TPC-H DataFrame pattern using:\n- Expression family (Polars, PySpark, DataFusion): ctx.col(), ctx.lit(), lazy expressions\n- Pandas family (Pandas, Modin, Dask, cuDF): df['column'], boolean indexing, eager ops", "id": "dataframe-read-primitives", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Implement DataFrame Version of Read Primitives Benchmark", "verifications": [], "work": [], "worktree": "dataframe-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Assess architecture requirements for Read Primitives DataFrame support in this\ninitiative context.\n\nRead Primitives includes catalog-driven and DB-specific operations that may need\na separate execution path and applicability filtering rather than direct mapping\nthrough the current query model.", "id": "dataframe-read-primitives-architecture", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Assess DataFrame Architecture for Read Primitives Integration", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-25T00:00:00Z", "deferrals": [], "deps": [], "description": "Complete the DataFrame implementation for ALL 136 Read Primitives queries.\n\nCurrent state: 102 queries implemented, 34 remaining\nTarget: 133 queries with implementations (only 3 correlated-subquery queries are truly SQL-only)\n\nThis is a CORRECTION to the previous incomplete implementation that incorrectly\nmarked 49 queries as \"skipped\" or \"expression-family-only\" when they should have\nbeen implemented.\n\nKEY INSIGHT: Spark DataFrames are DECLARATIVE, not imperative. They build a logical\nplan that goes through Catalyst optimizer - the same optimizations apply as SQL.\nTherefore, 10 of the 13 \"optimizer\" queries CAN be tested via DataFrames by writing\nthem in \"naive\" form and letting Catalyst optimize.", "id": "dataframe-read-primitives-complete", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "Complete Read Primitives DataFrame Implementation (All 136 Queries)", "verifications": [], "work": [], "worktree": "dataframe-expansion"} -{"anti_patterns": [{"INSTEAD": "classify the old runner instead.", "dont": "call every DataFrame path canonical", "why": "production appears adapter-mixin based"}, {"INSTEAD": "start with exported invariants.", "dont": "fully unify SQL and DataFrame builders as a first move", "why": "that may over-couple different execution models"}, {"INSTEAD": "compare real exported bundles.", "dont": "write only synthetic parity tests", "why": "synthetic builders miss export-time omissions"}], "approach": "Treat current adapter-mixin execution as the likely production path, but prove\nit before changing docs. Separate lifecycle cleanup from result-bundle parity\nso old-runner decisions do not block the most important contract test.\n", "blocked_reason": null, "category": "DataFrame", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-runner-lifecycle-and-bundle-parity", "id": 368, "reason": "May be unnecessary if exported-bundle parity catches drift with less coupling.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Reason self-negates: 'may be unnecessary if exported-bundle parity catches drift' -- parity was the chosen approach.", "summary": "Unify all SQL and DataFrame result builders into one implementation"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-runner-lifecycle-and-bundle-parity", "id": 369, "reason": "Only safe after lifecycle classification and compatibility-window decision.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Parent item's w1/w2 explicitly rejected immediate deletion and classified dataframe_runner.py as a deprecated compat shim; file still present by design.", "summary": "Delete dataframe_runner.py immediately"}], "deps": ["architecture-contract-map-and-support-taxonomy", "result-schema-policy-and-explorer-input-gate"], "description": "Production DataFrame execution appears to route through adapter.run_benchmark\nand the DataFrame benchmark mixin. The older core/runner/dataframe_runner.py\nstill exists, is documented like a live runner, and has tests. The sharper\narchitectural risk is not that production has three canonical paths; it is\nthat an old path has unclear lifecycle status and no real exported-bundle\nSQL-vs-DataFrame contract guards the result boundary.\n\nThis item classifies dataframe_runner.py as supported, internal compatibility,\nor deprecated, then adds one real SQL-vs-DataFrame exported-result bundle\ncontract test.", "id": "dataframe-runner-lifecycle-and-bundle-parity", "preserves": ["Do not break production DataFrame platforms: polars-df, pandas-df, datafusion-df, and dask-df.", "Do not compare exact execution timings between SQL and DataFrame parity bundles.", "Do not remove tested wrapper facade behavior while classifying DataFrame runner internals."], "prior_art": [{"concept": "benchbox/platforms/dataframe/benchmark_mixin.py: current DataFrame adapter execution path -- prefer as production reference.", "decision": "reuse", "path": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"concept": "docs/reference/backward-compatibility.md: existing lifecycle registry for harmonization shims -- register dataframe_runner.py status here.", "decision": "reuse", "path": "docs/reference/backward-compatibility.md"}, {"concept": "tests/unit/core/results/test_result_parity.py: synthetic parity coverage -- extend with real exported-bundle coverage, not a replacement.", "decision": "extend", "path": "tests/unit/core/results/test_result_parity.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/athena.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/bigquery.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "docs/design/architecture.md"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "tests/integration/validation/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/runner/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpch/"}], "state": "done", "title": "Architecture hardening: retire or revive old DataFrame runner and add exported-bundle parity", "verifications": [{"command": "uv run -- python -m pytest tests -k 'dataframe and parity' --tb=short", "description": "Run focused DataFrame runner and parity tests.", "expected": "Focused DataFrame parity tests pass.", "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/results -q", "description": "Run focused result export tests.", "expected": "Result export tests pass.", "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": [], "notes": "Completed 2026-05-21 at checked SHA\n8604d0a413f6c3d4bd7db211bb472c2370a932c3.\n\nRevalidation summary:\n - `benchbox/core/runner/runner.py` routes DataFrame execute and\n load-only branches through `adapter.run_benchmark()` with\n `DataFramePhases` and `DataFrameRunOptions`.\n - `benchbox/platforms/dataframe/benchmark_mixin.py` is the production\n DataFrame lifecycle owner for result construction, phase status,\n query execution, skip summaries, and plan-capture counters.\n - `benchbox/core/runner/dataframe_runner.py` is not called by the\n production lifecycle runner; remaining references are docs, tests, and\n internal helper imports in CLI/dry-run/monitoring paths.\n - `docs/design/architecture.md` pointed at the old runner before this\n change.\n - `tests/unit/core/results/test_result_parity.py` had synthetic\n payload parity only; exported JSON parity was added here.\n\nCommit a compact summary of checked SHA, production execution path, old\nrunner call sites, docs references, and existing parity tests.\n\nMinimum evidence to re-check:\n - benchbox/core/runner/runner.py DataFrame adapter branch.\n - benchbox/platforms/dataframe/benchmark_mixin.py run_benchmark path.\n - benchbox/core/runner/dataframe_runner.py call sites and tests.\n - docs/design/architecture.md DataFrame runner references.\n - tests/unit/core/results/test_result_parity.py.\n - focused TPC-DI/write/read primitive SQL-vs-DataFrame parity tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate current SQL and DataFrame execution paths and parity coverage", "wid": "w0"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w0"], "notes": "Decision: `benchbox/core/runner/dataframe_runner.py` is a deprecated\ninternal compatibility runner. Production DataFrame execution is the\nadapter-mixin path:\n`run_benchmark_lifecycle()` -> `adapter.run_benchmark()` ->\n`BenchmarkExecutionMixin.run_benchmark()`.\n\nProduction behavior tests are DataFrame mixin/adapter lifecycle tests and\nexported bundle parity. `tests/unit/core/runner/test_dataframe_runner.py`\nand `tests/unit/core/runner/test_dataframe_runner_lifecycle.py` are\ncompatibility tests for the deprecated standalone runner.\n\nChoose exactly one status and record it in\ndocs/reference/backward-compatibility.md and docs/reference/public-contracts.md:\n - supported public/internal runner,\n - internal compatibility shim,\n - deprecated compatibility fossil,\n - removed after one beta window.\n\nThe decision must identify which tests are production behavior tests and\nwhich are compatibility tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decision gate: classify core/runner/dataframe_runner.py lifecycle", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w1"], "notes": "Added module lifecycle constants and docstring guidance to\n`benchbox/core/runner/dataframe_runner.py`, registered the lifecycle in\n`docs/reference/backward-compatibility.md`, and updated\n`docs/reference/public-contracts.md` plus\n`docs/design/architecture.md` to direct production DataFrame execution to\n`adapter.run_benchmark()` and `BenchmarkExecutionMixin`.\n\nIf w1 chooses deprecation or compatibility fossil: update docs so\nDataFrame execution is described through adapter.run_benchmark and the\nDataFrame benchmark mixin. Keep only narrow compatibility tests for\ndataframe_runner.py until the removal window ends.\n\nDo not delete the runner in the same change unless the compatibility\nregistry says the removal window is already closed.\n\nIf w1 chooses supported/internal runner: dataframe_runner.py should\ndelegate to the same adapter-mixin production path or share a lower-level\nhelper so field population cannot drift.\n\nAvoid maintaining a separate result-construction path unless a contract\ntest proves the divergence is intentional.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the selected dataframe_runner.py lifecycle path", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w2"], "notes": "Added `test_exported_sql_and_dataframe_bundles_share_cross_mode_contract`\nin `tests/unit/core/results/test_result_parity.py`. The test creates a\nSQL result through a concrete `PlatformAdapter.run_benchmark()` path,\ncreates a DataFrame result through `BenchmarkExecutionMixin.run_benchmark()`,\nexports both through `ResultExporter`, reloads the JSON payloads, and\ncompares schema version, benchmark/config identity, phases, query IDs,\nrows, validation, errors, execution metadata keys, and timing field\nnames/units without comparing exact timing values.\n\nPick the smallest reliable benchmark/platform pair available in fast or\nfocused integration tests. Compare exported bundle fields, not synthetic\ndataclass construction alone.\n\nRequired invariants:\n - schema version,\n - benchmark/config identity,\n - phase presence and skipped/not-run semantics,\n - query IDs,\n - row counts when available,\n - validation state,\n - error fields,\n - execution metadata keys,\n - timing field names and units.\n\nDo not assert exact timing values.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add one real SQL-vs-DataFrame exported-bundle parity contract test", "wid": "w3"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w3"], "notes": "Documented SQL/DataFrame result bundle invariants and allowed differences\nin `docs/reference/public-contracts.md`. Allowed differences include\nexecution mode, platform metadata, benchmark-managed DataFrame loading\nstatus such as `SKIPPED`, and exact timing values.\n\nAdd a short section to docs/reference/public-contracts.md or a dedicated\nresult contract doc that states which fields must match between SQL and\nDataFrame runs, which may differ, and why.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document cross-mode invariants and known allowed differences", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "call every DataFrame path canonical", "instead": "classify the old runner instead.", "why": "production appears adapter-mixin based"}, {"dont": "fully unify SQL and DataFrame builders as a first move", "instead": "start with exported invariants.", "why": "that may over-couple different execution models"}, {"dont": "write only synthetic parity tests", "instead": "compare real exported bundles.", "why": "synthetic builders miss export-time omissions"}], "approach": "Treat current adapter-mixin execution as the likely production path, but prove\nit before changing docs. Separate lifecycle cleanup from result-bundle parity\nso old-runner decisions do not block the most important contract test.\n", "blocked_reason": null, "category": "DataFrame", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-runner-lifecycle-and-bundle-parity", "id": 368, "reason": "May be unnecessary if exported-bundle parity catches drift with less coupling.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Reason self-negates: 'may be unnecessary if exported-bundle parity catches drift' -- parity was the chosen approach.", "summary": "Unify all SQL and DataFrame result builders into one implementation"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-runner-lifecycle-and-bundle-parity", "id": 369, "reason": "Only safe after lifecycle classification and compatibility-window decision.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Parent item's w1/w2 explicitly rejected immediate deletion and classified dataframe_runner.py as a deprecated compat shim; file still present by design.", "summary": "Delete dataframe_runner.py immediately"}], "deps": ["architecture-contract-map-and-support-taxonomy", "result-schema-policy-and-explorer-input-gate"], "description": "Production DataFrame execution appears to route through adapter.run_benchmark\nand the DataFrame benchmark mixin. The older core/runner/dataframe_runner.py\nstill exists, is documented like a live runner, and has tests. The sharper\narchitectural risk is not that production has three canonical paths; it is\nthat an old path has unclear lifecycle status and no real exported-bundle\nSQL-vs-DataFrame contract guards the result boundary.\n\nThis item classifies dataframe_runner.py as supported, internal compatibility,\nor deprecated, then adds one real SQL-vs-DataFrame exported-result bundle\ncontract test.", "id": "dataframe-runner-lifecycle-and-bundle-parity", "preserves": ["Do not break production DataFrame platforms: polars-df, pandas-df, datafusion-df, and dask-df.", "Do not compare exact execution timings between SQL and DataFrame parity bundles.", "Do not remove tested wrapper facade behavior while classifying DataFrame runner internals."], "prior_art": [{"concept": "benchbox/platforms/dataframe/benchmark_mixin.py: current DataFrame adapter execution path -- prefer as production reference.", "decision": "reuse", "path": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"concept": "docs/reference/backward-compatibility.md: existing lifecycle registry for harmonization shims -- register dataframe_runner.py status here.", "decision": "reuse", "path": "docs/reference/backward-compatibility.md"}, {"concept": "tests/unit/core/results/test_result_parity.py: synthetic parity coverage -- extend with real exported-bundle coverage, not a replacement.", "decision": "extend", "path": "tests/unit/core/results/test_result_parity.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/mcp/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/athena.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/bigquery.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "docs/design/architecture.md"}, {"kind": "only_modify", "path_glob": "docs/reference/"}, {"kind": "only_modify", "path_glob": "tests/integration/validation/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/runner/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpch/"}], "state": "done", "title": "Architecture hardening: retire or revive old DataFrame runner and add exported-bundle parity", "verifications": [{"command": "uv run -- python -m pytest tests -k 'dataframe and parity' --tb=short", "description": "Run focused DataFrame runner and parity tests.", "expected": "Focused DataFrame parity tests pass.", "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/results -q", "description": "Run focused result export tests.", "expected": "Result export tests pass.", "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": [], "notes": "Completed 2026-05-21 at checked SHA\n8604d0a413f6c3d4bd7db211bb472c2370a932c3.\n\nRevalidation summary:\n - `benchbox/core/runner/runner.py` routes DataFrame execute and\n load-only branches through `adapter.run_benchmark()` with\n `DataFramePhases` and `DataFrameRunOptions`.\n - `benchbox/platforms/dataframe/benchmark_mixin.py` is the production\n DataFrame lifecycle owner for result construction, phase status,\n query execution, skip summaries, and plan-capture counters.\n - `benchbox/core/runner/dataframe_runner.py` is not called by the\n production lifecycle runner; remaining references are docs, tests, and\n internal helper imports in CLI/dry-run/monitoring paths.\n - `docs/design/architecture.md` pointed at the old runner before this\n change.\n - `tests/unit/core/results/test_result_parity.py` had synthetic\n payload parity only; exported JSON parity was added here.\n\nCommit a compact summary of checked SHA, production execution path, old\nrunner call sites, docs references, and existing parity tests.\n\nMinimum evidence to re-check:\n - benchbox/core/runner/runner.py DataFrame adapter branch.\n - benchbox/platforms/dataframe/benchmark_mixin.py run_benchmark path.\n - benchbox/core/runner/dataframe_runner.py call sites and tests.\n - docs/design/architecture.md DataFrame runner references.\n - tests/unit/core/results/test_result_parity.py.\n - focused TPC-DI/write/read primitive SQL-vs-DataFrame parity tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate current SQL and DataFrame execution paths and parity coverage", "wid": "w0"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w0"], "notes": "Decision: `benchbox/core/runner/dataframe_runner.py` is a deprecated\ninternal compatibility runner. Production DataFrame execution is the\nadapter-mixin path:\n`run_benchmark_lifecycle()` -> `adapter.run_benchmark()` ->\n`BenchmarkExecutionMixin.run_benchmark()`.\n\nProduction behavior tests are DataFrame mixin/adapter lifecycle tests and\nexported bundle parity. `tests/unit/core/runner/test_dataframe_runner.py`\nand `tests/unit/core/runner/test_dataframe_runner_lifecycle.py` are\ncompatibility tests for the deprecated standalone runner.\n\nChoose exactly one status and record it in\ndocs/reference/backward-compatibility.md and docs/reference/public-contracts.md:\n - supported public/internal runner,\n - internal compatibility shim,\n - deprecated compatibility fossil,\n - removed after one beta window.\n\nThe decision must identify which tests are production behavior tests and\nwhich are compatibility tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decision gate: classify core/runner/dataframe_runner.py lifecycle", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w1"], "notes": "Added module lifecycle constants and docstring guidance to\n`benchbox/core/runner/dataframe_runner.py`, registered the lifecycle in\n`docs/reference/backward-compatibility.md`, and updated\n`docs/reference/public-contracts.md` plus\n`docs/design/architecture.md` to direct production DataFrame execution to\n`adapter.run_benchmark()` and `BenchmarkExecutionMixin`.\n\nIf w1 chooses deprecation or compatibility fossil: update docs so\nDataFrame execution is described through adapter.run_benchmark and the\nDataFrame benchmark mixin. Keep only narrow compatibility tests for\ndataframe_runner.py until the removal window ends.\n\nDo not delete the runner in the same change unless the compatibility\nregistry says the removal window is already closed.\n\nIf w1 chooses supported/internal runner: dataframe_runner.py should\ndelegate to the same adapter-mixin production path or share a lower-level\nhelper so field population cannot drift.\n\nAvoid maintaining a separate result-construction path unless a contract\ntest proves the divergence is intentional.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the selected dataframe_runner.py lifecycle path", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w2"], "notes": "Added `test_exported_sql_and_dataframe_bundles_share_cross_mode_contract`\nin `tests/unit/core/results/test_result_parity.py`. The test creates a\nSQL result through a concrete `PlatformAdapter.run_benchmark()` path,\ncreates a DataFrame result through `BenchmarkExecutionMixin.run_benchmark()`,\nexports both through `ResultExporter`, reloads the JSON payloads, and\ncompares schema version, benchmark/config identity, phases, query IDs,\nrows, validation, errors, execution metadata keys, and timing field\nnames/units without comparing exact timing values.\n\nPick the smallest reliable benchmark/platform pair available in fast or\nfocused integration tests. Compare exported bundle fields, not synthetic\ndataclass construction alone.\n\nRequired invariants:\n - schema version,\n - benchmark/config identity,\n - phase presence and skipped/not-run semantics,\n - query IDs,\n - row counts when available,\n - validation state,\n - error fields,\n - execution metadata keys,\n - timing field names and units.\n\nDo not assert exact timing values.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add one real SQL-vs-DataFrame exported-bundle parity contract test", "wid": "w3"}, {"evidence": null, "item_id": "dataframe-runner-lifecycle-and-bundle-parity", "needs": ["w3"], "notes": "Documented SQL/DataFrame result bundle invariants and allowed differences\nin `docs/reference/public-contracts.md`. Allowed differences include\nexecution mode, platform metadata, benchmark-managed DataFrame loading\nstatus such as `SKIPPED`, and exact timing values.\n\nAdd a short section to docs/reference/public-contracts.md or a dedicated\nresult contract doc that states which fields must match between SQL and\nDataFrame runs, which may differ, and why.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document cross-mode invariants and known allowed differences", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:50Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #533 of write-primitives-sketch-pyspark-dataframe-surface.\nDeferred: DataFrame-side storage-size validation for sketch persistence\nReason deferred: Storage-size methodology is being added in `write-primitives-sketch-clickhouse-and-storage-metrics` for the SQL surface. Once that lands, mirror it for the DataFrame surface \u2014 but do that as a follow-up, not in this TODO.", "id": "dataframe-side-storage-size-validation-for-sketch-persistenc", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "DataFrame-side storage-size validation for sketch persistence", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-12T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Final polish, release notes, and public announcement preparation for\nDataFrame support.", "id": "dataframe-support-release-preparation", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "DataFrame Support Release Preparation", "verifications": [], "work": [], "worktree": "dataframe-release"} -{"anti_patterns": [{"INSTEAD": "use function-scoped fixtures or session-scoped immutable data", "dont": "create one giant test fixture for all queries", "why": "shared mutable state causes flaky tests"}, {"INSTEAD": "use real Polars/Pandas with small test data", "dont": "mock the DataFrame engine to test query logic", "why": "you're testing mock behavior not real execution"}, {"INSTEAD": "assert specific column names and types", "dont": "test DataFrame queries by asserting only `.columns` is truthy", "why": "truthy check passes for any non-empty sequence"}, {"INSTEAD": "exclude from scope and test framework instead", "dont": "test TPC-DS queries individually", "why": "there are 99 of them and the effort vastly exceeds the value"}], "approach": null, "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-25T00:00:00Z", "completed_pr": null, "created_at": "2026-03-23T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 370, "reason": "ClickBench uses a single wide table with specific data patterns. Testing requires the ClickBench dataset or representative fixtures. Lower priority - push after main targets are hit.", "resolution": "promoted", "resolved_item": "clickbench-dataframe-queries-446-stmts-30-5", "resolved_reason": null, "summary": "ClickBench dataframe queries (446 stmts, 30.5%)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 371, "reason": "Already at reasonable coverage. Experimental benchmark type. Cover when AI primitives benchmark stabilizes.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Deferred pending AI-primitives benchmark stabilization; coverage already reasonable (68.3%) and no signal the experimental benchmark has stabilized.", "summary": "AI primitives dataframe operations (385 stmts, 68.3%)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 372, "reason": "Small benchmark with straightforward queries. Already >60%. Low ROI for the effort to build test fixtures.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Coffeeshop dataframe already >60% covered; reason explicitly notes low ROI for effort, small benchmark, straightforward queries.", "summary": "Coffeeshop dataframe queries (254 stmts, 61.8%)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 373, "reason": "Table format maintenance (compaction, optimization, vacuum) requires real table format infrastructure. Better covered by the platform-live-integration-tests TODO.", "resolution": "promoted", "resolved_item": "iceberg-delta-lake-hudi-maintenance-operations", "resolved_reason": null, "summary": "Iceberg/Delta Lake/Hudi maintenance operations"}], "deps": ["fix-coverage-theater"], "description": "The DataFrame subsystem spans three layers totaling 21,643 statements at 50.7%\ncoverage - the single largest coverage gap in the project.\n\nLayer breakdown:\n- DF query implementations: 12,180 stmts at 36.6% (benchmark SQL-to-DF translations)\n- DF platform adapters: 5,847 stmts at 67.7% (unified_frame, polars_df, pandas_df, etc.)\n- DF core framework: 3,285 stmts at 73.5% (benchmark_suite, data_loader, tuning)\n- DF runner: 331 stmts at 50.2% (dataframe_runner.py)\n\nStrategy: test the platform/framework/runner layers properly (~4,500 stmts gain),\ntest the most-used query sets (TPC-H, SSB, read_primitives), and exclude the\nmechanical bulk (TPC-DS 99-query set at 6.9%, datavault at 12.3%) from coverage\nscope. All locally testable with Polars + Pandas (both are project dependencies).", "id": "dataframe-test-coverage", "preserves": ["Existing DataFrame tests must continue to pass - new tests are additive", "Polars and Pandas must remain the only required test engines - no pyspark/dask in fast tests", "Test suite speed: DataFrame tests must stay under fast marker (<1s each)", "unified_frame.py public API must not change - tests validate existing interface"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Increase DataFrame subsystem test coverage with behavioral tests", "verifications": [], "work": [{"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Exclude low-ROI bulk queries from coverage scope (quick win: +4pp)", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-H dataframe queries against known results at SF=0.01", "wid": "w10"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test SSB dataframe queries", "wid": "w11"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test read_primitives and transaction_primitives dataframe operations", "wid": "w12"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test pyspark_df and lakesail_df adapter stubs", "wid": "w13"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py: column, filter, and sort operations", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py: aggregation, joins, and window functions", "wid": "w3"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py: expression building and error paths", "wid": "w4"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFrame benchmark_suite.py orchestration lifecycle", "wid": "w5"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFrame platform adapters: expression_family + pandas_family + benchmark_mixin", "wid": "w6"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test datafusion_df.py and polars_df.py adapter operations", "wid": "w7"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test dataframe_runner.py lifecycle with real engines", "wid": "w8"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DF core framework: data_loader, profiling, tuning, context", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "create one giant test fixture for all queries", "instead": "use function-scoped fixtures or session-scoped immutable data", "why": "shared mutable state causes flaky tests"}, {"dont": "mock the DataFrame engine to test query logic", "instead": "use real Polars/Pandas with small test data", "why": "you're testing mock behavior not real execution"}, {"dont": "test DataFrame queries by asserting only `.columns` is truthy", "instead": "assert specific column names and types", "why": "truthy check passes for any non-empty sequence"}, {"dont": "test TPC-DS queries individually", "instead": "exclude from scope and test framework instead", "why": "there are 99 of them and the effort vastly exceeds the value"}], "approach": null, "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-25T00:00:00Z", "completed_pr": null, "created_at": "2026-03-23T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 370, "reason": "ClickBench uses a single wide table with specific data patterns. Testing requires the ClickBench dataset or representative fixtures. Lower priority - push after main targets are hit.", "resolution": "promoted", "resolved_item": "clickbench-dataframe-queries-446-stmts-30-5", "resolved_reason": null, "summary": "ClickBench dataframe queries (446 stmts, 30.5%)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 371, "reason": "Already at reasonable coverage. Experimental benchmark type. Cover when AI primitives benchmark stabilizes.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Deferred pending AI-primitives benchmark stabilization; coverage already reasonable (68.3%) and no signal the experimental benchmark has stabilized.", "summary": "AI primitives dataframe operations (385 stmts, 68.3%)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 372, "reason": "Small benchmark with straightforward queries. Already >60%. Low ROI for the effort to build test fixtures.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Coffeeshop dataframe already >60% covered; reason explicitly notes low ROI for effort, small benchmark, straightforward queries.", "summary": "Coffeeshop dataframe queries (254 stmts, 61.8%)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dataframe-test-coverage", "id": 373, "reason": "Table format maintenance (compaction, optimization, vacuum) requires real table format infrastructure. Better covered by the platform-live-integration-tests TODO.", "resolution": "promoted", "resolved_item": "iceberg-delta-lake-hudi-maintenance-operations", "resolved_reason": null, "summary": "Iceberg/Delta Lake/Hudi maintenance operations"}], "deps": ["fix-coverage-theater"], "description": "The DataFrame subsystem spans three layers totaling 21,643 statements at 50.7%\ncoverage - the single largest coverage gap in the project.\n\nLayer breakdown:\n- DF query implementations: 12,180 stmts at 36.6% (benchmark SQL-to-DF translations)\n- DF platform adapters: 5,847 stmts at 67.7% (unified_frame, polars_df, pandas_df, etc.)\n- DF core framework: 3,285 stmts at 73.5% (benchmark_suite, data_loader, tuning)\n- DF runner: 331 stmts at 50.2% (dataframe_runner.py)\n\nStrategy: test the platform/framework/runner layers properly (~4,500 stmts gain),\ntest the most-used query sets (TPC-H, SSB, read_primitives), and exclude the\nmechanical bulk (TPC-DS 99-query set at 6.9%, datavault at 12.3%) from coverage\nscope. All locally testable with Polars + Pandas (both are project dependencies).", "id": "dataframe-test-coverage", "preserves": ["Existing DataFrame tests must continue to pass - new tests are additive", "Polars and Pandas must remain the only required test engines - no pyspark/dask in fast tests", "Test suite speed: DataFrame tests must stay under fast marker (<1s each)", "unified_frame.py public API must not change - tests validate existing interface"], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Increase DataFrame subsystem test coverage with behavioral tests", "verifications": [], "work": [{"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Exclude low-ROI bulk queries from coverage scope (quick win: +4pp)", "wid": "w1"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test TPC-H dataframe queries against known results at SF=0.01", "wid": "w10"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test SSB dataframe queries", "wid": "w11"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test read_primitives and transaction_primitives dataframe operations", "wid": "w12"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test pyspark_df and lakesail_df adapter stubs", "wid": "w13"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py: column, filter, and sort operations", "wid": "w2"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py: aggregation, joins, and window functions", "wid": "w3"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test unified_frame.py: expression building and error paths", "wid": "w4"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFrame benchmark_suite.py orchestration lifecycle", "wid": "w5"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DataFrame platform adapters: expression_family + pandas_family + benchmark_mixin", "wid": "w6"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test datafusion_df.py and polars_df.py adapter operations", "wid": "w7"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test dataframe_runner.py lifecycle with real engines", "wid": "w8"}, {"evidence": null, "item_id": "dataframe-test-coverage", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test DF core framework: data_loader, profiling, tuning, context", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Define a dedicated architecture for TPC-DI DataFrame support.\n\nTPC-DI includes mutation-heavy ETL semantics (SCD Type 2, incremental loads,\nstateful transforms) that do not fit the current read-oriented DataFrameQuery/\nDataFrameContext pattern used for analytical query benchmarks.\n\nThis item designs a fit-for-purpose model before implementation.", "id": "dataframe-tpcdi-architecture", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Design DataFrame Architecture for TPC-DI ETL Workload", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Plan DataFrame support for TPC-DS-OBT with explicit OBT-specific adaptation.\n\nTPC-DS-OBT query set and schema differ materially from normalized TPC-DS.\nExisting TPC-DS DataFrame implementations are not directly reusable without\ndenormalized-schema-specific rewriting.", "id": "dataframe-tpcds-obt-adaptation", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Plan DataFrame Adaptation for TPC-DS-OBT", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Evaluate whether any subset of TPC-Havoc is meaningful for DataFrame benchmarking.\n\nTPC-Havoc variants primarily test SQL optimizer behavior (query rewrites,\npredicate pushdown, planner choices). DataFrame execution bypasses SQL parsing\nand optimizer layers, so direct translation likely provides low benchmark value.\n\nThis item captures a formal feasibility decision and, if justified, a narrowly\nscoped DataFrame subset.", "id": "dataframe-tpchavoc-feasibility", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Evaluate DataFrame Scope for TPC-Havoc", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} @@ -326,38 +326,38 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-25T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement DataFrame versions of Write Primitives benchmark operations. This benchmark\ntests fundamental write operations: INSERT, UPDATE, DELETE, BULK_LOAD, MERGE, DDL,\nand TRANSACTION operations.\n\nDataFrame platforms have extensive write capabilities that map to these operations:\n\n**PySpark + Delta Lake:**\n- INSERT \u2192 df.write.mode(\"append\").saveAsTable()\n- UPDATE \u2192 DeltaTable.update()\n- DELETE \u2192 DeltaTable.delete()\n- MERGE \u2192 DeltaTable.merge()\n- DDL \u2192 spark.sql() for schema operations\n\n**PySpark + Iceberg:**\n- INSERT \u2192 df.writeTo().append()\n- UPDATE \u2192 Row-level updates via MERGE\n- DELETE \u2192 Row-level deletes\n- MERGE \u2192 df.writeTo().overwritePartitions()\n\n**Polars:**\n- INSERT \u2192 write_parquet() with append\n- BULK_LOAD \u2192 scan_parquet() + write_parquet()\n- File-level operations (no row-level UPDATE/DELETE)\n\n**All DataFrame Platforms:**\n- BULK_LOAD \u2192 read + write with compression, partitioning, sorting\n- Physical layout tuning (row groups, dictionary encoding)", "id": "dataframe-write-primitives", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Implement DataFrame Version of Write Primitives Benchmark", "verifications": [], "work": [], "worktree": "dataframe-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-07T00:00:00Z", "completed_pr": null, "created_at": "2026-02-07T00:00:00Z", "deferrals": [], "deps": [], "description": "Define how Write Primitives should execute in DataFrame mode.\n\nThe benchmark includes DML/DDL-style operations that do not fit the current\nread-query DataFrameContext contract. A clear write-capable execution model is\nrequired before implementation under this initiative.", "id": "dataframe-write-primitives-model", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Define DataFrame Execution Model for Write Primitives", "verifications": [], "work": [], "worktree": "dataframe-benchmarks"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Enhancement", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Implement a DataFusion DataFrame adapter for BenchBox, enabling TPC-H benchmark\nexecution using DataFusion's Python bindings. DataFusion belongs to the Expression\nfamily (like Polars), offering lazy evaluation, Arrow-native execution, and\ncompetitive single-node performance.\n\nCurrent State:\n- BenchBox has no DataFusion integration (neither SQL nor DataFrame)\n- Expression family base class exists (ExpressionFamilyAdapter)\n- Polars adapter serves as reference implementation for Expression family\n- TPC-H queries have dual implementations (pandas_impl, expression_impl)\n\nValue Proposition:\n- Fastest single-node Parquet engine (ClickBench 2024)\n- Apache Arrow native - zero-copy data interchange\n- SQL and DataFrame API in same engine\n- Rust core with Python bindings - memory safe, performant\n- Growing adoption (InfluxDB 3, GlareDB, OpenObserve)", "id": "datafusion-dataframe-adapter", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "DataFusion DataFrame Adapter Implementation", "verifications": [], "work": [], "worktree": "dataframe-expansion"} -{"anti_patterns": [{"INSTEAD": "keep rewrites in the platform adapter", "dont": "add rewrites to the SQLGlot translation layer", "why": "the issue is DataFusion execution semantics not SQL syntax"}, {"INSTEAD": "rewrite at the adapter level instead", "dont": "modify qgen output or query templates", "why": "they are the official TPC-H reference"}, {"INSTEAD": "create a separate DataFusion-specific transformer", "dont": "modify the ClickHouse query transformer", "why": "it is independent and already working"}, {"INSTEAD": "use query_id dispatch with targeted extraction instead", "dont": "use generic regex-based SQL rewriting for complex structural changes", "why": "nested SQL regex is fragile and unmaintainable"}], "approach": "Follow the ClickHouse query transformer pattern exactly:\n- Reference: benchbox/platforms/clickhouse/query_transformer.py (class structure, transform() entry point, transformations_applied tracking)\n- Reference: benchbox/platforms/clickhouse/workload.py:293-295 (integration into execute_query())\n- Reference: benchbox/core/tpch/dataframe_queries.py (DataFrame implementations of Q11, Q16, Q18, Q20 show the correct algorithmic approach to translate into SQL rewrites)\n\nUse query_id-based dispatch rather than regex pattern matching on SQL text.\nEach rewrite method extracts parameterized values (nation names, thresholds,\ndates) from the qgen SQL via targeted regex, then reconstructs the query with\nDataFusion-compatible structure. Literal values are preserved; only structure changes.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": ["fix-tpch-reference-seed-and-parameter-parity"], "description": "DataFusion's SQL engine misexecutes 4 TPC-H queries (Q11, Q16, Q18, Q20) that\nuse SQL patterns it handles differently from PostgreSQL: HAVING with scalar\nsubqueries, NOT IN subqueries, IN with HAVING, and nested correlated subqueries.\n\nThese failures were exposed when the seed parity fix changed validation from\n\"loose\" (\u00b150% tolerance) to \"exact\" (0% tolerance). The DataFrame implementations\nof the same queries produce correct results because they use explicit joins,\nanti-joins, and semi-joins instead of these problematic SQL patterns.\n\nThe fix follows the established ClickHouse pattern: create a DataFusionQueryTransformer\nthat rewrites the 4 problematic queries before execution, transforming them into\nsemantically equivalent SQL that DataFusion handles correctly.\n\nRow count discrepancies at SF=1 with qgen -d defaults:\n Q11: Expected 1,048 \u2192 Got 1,289 (HAVING + scalar subquery threshold)\n Q16: Expected 18,314 \u2192 Got 18,200 (NOT IN subquery, NULL semantics)\n Q18: Expected 57 \u2192 Got 9 (IN subquery with HAVING clause)\n Q20: Expected 186 \u2192 Got 173 (nested IN with correlated subquery)", "id": "datafusion-sql-query-transformer", "preserves": ["All 18 non-affected TPC-H queries (Q1-Q10, Q12-Q15, Q17, Q19, Q21-Q22) pass through transformer unchanged", "ClickHouseQueryTransformer remains independent and unchanged", "DataFusion DataFrame mode (datafusion-df) execution path is not affected", "DataFusionAdapter.execute_query() continues to work for non-TPC-H queries (no transformer applied when query_id is not Q11/Q16/Q18/Q20)"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/power_test.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion_query_transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/dialect_utils.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_datafusion_query_transformer.py"}], "state": "done", "title": "Add DataFusion SQL query transformer for TPC-H Q11, Q16, Q18, Q20", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_datafusion_query_transformer.py -v", "description": "New transformer unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_datafusion_adapter.py -v", "description": "Existing DataFusion adapter tests still pass", "expected": "all tests pass, 0 failures", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check benchbox/platforms/datafusion_query_transformer.py benchbox/platforms/datafusion.py benchbox/utils/dialect_utils.py", "description": "Lint clean on new and modified files", "expected": "All checks passed!", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 3}, {"command": "benchbox run --platform datafusion --benchmark tpch --scale 1 --queries Q11,Q16,Q18,Q20", "description": "Integration: Q11, Q16, Q18, Q20 produce correct row counts at SF=1", "expected": "Q11: 1048 rows, Q16: 18314 rows, Q18: 57 rows, Q20: 186 rows, 0 validation failures", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 4}], "work": [], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "blindly append a dummy column to every file", "why": "use has_trailing_delimiter (field-count aware) so TPC-DS .dat with a terminating-but-complete row count is untouched."}, {"INSTEAD": "(not specified)", "dont": "drop the dummy column by positional index", "why": "drop it by the named TRAILING_DUMMY_COLUMN sentinel so a schema/order change cannot drop a real column."}, {"INSTEAD": "(not specified)", "dont": "fork yet another trailing-delimiter implementation", "why": "reuse benchbox/utils/file_format helpers like the other adapters do."}, {"INSTEAD": "(not specified)", "dont": "strip the trailing delimiter by rewriting the .tbl/.dat files on disk", "why": "handle it at read/registration time."}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-15T00:00:00Z", "deferrals": [], "deps": [], "description": "DataFusionAdapter.load_data cannot ingest raw TPC-H .tbl (or TPC-DS .dat) data\nfiles that carry a field-TERMINATING trailing delimiter - the format dbgen/dsdgen\nemit by default, where a row of N values ends with a trailing `|` and so splits\ninto N+1 fields. BOTH of the adapter's load paths fail on it:\n\n - Parquet path (the default, data_format=\"parquet\"): _build_pyarrow_columns\n returns the N schema column names; _write_csv_file_to_parquet then calls\n pyarrow.csv.read_csv with column_names=, and the trailing pipe makes\n PyArrow see N+1 fields, raising `ArrowInvalid: CSV parse error: Expected N\n columns, got N+1` (observed on customer.tbl: \"Expected 8 columns, got 9\").\n - CSV external-table path (data_format=\"csv\"): _load_table_csv builds a\n CREATE EXTERNAL TABLE with an N-column schema and relies on DataFusion's CSV\n reader, which \"doesn't have a direct 'ignore trailing delimiter' option\"\n (the adapter's own comment, ~benchbox/platforms/datafusion.py:923), so the\n extra field shifts/!breaks parsing.\n\nThis was hit directly while building the third-engine TPC-Havoc equivalence\nsample (PR #806). Because the adapter was out of that change's scope, the sample\nhad to HAND-ROLL a PyArrow load in\nbenchbox/core/tpchavoc/equivalence.py:build_datafusion_with_tpch (reading the\n.tbl with the shared trailing-delimiter helpers, dropping the dummy column, and\nregistering Arrow batches) and reach into the private\nDataFusionAdapter._map_schema_type_to_pyarrow - a workaround the PR review\nflagged (reuse / private-access / load-fidelity-drift). Fixing the adapter lets\nthat workaround be removed and lets ANY real DataFusion .tbl/.dat ingest work.\n\nThe correct fix already exists and is used by the OTHER adapters: the shared\nhelpers benchbox/utils/file_format.has_trailing_delimiter +\nget_column_names_with_trailing + the TRAILING_DUMMY_COLUMN sentinel. The\nCOPY/CTAS loaders in benchbox/platforms/base/data_loading.py (~lines 1083 and\n1168) detect the trailing delimiter per file and append a dummy column for the\nextra field; the DataFusion adapter is the one that skips this step. Critically,\nhas_trailing_delimiter is field-count-aware (it returns False for TPC-DS\ntime_dim where dsdgen emits exactly N values with a terminating delimiter), so\nthe fix must NOT blindly append a dummy column to every file.", "id": "datafusion-tbl-trailing-delimiter-load", "preserves": ["No on-disk rewriting of source data files; handle the trailing delimiter at read time only.", "Non-trailing files (e.g. TPC-DS time_dim, where field count already equals the schema) are not regressed - rely on has_trailing_delimiter's field-count awareness, never a blind append.", "Query results and row counts are unchanged: the trailing dummy column never leaks into a result schema, COUNT(*), or row total.", "The DataFusion TPC-Havoc equivalence sample stays green (206/206, DATAFUSION_KNOWN_DIVERGENCES empty)."], "prior_art": [{"concept": "benchbox/core/tpchavoc/equivalence.py:build_datafusion_with_tpch - the in-scope hand-rolled workaround (PR #806) to delete once the adapter is fixed; it is the canonical example of the correct read+drop sequence.", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py:build_datafusion_with_tpch"}, {"concept": "benchbox/platforms/base/data_loading.py (~1083, ~1168) - the reference pattern: other adapters detect trailing delimiter per file and add the dummy column.", "decision": "reuse", "path": "benchbox/platforms/base/data_loading.py"}, {"concept": "benchbox/platforms/datafusion.py:_build_pyarrow_columns (~1257), _convert_and_register_parquet (~1331), _write_csv_file_to_parquet (~1276) - the parquet path that reads CSV with the bare schema column count and errors on the trailing field.", "decision": "reuse", "path": "benchbox/platforms/datafusion.py:_build_pyarrow_columns"}, {"concept": "benchbox/platforms/datafusion.py:_load_table_csv (~845) and its comment (~923) - the CREATE EXTERNAL TABLE path with no ignore-trailing-delimiter handling.", "decision": "reuse", "path": "benchbox/platforms/datafusion.py:_load_table_csv"}, {"concept": "benchbox/utils/file_format.py:has_trailing_delimiter (404), get_column_names_with_trailing (468), TRAILING_DUMMY_COLUMN (401) - the field-count-aware detection + dummy-column helpers to reuse.", "decision": "reuse", "path": "benchbox/utils/file_format.py:has_trailing_delimiter"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "tests/integration/"}], "state": "done", "title": "Fix DataFusion adapter ingest of trailing-delimiter TPC .tbl/.dat files", "verifications": [{"command": "uv run -- python -m pytest tests/integration -k datafusion_tbl_load -n 0", "description": "Real TPC-H .tbl loads through the adapter on both formats with correct row counts", "expected": "row counts match the generator; no dummy column in result schemas; passes for parquet and csv", "item_id": "datafusion-tbl-trailing-delimiter-load", "last_result": null, "last_run": null, "seq": 1}, {"command": "make tpchavoc-equivalence-report-datafusion", "description": "The third-engine equivalence sample still passes after the helper is simplified to reuse the adapter", "expected": "checked 206 variants - 0 divergent, 206 equivalent; exit 0", "item_id": "datafusion-tbl-trailing-delimiter-load", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": [], "notes": "RESOLVED: in _convert_and_register_parquet, detect the trailing delimiter on\nfile_paths[0] with has_trailing_delimiter(path, delimiter, schema_column_names).\nWhen present, read with TRAILING_DUMMY_COLUMN appended (via\nget_column_names_with_trailing) AND set ConvertOptions.include_columns to the\nreal columns - so PyArrow parses the N+1 fields but the output Table contains\nonly the N real columns. Using include_columns (rather than a post-hoc\ndrop_columns) works uniformly for both the read_csv and streaming open_csv\nbranches and needs no PyArrow-version-specific RecordBatch.drop. Non-trailing\nfiles use include_columns=[] (\"all columns\"), so they are unchanged; detection\nis field-count aware, so TPC-DS .dat with a complete row count is untouched.\nThe trailing branch also disables PyArrow quoting/escaping (quote_char=False),\nbecause raw TPC dbgen/dsdgen output is never quoted - so a text field that\nbegins with `\"` (or contains a backslash) cannot be parsed as a quote/escape\nand mis-split the row. Genuinely quoted CSV benchmarks have no trailing\ndelimiter and keep the standard quote_char='\"'/escape_char handling. This\ngeneralizes the defense the equivalence helper previously applied locally.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the parquet conversion path to handle a trailing delimiter", "wid": "w0"}, {"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": ["w0"], "notes": "RESOLVED: chose option (b). In _load_table_csv, once the schema is known,\ndetect a trailing delimiter (has_trailing_delimiter over the schema column\nnames) and, when present, delegate to _load_table_parquet (which routes to the\nnow-fixed _convert_and_register_parquet). This reuses w0's correct read+project\nlogic instead of duplicating a declare-then-project-away view across both the\ndirect and union CSV sub-paths. Results and row counts are identical; only the\nstorage form (Parquet in the working dir) differs. Well-formed CSVs (no\ntrailing delimiter) keep the CREATE EXTERNAL TABLE path unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or explicitly handle the CSV external-table path", "wid": "w1"}, {"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": ["w0", "w1"], "notes": "RESOLVED: tests/integration/platforms/test_datafusion_tbl_load.py generates\nreal TPC-H .tbl at SF=0.01 once (module fixture) and loads via\nDataFusionAdapter.load_data parametrized over data_format in {parquet, csv}.\nAsserts every table has >0 rows, that TRAILING_DUMMY_COLUMN never appears in\nlineitem's information_schema columns, that the real column count is exact\n(16), and that COUNT(*) matches the reported load stats. Marked slow (it\ngenerates data), so it runs in the non-blocking datafusion-integration CI job\nalongside the equivalence sample (selected via -m \"slow\").\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an integration test loading real TPC-H .tbl via the adapter", "wid": "w2"}, {"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": ["w0", "w1", "w2"], "notes": "RESOLVED: build_datafusion_with_tpch now calls adapter.create_schema +\nadapter.load_data (mirroring build_duckdb_with_tpch / build_postgres_with_tpch),\nremoving the hand-rolled PyArrow read, the duplicated trailing-delimiter\nhandling, and the private _map_schema_type_to_pyarrow access flagged in the\nPR #806 review. The deliberate quote_char=False defense the helper applied\nlocally is preserved - it now lives in the adapter's trailing branch (w0), so\nit covers every TPC load, not just this sample. The per-table row-count guard\n(>0) is preserved. The DataFusion equivalence sample stays green: 206/206\nexecutable variants, exit 0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify the third-engine equivalence helper to reuse the adapter", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add rewrites to the SQLGlot translation layer", "instead": "keep rewrites in the platform adapter", "why": "the issue is DataFusion execution semantics not SQL syntax"}, {"dont": "modify qgen output or query templates", "instead": "rewrite at the adapter level instead", "why": "they are the official TPC-H reference"}, {"dont": "modify the ClickHouse query transformer", "instead": "create a separate DataFusion-specific transformer", "why": "it is independent and already working"}, {"dont": "use generic regex-based SQL rewriting for complex structural changes", "instead": "use query_id dispatch with targeted extraction instead", "why": "nested SQL regex is fragile and unmaintainable"}], "approach": "Follow the ClickHouse query transformer pattern exactly:\n- Reference: benchbox/platforms/clickhouse/query_transformer.py (class structure, transform() entry point, transformations_applied tracking)\n- Reference: benchbox/platforms/clickhouse/workload.py:293-295 (integration into execute_query())\n- Reference: benchbox/core/tpch/dataframe_queries.py (DataFrame implementations of Q11, Q16, Q18, Q20 show the correct algorithmic approach to translate into SQL rewrites)\n\nUse query_id-based dispatch rather than regex pattern matching on SQL text.\nEach rewrite method extracts parameterized values (nation names, thresholds,\ndates) from the qgen SQL via targeted regex, then reconstructs the query with\nDataFusion-compatible structure. Literal values are preserved; only structure changes.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": ["fix-tpch-reference-seed-and-parameter-parity"], "description": "DataFusion's SQL engine misexecutes 4 TPC-H queries (Q11, Q16, Q18, Q20) that\nuse SQL patterns it handles differently from PostgreSQL: HAVING with scalar\nsubqueries, NOT IN subqueries, IN with HAVING, and nested correlated subqueries.\n\nThese failures were exposed when the seed parity fix changed validation from\n\"loose\" (\u00b150% tolerance) to \"exact\" (0% tolerance). The DataFrame implementations\nof the same queries produce correct results because they use explicit joins,\nanti-joins, and semi-joins instead of these problematic SQL patterns.\n\nThe fix follows the established ClickHouse pattern: create a DataFusionQueryTransformer\nthat rewrites the 4 problematic queries before execution, transforming them into\nsemantically equivalent SQL that DataFusion handles correctly.\n\nRow count discrepancies at SF=1 with qgen -d defaults:\n Q11: Expected 1,048 \u2192 Got 1,289 (HAVING + scalar subquery threshold)\n Q16: Expected 18,314 \u2192 Got 18,200 (NOT IN subquery, NULL semantics)\n Q18: Expected 57 \u2192 Got 9 (IN subquery with HAVING clause)\n Q20: Expected 186 \u2192 Got 173 (nested IN with correlated subquery)", "id": "datafusion-sql-query-transformer", "preserves": ["All 18 non-affected TPC-H queries (Q1-Q10, Q12-Q15, Q17, Q19, Q21-Q22) pass through transformer unchanged", "ClickHouseQueryTransformer remains independent and unchanged", "DataFusion DataFrame mode (datafusion-df) execution path is not affected", "DataFusionAdapter.execute_query() continues to work for non-TPC-H queries (no transformer applied when query_id is not Q11/Q16/Q18/Q20)"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/benchmark.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/power_test.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/clickhouse/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion_query_transformer.py"}, {"kind": "only_modify", "path_glob": "benchbox/utils/dialect_utils.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_datafusion_query_transformer.py"}], "state": "done", "title": "Add DataFusion SQL query transformer for TPC-H Q11, Q16, Q18, Q20", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_datafusion_query_transformer.py -v", "description": "New transformer unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_datafusion_adapter.py -v", "description": "Existing DataFusion adapter tests still pass", "expected": "all tests pass, 0 failures", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check benchbox/platforms/datafusion_query_transformer.py benchbox/platforms/datafusion.py benchbox/utils/dialect_utils.py", "description": "Lint clean on new and modified files", "expected": "All checks passed!", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 3}, {"command": "benchbox run --platform datafusion --benchmark tpch --scale 1 --queries Q11,Q16,Q18,Q20", "description": "Integration: Q11, Q16, Q18, Q20 produce correct row counts at SF=1", "expected": "Q11: 1048 rows, Q16: 18314 rows, Q18: 57 rows, Q20: 186 rows, 0 validation failures", "item_id": "datafusion-sql-query-transformer", "last_result": null, "last_run": null, "seq": 4}], "work": [], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "blindly append a dummy column to every file", "instead": "(not specified)", "why": "use has_trailing_delimiter (field-count aware) so TPC-DS .dat with a terminating-but-complete row count is untouched."}, {"dont": "drop the dummy column by positional index", "instead": "(not specified)", "why": "drop it by the named TRAILING_DUMMY_COLUMN sentinel so a schema/order change cannot drop a real column."}, {"dont": "fork yet another trailing-delimiter implementation", "instead": "(not specified)", "why": "reuse benchbox/utils/file_format helpers like the other adapters do."}, {"dont": "strip the trailing delimiter by rewriting the .tbl/.dat files on disk", "instead": "(not specified)", "why": "handle it at read/registration time."}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-06-15T00:00:00Z", "deferrals": [], "deps": [], "description": "DataFusionAdapter.load_data cannot ingest raw TPC-H .tbl (or TPC-DS .dat) data\nfiles that carry a field-TERMINATING trailing delimiter - the format dbgen/dsdgen\nemit by default, where a row of N values ends with a trailing `|` and so splits\ninto N+1 fields. BOTH of the adapter's load paths fail on it:\n\n - Parquet path (the default, data_format=\"parquet\"): _build_pyarrow_columns\n returns the N schema column names; _write_csv_file_to_parquet then calls\n pyarrow.csv.read_csv with column_names=, and the trailing pipe makes\n PyArrow see N+1 fields, raising `ArrowInvalid: CSV parse error: Expected N\n columns, got N+1` (observed on customer.tbl: \"Expected 8 columns, got 9\").\n - CSV external-table path (data_format=\"csv\"): _load_table_csv builds a\n CREATE EXTERNAL TABLE with an N-column schema and relies on DataFusion's CSV\n reader, which \"doesn't have a direct 'ignore trailing delimiter' option\"\n (the adapter's own comment, ~benchbox/platforms/datafusion.py:923), so the\n extra field shifts/!breaks parsing.\n\nThis was hit directly while building the third-engine TPC-Havoc equivalence\nsample (PR #806). Because the adapter was out of that change's scope, the sample\nhad to HAND-ROLL a PyArrow load in\nbenchbox/core/tpchavoc/equivalence.py:build_datafusion_with_tpch (reading the\n.tbl with the shared trailing-delimiter helpers, dropping the dummy column, and\nregistering Arrow batches) and reach into the private\nDataFusionAdapter._map_schema_type_to_pyarrow - a workaround the PR review\nflagged (reuse / private-access / load-fidelity-drift). Fixing the adapter lets\nthat workaround be removed and lets ANY real DataFusion .tbl/.dat ingest work.\n\nThe correct fix already exists and is used by the OTHER adapters: the shared\nhelpers benchbox/utils/file_format.has_trailing_delimiter +\nget_column_names_with_trailing + the TRAILING_DUMMY_COLUMN sentinel. The\nCOPY/CTAS loaders in benchbox/platforms/base/data_loading.py (~lines 1083 and\n1168) detect the trailing delimiter per file and append a dummy column for the\nextra field; the DataFusion adapter is the one that skips this step. Critically,\nhas_trailing_delimiter is field-count-aware (it returns False for TPC-DS\ntime_dim where dsdgen emits exactly N values with a terminating delimiter), so\nthe fix must NOT blindly append a dummy column to every file.", "id": "datafusion-tbl-trailing-delimiter-load", "preserves": ["No on-disk rewriting of source data files; handle the trailing delimiter at read time only.", "Non-trailing files (e.g. TPC-DS time_dim, where field count already equals the schema) are not regressed - rely on has_trailing_delimiter's field-count awareness, never a blind append.", "Query results and row counts are unchanged: the trailing dummy column never leaks into a result schema, COUNT(*), or row total.", "The DataFusion TPC-Havoc equivalence sample stays green (206/206, DATAFUSION_KNOWN_DIVERGENCES empty)."], "prior_art": [{"concept": "benchbox/core/tpchavoc/equivalence.py:build_datafusion_with_tpch - the in-scope hand-rolled workaround (PR #806) to delete once the adapter is fixed; it is the canonical example of the correct read+drop sequence.", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py:build_datafusion_with_tpch"}, {"concept": "benchbox/platforms/base/data_loading.py (~1083, ~1168) - the reference pattern: other adapters detect trailing delimiter per file and add the dummy column.", "decision": "reuse", "path": "benchbox/platforms/base/data_loading.py"}, {"concept": "benchbox/platforms/datafusion.py:_build_pyarrow_columns (~1257), _convert_and_register_parquet (~1331), _write_csv_file_to_parquet (~1276) - the parquet path that reads CSV with the bare schema column count and errors on the trailing field.", "decision": "reuse", "path": "benchbox/platforms/datafusion.py:_build_pyarrow_columns"}, {"concept": "benchbox/platforms/datafusion.py:_load_table_csv (~845) and its comment (~923) - the CREATE EXTERNAL TABLE path with no ignore-trailing-delimiter handling.", "decision": "reuse", "path": "benchbox/platforms/datafusion.py:_load_table_csv"}, {"concept": "benchbox/utils/file_format.py:has_trailing_delimiter (404), get_column_names_with_trailing (468), TRAILING_DUMMY_COLUMN (401) - the field-count-aware detection + dummy-column helpers to reuse.", "decision": "reuse", "path": "benchbox/utils/file_format.py:has_trailing_delimiter"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/datafusion.py"}, {"kind": "only_modify", "path_glob": "tests/integration/"}], "state": "done", "title": "Fix DataFusion adapter ingest of trailing-delimiter TPC .tbl/.dat files", "verifications": [{"command": "uv run -- python -m pytest tests/integration -k datafusion_tbl_load -n 0", "description": "Real TPC-H .tbl loads through the adapter on both formats with correct row counts", "expected": "row counts match the generator; no dummy column in result schemas; passes for parquet and csv", "item_id": "datafusion-tbl-trailing-delimiter-load", "last_result": null, "last_run": null, "seq": 1}, {"command": "make tpchavoc-equivalence-report-datafusion", "description": "The third-engine equivalence sample still passes after the helper is simplified to reuse the adapter", "expected": "checked 206 variants - 0 divergent, 206 equivalent; exit 0", "item_id": "datafusion-tbl-trailing-delimiter-load", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": [], "notes": "RESOLVED: in _convert_and_register_parquet, detect the trailing delimiter on\nfile_paths[0] with has_trailing_delimiter(path, delimiter, schema_column_names).\nWhen present, read with TRAILING_DUMMY_COLUMN appended (via\nget_column_names_with_trailing) AND set ConvertOptions.include_columns to the\nreal columns - so PyArrow parses the N+1 fields but the output Table contains\nonly the N real columns. Using include_columns (rather than a post-hoc\ndrop_columns) works uniformly for both the read_csv and streaming open_csv\nbranches and needs no PyArrow-version-specific RecordBatch.drop. Non-trailing\nfiles use include_columns=[] (\"all columns\"), so they are unchanged; detection\nis field-count aware, so TPC-DS .dat with a complete row count is untouched.\nThe trailing branch also disables PyArrow quoting/escaping (quote_char=False),\nbecause raw TPC dbgen/dsdgen output is never quoted - so a text field that\nbegins with `\"` (or contains a backslash) cannot be parsed as a quote/escape\nand mis-split the row. Genuinely quoted CSV benchmarks have no trailing\ndelimiter and keep the standard quote_char='\"'/escape_char handling. This\ngeneralizes the defense the equivalence helper previously applied locally.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the parquet conversion path to handle a trailing delimiter", "wid": "w0"}, {"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": ["w0"], "notes": "RESOLVED: chose option (b). In _load_table_csv, once the schema is known,\ndetect a trailing delimiter (has_trailing_delimiter over the schema column\nnames) and, when present, delegate to _load_table_parquet (which routes to the\nnow-fixed _convert_and_register_parquet). This reuses w0's correct read+project\nlogic instead of duplicating a declare-then-project-away view across both the\ndirect and union CSV sub-paths. Results and row counts are identical; only the\nstorage form (Parquet in the working dir) differs. Well-formed CSVs (no\ntrailing delimiter) keep the CREATE EXTERNAL TABLE path unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or explicitly handle the CSV external-table path", "wid": "w1"}, {"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": ["w0", "w1"], "notes": "RESOLVED: tests/integration/platforms/test_datafusion_tbl_load.py generates\nreal TPC-H .tbl at SF=0.01 once (module fixture) and loads via\nDataFusionAdapter.load_data parametrized over data_format in {parquet, csv}.\nAsserts every table has >0 rows, that TRAILING_DUMMY_COLUMN never appears in\nlineitem's information_schema columns, that the real column count is exact\n(16), and that COUNT(*) matches the reported load stats. Marked slow (it\ngenerates data), so it runs in the non-blocking datafusion-integration CI job\nalongside the equivalence sample (selected via -m \"slow\").\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add an integration test loading real TPC-H .tbl via the adapter", "wid": "w2"}, {"evidence": null, "item_id": "datafusion-tbl-trailing-delimiter-load", "needs": ["w0", "w1", "w2"], "notes": "RESOLVED: build_datafusion_with_tpch now calls adapter.create_schema +\nadapter.load_data (mirroring build_duckdb_with_tpch / build_postgres_with_tpch),\nremoving the hand-rolled PyArrow read, the duplicated trailing-delimiter\nhandling, and the private _map_schema_type_to_pyarrow access flagged in the\nPR #806 review. The deliberate quote_char=False defense the helper applied\nlocally is preserved - it now lives in the adapter's trailing branch (w0), so\nit covers every TPC load, not just this sample. The per-table row-count guard\n(>0) is preserved. The DataFusion equivalence sample stays green: 206/206\nexecutable variants, exit 0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify the third-engine equivalence helper to reuse the adapter", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-24T00:00:00Z", "deferrals": [], "deps": ["gcp-dataproc-platform"], "description": "Add GCP Dataproc Serverless platform adapter. Serverless variant provides\nfully managed Spark with no cluster management and auto-scaling.\n\nImplementation shares 80% code with Dataproc cluster adapter. Uses Batch\nController API instead of Jobs API.", "id": "dataproc-serverless-platform", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "GCP Dataproc Serverless Platform", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2025-11-28T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Implement a Data Vault 2.0 benchmark for BenchBox using TPC-H as the source data. This benchmark transforms TPC-H's 8 tables into 21 Data Vault tables (7 Hubs, 6 Links, 8 Satellites) and adapts the 22 TPC-H queries to work with the Hub-Link-Satellite data model.", "id": "datavault-benchmark-implementation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Add Data Vault Benchmark Implementation", "verifications": [], "work": [], "worktree": "benchmark-expansion"} -{"anti_patterns": [{"INSTEAD": "extend to SHA-256 only", "dont": "add SHA-1 to the allowlist", "why": "40-char output doesn't map to either VARCHAR(32) or VARCHAR(64) cleanly, and SHA-1 offers no security advantage over SHA-256"}, {"INSTEAD": "change HASHKEY in place since VARCHAR(64) is backward-compatible", "dont": "add a second HASHKEY_WIDE column type alongside HASHKEY", "why": "that creates two parallel schema paths that diverge"}, {"INSTEAD": "remove it entirely once VARCHAR(64) lands", "dont": "keep the [:32] truncation as a compatibility shim", "why": "it destroys entropy and makes SHA-256 no better than MD5"}, {"INSTEAD": "derive all enforcement from SUPPORTED_HASH_ALGORITHMS", "dont": "replicate the algorithm check in each layer independently", "why": "that is the root cause of this entire bug"}], "approach": "Execute as two gate-checked phases: w1 (stop the false promise) before any\nschema changes, then w3\u2192w4 (schema + implementation) before w5 (tests). Each\nwork unit must pass the fast test suite before the next begins.\n\nThe schema change (w3) is backward-compatible: VARCHAR(64) stores existing\n32-char MD5 hashes without modification. No data migration is required for\ntest fixtures; only DDL generation changes.\n\nSUPPORTED_HASH_ALGORITHMS in benchmark.py is the single source of truth after\nthis work. Any future algorithm addition touches that constant and nothing else.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-17T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "The DataVault benchmark's hash key columns are defined as VARCHAR(32), sized for\nMD5's 32-char hex output. MD5 has known collision vulnerabilities that make it\nunsuitable for Data Vault 2.0 in any production-representative benchmark - the\nwhole point of DV2.0 hash keys is collision-free surrogate identity.\n\nA root-cause analysis revealed a five-layer inconsistency across the codebase:\n\n 1. BenchmarkOptionSpec (benchmark.py:485) declares choices=(\"md5\", \"sha256\") -\n the CLI accepts sha256 and the docs advertise it.\n 2. SUPPORTED_HASH_ALGORITHMS = (\"md5\",) (benchmark.py:56) immediately rejects\n sha256 at construction time with ValueError.\n 3. HashAlgorithm = Literal[\"md5\"] (hash_functions.py:16) narrows the type to\n match the runtime - the unreachable sha1/sha256 branches were cleaned up in\n fc6509c35 along with the [:32] truncation; sha256 must now be added back cleanly.\n 4. (Resolved by fc6509c35) The sha256 branch truncated to [:32], destroying\n half the entropy. That branch is gone; the function now raises ValueError for\n non-md5 inputs. Remaining work is to add sha256 properly post-schema migration.\n 5. DataVaultETLTransformer hard-codes `if hash_algorithm != \"md5\"` (transformer.py:136)\n independently of SUPPORTED_HASH_ALGORITHMS.\n\nThe fix has two sequential phases: (a) immediately stop the CLI from advertising\nsha256 while the schema cannot support it, and (b) migrate the schema to VARCHAR(64)\nand wire sha256 through all remaining layers cleanly.\n\nAfter this work, `benchbox run --platform duckdb --benchmark datavault\n--benchmark-option hash_algorithm=sha256` must execute successfully end-to-end.", "id": "datavault-sha256-hash-migration", "preserves": ["Existing VARCHAR(32) test fixtures remain valid - stored MD5 hashes are still 32 chars and fit in VARCHAR(64) without truncation", "MD5 remains the default - hash_algorithm defaults to 'md5' in all constructors and CLI options", "SHA-1 stays unsupported - 40-char output doesn't align with either VARCHAR(32) or VARCHAR(64), and SHA-1 is cryptographically weaker than SHA-256", "generate_hash_key_sql() remains MD5-only - DuckDB's md5() SQL function is the only hash SQL available without custom extensions; that restriction stands"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/datavault/"}, {"kind": "only_modify", "path_glob": "docs/reference/cli/run.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/datavault/"}], "state": "done", "title": "Migrate DataVault hash key schema from VARCHAR(32) to VARCHAR(64) and enable SHA-256", "verifications": [{"command": "uv run -- python -m benchbox run --platform duckdb --benchmark datavault --scale 0.01 --benchmark-option hash_algorithm=sha256 --phases generate,load,power", "description": "SHA-256 DataVault run completes without error", "expected": "Benchmark completed successfully", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.datavault.etl.hash_functions import generate_hash_key; r = generate_hash_key(1, algorithm='sha256'); assert len(r) == 64, f'got {len(r)}'; print('OK')\"", "description": "SHA-256 hash key is 64 chars (not truncated)", "expected": "OK", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.core.datavault.benchmark import DataVaultBenchmark; DataVaultBenchmark(scale_factor=0.01, hash_algorithm='sha1')\" 2>&1 | grep -c ValueError", "description": "SHA-1 is still rejected", "expected": "1", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.datavault.schema import HASHKEY; assert HASHKEY == 'VARCHAR(64)', f'got {HASHKEY}'; print('OK')\"", "description": "Hub DDL uses VARCHAR(64) for hash key column", "expected": "OK", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/datavault/ -q", "description": "Datavault fast tests pass", "expected": "All tests pass", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": [], "notes": "In benchbox/core/datavault/benchmark.py:485, change:\n choices=(\"md5\", \"sha256\")\nto:\n choices=(\"md5\",)\n\nAlso update docs/reference/cli/run.md - the hash_algorithm row currently says\n\"md5, sha256\"; change it to \"md5\" with a note that sha256 support is planned.\n\nThis eliminates the user-visible false promise without any behavioral change.\nThe validator at line 89 already rejects sha256 - this makes the CLI agree.\n\nVerification: `benchbox run --benchmark datavault --benchmark-option hash_algorithm=sha256`\nmust now fail at CLI validation (choices check) rather than silently passing\nthe CLI and failing inside the constructor.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the immediate CLI false promise: remove sha256 from BenchmarkOptionSpec choices", "wid": "w1"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w1"], "notes": "Read and map every place the algorithm set is defined or enforced:\n - HashAlgorithm Literal in hash_functions.py:16\n - SUPPORTED_HASH_ALGORITHMS in benchmark.py:56\n - BenchmarkOptionSpec.choices in benchmark.py:485 (fixed in w1)\n - Validator in benchmark.py:89\n - Hard-coded check in transformer.py:136\n - Schema column HASHKEY = \"VARCHAR(32)\" in schema.py:34\n\nDecide: SUPPORTED_HASH_ALGORITHMS in benchmark.py should be the single\nauthoritative source. HashAlgorithm Literal derives from it (or is kept\nin sync). BenchmarkOptionSpec.choices must exactly match it. Transformer\nmust import and reference it rather than hard-coding.\n\nOutput: inline comment in benchmark.py above SUPPORTED_HASH_ALGORITHMS\ndocumenting it as the canonical source, and what must be updated in sync\nwhen it changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit every source of truth for the algorithm allowlist - document the single-source design", "wid": "w2"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w2"], "notes": "In benchbox/core/datavault/schema.py, change:\n HASHKEY = \"VARCHAR(32)\" # MD5 hash keys are always 32 hex characters\nto:\n HASHKEY = \"VARCHAR(64)\" # Hash key columns - 32 chars for MD5, 40 for SHA-1, 64 for SHA-256\n\nVerify that HASHKEY is the single definition driving all hub/link/satellite DDL.\nGrep for any other hard-coded VARCHAR(32) references tied to hash columns and\nupdate those too. Do not change non-hash columns.\n\nThe wider column is backward-compatible for existing MD5 data - VARCHAR(64)\nstores 32-char strings without truncation or padding.\n\nRun datavault schema generation smoke test to confirm DDL output reflects the\nnew width before proceeding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Schema migration: widen HASHKEY column from VARCHAR(32) to VARCHAR(64)", "wid": "w3"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w3"], "notes": "Exact changes in order:\n\n1. hash_functions.py:16 - widen type:\n HashAlgorithm = Literal[\"md5\", \"sha256\"]\n SHA-1 stays excluded: it's neither collision-safe nor schema-compatible (40 chars).\n\n2. hash_functions.py:43-45 - add sha256 branch before the raise:\n (The old truncated [:32] sha256 branch was removed in fc6509c35;\n the function body is now just md5 + raise ValueError.)\n Add:\n elif algorithm == \"sha256\":\n return hashlib.sha256(key_string.encode(\"utf-8\")).hexdigest()\n The 64-char output fits VARCHAR(64).\n\n3. benchmark.py:56 - widen the allowlist:\n SUPPORTED_HASH_ALGORITHMS = (\"md5\", \"sha256\")\n\n4. benchmark.py:485 - restore sha256 in CLI choices:\n choices=(\"md5\", \"sha256\")\n\n5. transformer.py:136-137 - replace the hard-coded check with a reference:\n from benchbox.core.datavault.benchmark import DataVaultBenchmark\n if hash_algorithm not in DataVaultBenchmark.SUPPORTED_HASH_ALGORITHMS:\n raise ValueError(...)\n Or import SUPPORTED_HASH_ALGORITHMS directly to avoid circular import.\n\nSHA-1 is deliberately not added: 40-char output doesn't map cleanly to\neither VARCHAR(32) or VARCHAR(64), and SHA-1 is cryptographically weaker\nthan SHA-256.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire sha256 through all five layers: type, constant, validator, transformer, function", "wid": "w4"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w4"], "notes": "In tests/unit/core/datavault/test_datavault_benchmark.py:\n - test_sha256_algorithm_rejected \u2192 test_sha256_algorithm_accepted: construct\n DataVaultBenchmark(hash_algorithm=\"sha256\") and assert no exception.\n - test_sha1_algorithm_rejected: keep - SHA-1 must still be rejected.\n - Add test_sha256_produces_64char_key: call generate_hash_key(1, algorithm=\"sha256\")\n and assert len(result) == 64.\n\nIn tests/unit/core/datavault/test_hash_functions.py:\n - test_sha256_algorithm: update assertion from len==32 to len==64.\n - Keep test_sha1_algorithm as-is (40-char output is correct for SHA-1 itself).\n\nIn tests/unit/core/datavault/ (or schema tests):\n - Add test_hashkey_column_width: generate DDL for a hub table and assert\n that hash key columns are VARCHAR(64), not VARCHAR(32).\n\nRun full datavault fast test suite before marking w5 done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests: flip sha256 rejection tests, add sha256 acceptance and schema width tests", "wid": "w5"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w4", "w5"], "notes": "Locations to update:\n - docs/reference/cli/run.md: hash_algorithm row - restore \"md5, sha256\"\n (already changed to \"md5\" in w1; re-add sha256 now that it works).\n - benchmark.py:55 comment: \"currently only MD5 due to VARCHAR(32) schema\n constraints\" - update to reflect VARCHAR(64) and sha256 support.\n - benchmark.py:51, 78: docstring mentions \"only 'md5' currently supported\"\n - update to list both md5 and sha256.\n - transformer.py:126: docstring \"(only 'md5' supported)\" - update.\n - hash_functions.py:28-29: generate_hash_key docstring says \"32-character\n hex string\" - update to describe md5 (32 chars) and sha256 (64 chars).\n - hash_functions.py: verify Returns docstring is accurate post-w4.\n\nGrep for \"only.*md5\" and \"VARCHAR(32)\" across the datavault package to catch\nany remaining stale references.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update all docs and comments that say sha256 is unsupported or MD5-only", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "enforce a module-wide grep->0 ratchet that would forbid legitimate capability-table dialect keys.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "re-add dialect if/elif to the rendering functions", "why": "the whole point is to retire them into capability data."}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The sql_compat capability engine has largely replaced ddl.py's dialect if/elif\nchains (type dispatch is now table-driven via get_type_mapping(), down from ~13\nbranches to ~2). But residual literal-dialect policy remains inside DDL RENDERING\nfunctions:\n - ddl.py:296 if dialect.lower() not in (\"snowflake\", \"bigquery\") (map column)\n - ddl.py:351 if pk_columns and dialect.lower() not in (\"clickhouse\", \"doris\")\n - ddl.py:390 if dialect.lower() == \"bigquery\" / elif \"clickhouse\" (view syntax)\nThese hardcode dialect names in the rendering path rather than resolving through a\ncapability table. They are the residual tail of the sql_compat migration.\n\nMove this policy into the existing sql_compat capability predicates/data\n(supports_complex_types, supports_foreign_keys, supports_views already exist) so\nthe DDL rendering functions carry no literal dialect names.", "id": "ddl-dialect-policy-migration", "preserves": ["Rendered DDL is byte-identical for every currently-supported dialect (snapshot before/after).", "sql_compat capability semantics are reused/extended, not duplicated."], "prior_art": [{"concept": "{'ref': 'benchbox/core/metadata_primitives/ddl.py:173', 'note': 'get_type_mapping() - the data-driven pattern the residual branches should follow.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/metadata_primitives/ddl.py:522, :535, :550', 'note': 'supports_complex_types / supports_views / supports_foreign_keys predicates - extend these (or the sql_compat table behind them) to cover map-column, PK, and view-syntax policy.'}", "decision": "extend", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/sql_compat/ (registry.py, decision.py, resolver.py, inventory.py)', 'note': 'The capability engine that owns dialect-specific data; add the map/PK/view-syntax policy here, do not re-add if/elif to ddl.py.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/ddl.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Migrate residual literal-dialect policy out of metadata_primitives/ddl.py rendering into sql_compat capability data", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/metadata_primitives/test_ddl.py -q", "description": "DDL unchanged per dialect; rendering carries no literal dialect policy", "expected": "rendered DDL byte-identical per dialect (snapshot); rendering-scoped no-literal-dialect test passes", "item_id": "ddl-dialect-policy-migration", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "ddl-dialect-policy-migration", "needs": [], "notes": "Replace the three literal-dialect branches (ddl.py:296/351/390) with capability\nlookups: map-column support, primary-key-clause support, and view-syntax\nvariant resolved from one sql_compat table/registry. Keep the rendered DDL\nbyte-identical for every current dialect (this is a refactor, not a behavior\nchange).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Resolve map-column, PK, and view-syntax policy via capability data, not literal dialect names", "wid": "w1"}, {"evidence": null, "item_id": "ddl-dialect-policy-migration", "needs": ["w1"], "notes": "Add a test scoped to the DDL RENDERING functions asserting they contain no\nliteral dialect-name policy (assert behavior via capability data and a snapshot\nof rendered DDL per dialect). Do NOT use a blunt module-wide\ngrep 'dialect.lower()' -> 0 ratchet: capability-table internals may legitimately\nreference dialect keys; the smell is literal dialect policy in the RENDERING\npath, not anywhere in the module.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a rendering-scoped no-literal-dialect-names test", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use extras groups", "dont": "add deps to core just to make undeclared imports satisfied", "why": "that bloats the minimal install"}, {"INSTEAD": "declarations only", "dont": "remove the guarded-import pattern in AI-primitives", "why": "the whole point is they remain optional"}], "approach": "Inventory-first. For each undeclared import, check the actual code path\nbefore deciding:\n - Is it shipped source or test-only?\n - Is it guarded behind try/except?\n - Is there a sensible extras group for it (existing `viz`, `dev`,\n or new `ai-primitives`)?\nAdd the minimum declaration needed. Update the inventory doc to close\nthe loop.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "declare-undeclared-runtime-imports", "id": 142, "reason": "Product-level choice; out of scope for this declaration cleanup.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Decide which AI-primitive backend is the canonical default"}], "deps": [], "description": "The dependency audit (`docs/development/dependency-inventory.md`,\nused-but-undeclared section) found three categories of imports that\nappear in shipped or tested source but have no `pyproject.toml`\ndeclaration:\n\n1. `PIL` (Pillow) - image processing in visualization image tests.\n2. `ansi2html` - terminal output \u2192 HTML conversion (likely tooling-only).\n3. AI-primitives ML stack - `sentence_transformers`, `spacy`, `textblob`,\n `torch`, used by `benchbox/core/ai_primitives/`. All are guarded\n behind `try/except ImportError`, but should still be declared in an\n explicit extras group so users who want AI-primitives can install\n a known-good set.\n\nGoal: each undeclared third-party import either gets an explicit\ndeclaration in the appropriate extras group, or is removed if the code\npath is dead. No silent transitive-reach behavior.", "id": "declare-undeclared-runtime-imports", "preserves": ["Existing guarded-import patterns in `benchbox/core/ai_primitives/` continue to work without the optional deps installed", "Fast suite green", "No new unconditional `import PIL`/`import torch`/etc - declarations only, not new imports", "Visualization image tests continue to work where PIL is currently relied on"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Declare currently-undeclared runtime imports (pillow, ansi2html, AI-primitives stack)", "verifications": [{"command": "uv run -- python -c \"import pathlib; t = pathlib.Path('docs/development/dependency-inventory.md').read_text(); assert 'used-but-undeclared' in t.lower(); print('OK: section still present')\"", "description": "Used-but-undeclared section in inventory updated to reflect new state", "expected": "OK: section still present", "item_id": "declare-undeclared-runtime-imports", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite passes", "expected": "no failures", "item_id": "declare-undeclared-runtime-imports", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": [], "notes": "Use: grep -rn '^from PIL\\|^import PIL' benchbox tests\nDecide: extras:viz vs dep-group:dev vs delete based on whether\nshipped source vs test-only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Locate PIL import sites; declare `pillow` in `extras:viz` or dev group, or remove the dependent code.", "wid": "w1"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Locate every ansi2html import site. If only used by tooling, add to `[dependency-groups] dev`. If used by shipped source, add to an appropriate runtime extras group.", "wid": "w2"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory AI-primitives imports: sentence_transformers, spacy, textblob, torch. Confirm guarded import pattern. Add an `ai-primitives` extras group declaring the supported optional set.", "wid": "w3"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update `docs/development/dependency-inventory.md` Used-But-Undeclared section to reflect the new declarations (or document any that were intentionally left undeclared, with reason).", "wid": "w4"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run fast suite to confirm no install regressions.", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add SHA-1 to the allowlist", "instead": "extend to SHA-256 only", "why": "40-char output doesn't map to either VARCHAR(32) or VARCHAR(64) cleanly, and SHA-1 offers no security advantage over SHA-256"}, {"dont": "add a second HASHKEY_WIDE column type alongside HASHKEY", "instead": "change HASHKEY in place since VARCHAR(64) is backward-compatible", "why": "that creates two parallel schema paths that diverge"}, {"dont": "keep the [:32] truncation as a compatibility shim", "instead": "remove it entirely once VARCHAR(64) lands", "why": "it destroys entropy and makes SHA-256 no better than MD5"}, {"dont": "replicate the algorithm check in each layer independently", "instead": "derive all enforcement from SUPPORTED_HASH_ALGORITHMS", "why": "that is the root cause of this entire bug"}], "approach": "Execute as two gate-checked phases: w1 (stop the false promise) before any\nschema changes, then w3\u2192w4 (schema + implementation) before w5 (tests). Each\nwork unit must pass the fast test suite before the next begins.\n\nThe schema change (w3) is backward-compatible: VARCHAR(64) stores existing\n32-char MD5 hashes without modification. No data migration is required for\ntest fixtures; only DDL generation changes.\n\nSUPPORTED_HASH_ALGORITHMS in benchmark.py is the single source of truth after\nthis work. Any future algorithm addition touches that constant and nothing else.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-17T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "The DataVault benchmark's hash key columns are defined as VARCHAR(32), sized for\nMD5's 32-char hex output. MD5 has known collision vulnerabilities that make it\nunsuitable for Data Vault 2.0 in any production-representative benchmark - the\nwhole point of DV2.0 hash keys is collision-free surrogate identity.\n\nA root-cause analysis revealed a five-layer inconsistency across the codebase:\n\n 1. BenchmarkOptionSpec (benchmark.py:485) declares choices=(\"md5\", \"sha256\") -\n the CLI accepts sha256 and the docs advertise it.\n 2. SUPPORTED_HASH_ALGORITHMS = (\"md5\",) (benchmark.py:56) immediately rejects\n sha256 at construction time with ValueError.\n 3. HashAlgorithm = Literal[\"md5\"] (hash_functions.py:16) narrows the type to\n match the runtime - the unreachable sha1/sha256 branches were cleaned up in\n fc6509c35 along with the [:32] truncation; sha256 must now be added back cleanly.\n 4. (Resolved by fc6509c35) The sha256 branch truncated to [:32], destroying\n half the entropy. That branch is gone; the function now raises ValueError for\n non-md5 inputs. Remaining work is to add sha256 properly post-schema migration.\n 5. DataVaultETLTransformer hard-codes `if hash_algorithm != \"md5\"` (transformer.py:136)\n independently of SUPPORTED_HASH_ALGORITHMS.\n\nThe fix has two sequential phases: (a) immediately stop the CLI from advertising\nsha256 while the schema cannot support it, and (b) migrate the schema to VARCHAR(64)\nand wire sha256 through all remaining layers cleanly.\n\nAfter this work, `benchbox run --platform duckdb --benchmark datavault\n--benchmark-option hash_algorithm=sha256` must execute successfully end-to-end.", "id": "datavault-sha256-hash-migration", "preserves": ["Existing VARCHAR(32) test fixtures remain valid - stored MD5 hashes are still 32 chars and fit in VARCHAR(64) without truncation", "MD5 remains the default - hash_algorithm defaults to 'md5' in all constructors and CLI options", "SHA-1 stays unsupported - 40-char output doesn't align with either VARCHAR(32) or VARCHAR(64), and SHA-1 is cryptographically weaker than SHA-256", "generate_hash_key_sql() remains MD5-only - DuckDB's md5() SQL function is the only hash SQL available without custom extensions; that restriction stands"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/datavault/"}, {"kind": "only_modify", "path_glob": "docs/reference/cli/run.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/datavault/"}], "state": "done", "title": "Migrate DataVault hash key schema from VARCHAR(32) to VARCHAR(64) and enable SHA-256", "verifications": [{"command": "uv run -- python -m benchbox run --platform duckdb --benchmark datavault --scale 0.01 --benchmark-option hash_algorithm=sha256 --phases generate,load,power", "description": "SHA-256 DataVault run completes without error", "expected": "Benchmark completed successfully", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.datavault.etl.hash_functions import generate_hash_key; r = generate_hash_key(1, algorithm='sha256'); assert len(r) == 64, f'got {len(r)}'; print('OK')\"", "description": "SHA-256 hash key is 64 chars (not truncated)", "expected": "OK", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.core.datavault.benchmark import DataVaultBenchmark; DataVaultBenchmark(scale_factor=0.01, hash_algorithm='sha1')\" 2>&1 | grep -c ValueError", "description": "SHA-1 is still rejected", "expected": "1", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.datavault.schema import HASHKEY; assert HASHKEY == 'VARCHAR(64)', f'got {HASHKEY}'; print('OK')\"", "description": "Hub DDL uses VARCHAR(64) for hash key column", "expected": "OK", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/datavault/ -q", "description": "Datavault fast tests pass", "expected": "All tests pass", "item_id": "datavault-sha256-hash-migration", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": [], "notes": "In benchbox/core/datavault/benchmark.py:485, change:\n choices=(\"md5\", \"sha256\")\nto:\n choices=(\"md5\",)\n\nAlso update docs/reference/cli/run.md - the hash_algorithm row currently says\n\"md5, sha256\"; change it to \"md5\" with a note that sha256 support is planned.\n\nThis eliminates the user-visible false promise without any behavioral change.\nThe validator at line 89 already rejects sha256 - this makes the CLI agree.\n\nVerification: `benchbox run --benchmark datavault --benchmark-option hash_algorithm=sha256`\nmust now fail at CLI validation (choices check) rather than silently passing\nthe CLI and failing inside the constructor.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix the immediate CLI false promise: remove sha256 from BenchmarkOptionSpec choices", "wid": "w1"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w1"], "notes": "Read and map every place the algorithm set is defined or enforced:\n - HashAlgorithm Literal in hash_functions.py:16\n - SUPPORTED_HASH_ALGORITHMS in benchmark.py:56\n - BenchmarkOptionSpec.choices in benchmark.py:485 (fixed in w1)\n - Validator in benchmark.py:89\n - Hard-coded check in transformer.py:136\n - Schema column HASHKEY = \"VARCHAR(32)\" in schema.py:34\n\nDecide: SUPPORTED_HASH_ALGORITHMS in benchmark.py should be the single\nauthoritative source. HashAlgorithm Literal derives from it (or is kept\nin sync). BenchmarkOptionSpec.choices must exactly match it. Transformer\nmust import and reference it rather than hard-coding.\n\nOutput: inline comment in benchmark.py above SUPPORTED_HASH_ALGORITHMS\ndocumenting it as the canonical source, and what must be updated in sync\nwhen it changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit every source of truth for the algorithm allowlist - document the single-source design", "wid": "w2"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w2"], "notes": "In benchbox/core/datavault/schema.py, change:\n HASHKEY = \"VARCHAR(32)\" # MD5 hash keys are always 32 hex characters\nto:\n HASHKEY = \"VARCHAR(64)\" # Hash key columns - 32 chars for MD5, 40 for SHA-1, 64 for SHA-256\n\nVerify that HASHKEY is the single definition driving all hub/link/satellite DDL.\nGrep for any other hard-coded VARCHAR(32) references tied to hash columns and\nupdate those too. Do not change non-hash columns.\n\nThe wider column is backward-compatible for existing MD5 data - VARCHAR(64)\nstores 32-char strings without truncation or padding.\n\nRun datavault schema generation smoke test to confirm DDL output reflects the\nnew width before proceeding.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Schema migration: widen HASHKEY column from VARCHAR(32) to VARCHAR(64)", "wid": "w3"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w3"], "notes": "Exact changes in order:\n\n1. hash_functions.py:16 - widen type:\n HashAlgorithm = Literal[\"md5\", \"sha256\"]\n SHA-1 stays excluded: it's neither collision-safe nor schema-compatible (40 chars).\n\n2. hash_functions.py:43-45 - add sha256 branch before the raise:\n (The old truncated [:32] sha256 branch was removed in fc6509c35;\n the function body is now just md5 + raise ValueError.)\n Add:\n elif algorithm == \"sha256\":\n return hashlib.sha256(key_string.encode(\"utf-8\")).hexdigest()\n The 64-char output fits VARCHAR(64).\n\n3. benchmark.py:56 - widen the allowlist:\n SUPPORTED_HASH_ALGORITHMS = (\"md5\", \"sha256\")\n\n4. benchmark.py:485 - restore sha256 in CLI choices:\n choices=(\"md5\", \"sha256\")\n\n5. transformer.py:136-137 - replace the hard-coded check with a reference:\n from benchbox.core.datavault.benchmark import DataVaultBenchmark\n if hash_algorithm not in DataVaultBenchmark.SUPPORTED_HASH_ALGORITHMS:\n raise ValueError(...)\n Or import SUPPORTED_HASH_ALGORITHMS directly to avoid circular import.\n\nSHA-1 is deliberately not added: 40-char output doesn't map cleanly to\neither VARCHAR(32) or VARCHAR(64), and SHA-1 is cryptographically weaker\nthan SHA-256.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire sha256 through all five layers: type, constant, validator, transformer, function", "wid": "w4"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w4"], "notes": "In tests/unit/core/datavault/test_datavault_benchmark.py:\n - test_sha256_algorithm_rejected \u2192 test_sha256_algorithm_accepted: construct\n DataVaultBenchmark(hash_algorithm=\"sha256\") and assert no exception.\n - test_sha1_algorithm_rejected: keep - SHA-1 must still be rejected.\n - Add test_sha256_produces_64char_key: call generate_hash_key(1, algorithm=\"sha256\")\n and assert len(result) == 64.\n\nIn tests/unit/core/datavault/test_hash_functions.py:\n - test_sha256_algorithm: update assertion from len==32 to len==64.\n - Keep test_sha1_algorithm as-is (40-char output is correct for SHA-1 itself).\n\nIn tests/unit/core/datavault/ (or schema tests):\n - Add test_hashkey_column_width: generate DDL for a hub table and assert\n that hash key columns are VARCHAR(64), not VARCHAR(32).\n\nRun full datavault fast test suite before marking w5 done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests: flip sha256 rejection tests, add sha256 acceptance and schema width tests", "wid": "w5"}, {"evidence": null, "item_id": "datavault-sha256-hash-migration", "needs": ["w4", "w5"], "notes": "Locations to update:\n - docs/reference/cli/run.md: hash_algorithm row - restore \"md5, sha256\"\n (already changed to \"md5\" in w1; re-add sha256 now that it works).\n - benchmark.py:55 comment: \"currently only MD5 due to VARCHAR(32) schema\n constraints\" - update to reflect VARCHAR(64) and sha256 support.\n - benchmark.py:51, 78: docstring mentions \"only 'md5' currently supported\"\n - update to list both md5 and sha256.\n - transformer.py:126: docstring \"(only 'md5' supported)\" - update.\n - hash_functions.py:28-29: generate_hash_key docstring says \"32-character\n hex string\" - update to describe md5 (32 chars) and sha256 (64 chars).\n - hash_functions.py: verify Returns docstring is accurate post-w4.\n\nGrep for \"only.*md5\" and \"VARCHAR(32)\" across the datavault package to catch\nany remaining stale references.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update all docs and comments that say sha256 is unsupported or MD5-only", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "enforce a module-wide grep->0 ratchet that would forbid legitimate capability-table dialect keys.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "re-add dialect if/elif to the rendering functions", "instead": "(not specified)", "why": "the whole point is to retire them into capability data."}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The sql_compat capability engine has largely replaced ddl.py's dialect if/elif\nchains (type dispatch is now table-driven via get_type_mapping(), down from ~13\nbranches to ~2). But residual literal-dialect policy remains inside DDL RENDERING\nfunctions:\n - ddl.py:296 if dialect.lower() not in (\"snowflake\", \"bigquery\") (map column)\n - ddl.py:351 if pk_columns and dialect.lower() not in (\"clickhouse\", \"doris\")\n - ddl.py:390 if dialect.lower() == \"bigquery\" / elif \"clickhouse\" (view syntax)\nThese hardcode dialect names in the rendering path rather than resolving through a\ncapability table. They are the residual tail of the sql_compat migration.\n\nMove this policy into the existing sql_compat capability predicates/data\n(supports_complex_types, supports_foreign_keys, supports_views already exist) so\nthe DDL rendering functions carry no literal dialect names.", "id": "ddl-dialect-policy-migration", "preserves": ["Rendered DDL is byte-identical for every currently-supported dialect (snapshot before/after).", "sql_compat capability semantics are reused/extended, not duplicated."], "prior_art": [{"concept": "{'ref': 'benchbox/core/metadata_primitives/ddl.py:173', 'note': 'get_type_mapping() - the data-driven pattern the residual branches should follow.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/metadata_primitives/ddl.py:522, :535, :550', 'note': 'supports_complex_types / supports_views / supports_foreign_keys predicates - extend these (or the sql_compat table behind them) to cover map-column, PK, and view-syntax policy.'}", "decision": "extend", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/sql_compat/ (registry.py, decision.py, resolver.py, inventory.py)', 'note': 'The capability engine that owns dialect-specific data; add the map/PK/view-syntax policy here, do not re-add if/elif to ddl.py.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/ddl.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Migrate residual literal-dialect policy out of metadata_primitives/ddl.py rendering into sql_compat capability data", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/metadata_primitives/test_ddl.py -q", "description": "DDL unchanged per dialect; rendering carries no literal dialect policy", "expected": "rendered DDL byte-identical per dialect (snapshot); rendering-scoped no-literal-dialect test passes", "item_id": "ddl-dialect-policy-migration", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "ddl-dialect-policy-migration", "needs": [], "notes": "Replace the three literal-dialect branches (ddl.py:296/351/390) with capability\nlookups: map-column support, primary-key-clause support, and view-syntax\nvariant resolved from one sql_compat table/registry. Keep the rendered DDL\nbyte-identical for every current dialect (this is a refactor, not a behavior\nchange).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Resolve map-column, PK, and view-syntax policy via capability data, not literal dialect names", "wid": "w1"}, {"evidence": null, "item_id": "ddl-dialect-policy-migration", "needs": ["w1"], "notes": "Add a test scoped to the DDL RENDERING functions asserting they contain no\nliteral dialect-name policy (assert behavior via capability data and a snapshot\nof rendered DDL per dialect). Do NOT use a blunt module-wide\ngrep 'dialect.lower()' -> 0 ratchet: capability-table internals may legitimately\nreference dialect keys; the smell is literal dialect policy in the RENDERING\npath, not anywhere in the module.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a rendering-scoped no-literal-dialect-names test", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add deps to core just to make undeclared imports satisfied", "instead": "use extras groups", "why": "that bloats the minimal install"}, {"dont": "remove the guarded-import pattern in AI-primitives", "instead": "declarations only", "why": "the whole point is they remain optional"}], "approach": "Inventory-first. For each undeclared import, check the actual code path\nbefore deciding:\n - Is it shipped source or test-only?\n - Is it guarded behind try/except?\n - Is there a sensible extras group for it (existing `viz`, `dev`,\n or new `ai-primitives`)?\nAdd the minimum declaration needed. Update the inventory doc to close\nthe loop.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "declare-undeclared-runtime-imports", "id": 142, "reason": "Product-level choice; out of scope for this declaration cleanup.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Decide which AI-primitive backend is the canonical default"}], "deps": [], "description": "The dependency audit (`docs/development/dependency-inventory.md`,\nused-but-undeclared section) found three categories of imports that\nappear in shipped or tested source but have no `pyproject.toml`\ndeclaration:\n\n1. `PIL` (Pillow) - image processing in visualization image tests.\n2. `ansi2html` - terminal output \u2192 HTML conversion (likely tooling-only).\n3. AI-primitives ML stack - `sentence_transformers`, `spacy`, `textblob`,\n `torch`, used by `benchbox/core/ai_primitives/`. All are guarded\n behind `try/except ImportError`, but should still be declared in an\n explicit extras group so users who want AI-primitives can install\n a known-good set.\n\nGoal: each undeclared third-party import either gets an explicit\ndeclaration in the appropriate extras group, or is removed if the code\npath is dead. No silent transitive-reach behavior.", "id": "declare-undeclared-runtime-imports", "preserves": ["Existing guarded-import patterns in `benchbox/core/ai_primitives/` continue to work without the optional deps installed", "Fast suite green", "No new unconditional `import PIL`/`import torch`/etc - declarations only, not new imports", "Visualization image tests continue to work where PIL is currently relied on"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Declare currently-undeclared runtime imports (pillow, ansi2html, AI-primitives stack)", "verifications": [{"command": "uv run -- python -c \"import pathlib; t = pathlib.Path('docs/development/dependency-inventory.md').read_text(); assert 'used-but-undeclared' in t.lower(); print('OK: section still present')\"", "description": "Used-but-undeclared section in inventory updated to reflect new state", "expected": "OK: section still present", "item_id": "declare-undeclared-runtime-imports", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite passes", "expected": "no failures", "item_id": "declare-undeclared-runtime-imports", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": [], "notes": "Use: grep -rn '^from PIL\\|^import PIL' benchbox tests\nDecide: extras:viz vs dep-group:dev vs delete based on whether\nshipped source vs test-only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Locate PIL import sites; declare `pillow` in `extras:viz` or dev group, or remove the dependent code.", "wid": "w1"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Locate every ansi2html import site. If only used by tooling, add to `[dependency-groups] dev`. If used by shipped source, add to an appropriate runtime extras group.", "wid": "w2"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory AI-primitives imports: sentence_transformers, spacy, textblob, torch. Confirm guarded import pattern. Add an `ai-primitives` extras group declaring the supported optional set.", "wid": "w3"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update `docs/development/dependency-inventory.md` Used-But-Undeclared section to reflect the new declarations (or document any that were intentionally left undeclared, with reason).", "wid": "w4"}, {"evidence": null, "item_id": "declare-undeclared-runtime-imports", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run fast suite to confirm no install regressions.", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": "For validate_session_cache_control:\n- Extract to a shared mixin or utility function accepting cache_control_query,\n setting_key, disabled_value, and platform_name parameters\n- Both Redshift and Snowflake inherit/call the shared implementation\n\nFor _prompt_default_output_location:\n- Extract to benchbox/platforms/credentials/shared.py with parameters for\n path_scheme, storage_label, permission_note, platform_name\n- BigQuery and Redshift credential modules call the shared function\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Two groups from the duplicate-check report identify shared logic across\ncloud platform adapters:\n\n- Group 21 (86 lines): `validate_session_cache_control` is duplicated between\n redshift.py:1417 and snowflake.py:914. Identical algorithm (result dict,\n try/except/finally, ConfigurationError, strict_validation branching) with\n only SQL query string, setting key, and platform name differing.\n\n- Group 29 (69 lines): `_prompt_default_output_location` is duplicated between\n credentials/bigquery.py:204 and credentials/redshift.py:247. Identical\n interactive prompt flow with only cloud-specific strings differing (gs:// vs\n s3://, storage_bucket vs s3_bucket, permission names).", "id": "dedup-cloud-platform-shared-utilities", "preserves": ["_prompt_default_output_location interactive flow unchanged from user perspective", "validate_session_cache_control returns identical result dict structure"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Extract shared cloud platform utilities (session cache validation, credential prompts)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'redshift or snowflake or bigquery'", "description": "Platform adapter tests pass", "expected": "all tests pass", "item_id": "dedup-cloud-platform-shared-utilities", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-cloud-platform-shared-utilities", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract validate_session_cache_control to shared cloud platform mixin", "wid": "w1"}, {"evidence": null, "item_id": "dedup-cloud-platform-shared-utilities", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor Redshift and Snowflake to use shared cache validation", "wid": "w2"}, {"evidence": null, "item_id": "dedup-cloud-platform-shared-utilities", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _prompt_default_output_location to credentials/shared.py", "wid": "w3"}, {"evidence": null, "item_id": "dedup-cloud-platform-shared-utilities", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor BigQuery and Redshift credentials to use shared prompt", "wid": "w4"}, {"evidence": null, "item_id": "dedup-cloud-platform-shared-utilities", "needs": ["w2", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform tests to verify no regressions", "wid": "w5"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "set it ~10% above baseline", "dont": "set threshold to exact current count", "why": "normal development causes small fluctuations"}, {"INSTEAD": "use ignore_rules with path_contains instead", "dont": "use ignore_function_names (global) for functions that should only be ignored in specific paths", "why": "it would suppress genuine duplicates elsewhere"}], "approach": "Add new entries to the existing ignore_rules list in pyproject.toml under\n[tool.benchbox.duplicates]. Follow the established pattern: group by\npath_contains scope, list specific function_names.\n\nStrategy per group type:\n1. Protocol/ABC stubs (G8,G9,G11,G22): Ignore by function name within\n benchbox/core/dataframe/ and benchbox/platforms/dataframe/ paths\n2. Platform adapter wrappers (G3,G10,G14): Ignore by function name within\n benchbox/platforms/ paths\n3. Benchmark facade methods (G2,G5): Ignore by function name (global or\n path-scoped to core/*/benchmark.py and top-level modules)\n4. TPC-DS/TPC-DI domain (G1,G6,G7,G26,G28): Ignore by function name\n within specific tpcds/tpcdi paths\n5. Multi-backend dispatch (G18,G20): Ignore within unified_frame.py and\n expression_family.py\n6. Benchmark config methods (G16): Ignore get_schema/get_csv_loading_config\n across benchmark modules\n\nAfter adding rules, run `make duplicate-check-verbose` to verify reduction,\nthen lower the threshold proportionally.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "The `make duplicate-check-verbose` report identifies 396 duplicate function groups\ntotaling 12,666 duplicated lines. Analysis of the top 30 groups reveals that 17 are\nintentional boilerplate - interface-conforming stubs, protocol/ABC declarations,\nplatform adapter wrappers, and TPC-DS spec-mandated query patterns.\n\nThese groups should be suppressed via ignore_rules in pyproject.toml\n[tool.benchbox.duplicates] to reduce noise and focus the duplicate checker on\ngenuine consolidation opportunities.\n\nGroups to ignore (by report rank):\n- G1 (344 lines): _fact_map/_role_aliases/_income_band_aliases in tpcds_obt/etl/transformer.py\n- G2 (180 lines): get_queries_by_category/get_query_info facade delegation methods across benchmark.py files\n- G3 (175 lines): _detect_format/col/get_dataframe_operations in dataframe platform adapters\n- G5 (140 lines): execute_rule/get_operation/get_table_row_count domain-specific manager protocols\n- G6 (132 lines): _load_base_queries/_load_etl_queries TPC-DI query dict builders\n- G7 (120 lines): get_queries_by_category/complexity/frequency in tpcdi/query_*.py\n- G8 (117 lines): cast_date/cast_string/col abstract stubs in dataframe context/protocols\n- G9 (116 lines): filter/limit/scalar_to_df abstract method declarations\n- G10 (100 lines): col/from_pandas/lit platform-specific thin wrappers\n- G11 (99 lines): can_provide/create_schema/date_add abstract method stubs across ABCs\n- G14 (91 lines): apply_platform_optimizations cloud platform stubs\n- G16 (90 lines): get_schema/get_csv_loading_config benchmark interface conformance\n- G18 (88 lines): first/floor/last multi-backend dispatch in unified_frame.py\n- G20 (87 lines): max_/mean/std multi-backend dispatch in expression_family.py\n- G22 (80 lines): _do_delete/delete_rows/generate_tuning_clauses Protocol+ABC stubs\n- G26 (75 lines): q33/q60 expression_impl TPC-DS spec-mandated query family\n- G28 (70 lines): q33/q60 pandas_impl TPC-DS spec-mandated query family\n\nAfter adding these rules, the threshold in pyproject.toml should be lowered to\nmatch the new effective duplicate count, preventing regressions.", "id": "dedup-ignore-rules-boilerplate", "preserves": ["Existing ignore_rules in pyproject.toml continue to work (do not break current suppressions)", "make duplicate-check exit code behavior unchanged"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Add ignore rules for 17 boilerplate duplicate groups identified by duplicate-check", "verifications": [{"command": "make duplicate-check", "description": "Duplicate check still passes with lowered threshold", "expected": "PASS: N duplicated lines within threshold of M", "item_id": "dedup-ignore-rules-boilerplate", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose --top-n 10", "description": "No false negatives - consolidation targets still appear in report", "expected": "Report still shows consolidation candidates (Groups 4,12,13,17,19,21,23-25,27,29,30)", "item_id": "dedup-ignore-rules-boilerplate", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: cast_date, cast_string, col, filter, limit, scalar_to_df,\ncan_provide, create_schema, date_add, date_sub, _do_delete, delete_rows,\ngenerate_tuning_clauses.\nPaths: benchbox/core/dataframe/, benchbox/core/operations/,\nbenchbox/platforms/dataframe/, benchbox/core/tuning/ddl_generator.py,\nbenchbox/platforms/base/\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for Protocol/ABC abstract stubs (G8,G9,G11,G22) - ~412 lines", "wid": "w1"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: _detect_format, col, from_pandas, lit, get_dataframe_operations,\napply_platform_optimizations (platform-level stubs).\nPaths: benchbox/platforms/dataframe/, benchbox/platforms/bigquery.py,\nbenchbox/platforms/databricks/, benchbox/platforms/snowflake.py, etc.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for platform adapter wrappers (G3,G10,G14) - ~366 lines", "wid": "w2"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: get_queries_by_category, get_query_info, get_operations_by_category,\nexecute_rule, get_operation, get_table_row_count, get_schema,\nget_csv_loading_config, apply_platform_optimizations (benchmark-level).\nPaths: benchbox/core/*/benchmark.py, benchbox/core/*/dataframe_operations.py,\nbenchbox/*.py (top-level modules)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for benchmark facade/delegation methods (G2,G5,G16) - ~410 lines", "wid": "w3"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: _fact_map, _role_aliases, _income_band_aliases, _load_base_queries,\n_load_etl_queries, get_queries_by_category, get_queries_by_complexity,\nget_queries_by_frequency, q33_expression_impl, q60_expression_impl,\nq33_pandas_impl, q60_pandas_impl.\nPaths: benchbox/core/tpcds_obt/etl/, benchbox/core/tpcdi/queries.py,\nbenchbox/core/tpcdi/query_*.py, benchbox/core/tpcds/dataframe_queries/\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for TPC-DS/TPC-DI domain patterns (G1,G6,G7,G26,G28) - ~741 lines", "wid": "w4"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: first, floor, last, abs_, round_, ceil, sqrt, log, quantile,\nmax_, mean, std, sum, min_, count, var.\nPaths: benchbox/platforms/dataframe/unified_frame.py,\nbenchbox/platforms/dataframe/expression_family.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for multi-backend dispatch methods (G18,G20) - ~175 lines", "wid": "w5"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Run make duplicate-check-verbose, verify suppressed groups are gone,\ncalculate new effective duplicate count, lower threshold to ~10% above\nthe new baseline. Current threshold: 13886, expect to drop significantly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify reduction and lower threshold in pyproject.toml", "wid": "w6"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "set threshold to exact current count", "instead": "set it ~10% above baseline", "why": "normal development causes small fluctuations"}, {"dont": "use ignore_function_names (global) for functions that should only be ignored in specific paths", "instead": "use ignore_rules with path_contains instead", "why": "it would suppress genuine duplicates elsewhere"}], "approach": "Add new entries to the existing ignore_rules list in pyproject.toml under\n[tool.benchbox.duplicates]. Follow the established pattern: group by\npath_contains scope, list specific function_names.\n\nStrategy per group type:\n1. Protocol/ABC stubs (G8,G9,G11,G22): Ignore by function name within\n benchbox/core/dataframe/ and benchbox/platforms/dataframe/ paths\n2. Platform adapter wrappers (G3,G10,G14): Ignore by function name within\n benchbox/platforms/ paths\n3. Benchmark facade methods (G2,G5): Ignore by function name (global or\n path-scoped to core/*/benchmark.py and top-level modules)\n4. TPC-DS/TPC-DI domain (G1,G6,G7,G26,G28): Ignore by function name\n within specific tpcds/tpcdi paths\n5. Multi-backend dispatch (G18,G20): Ignore within unified_frame.py and\n expression_family.py\n6. Benchmark config methods (G16): Ignore get_schema/get_csv_loading_config\n across benchmark modules\n\nAfter adding rules, run `make duplicate-check-verbose` to verify reduction,\nthen lower the threshold proportionally.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "The `make duplicate-check-verbose` report identifies 396 duplicate function groups\ntotaling 12,666 duplicated lines. Analysis of the top 30 groups reveals that 17 are\nintentional boilerplate - interface-conforming stubs, protocol/ABC declarations,\nplatform adapter wrappers, and TPC-DS spec-mandated query patterns.\n\nThese groups should be suppressed via ignore_rules in pyproject.toml\n[tool.benchbox.duplicates] to reduce noise and focus the duplicate checker on\ngenuine consolidation opportunities.\n\nGroups to ignore (by report rank):\n- G1 (344 lines): _fact_map/_role_aliases/_income_band_aliases in tpcds_obt/etl/transformer.py\n- G2 (180 lines): get_queries_by_category/get_query_info facade delegation methods across benchmark.py files\n- G3 (175 lines): _detect_format/col/get_dataframe_operations in dataframe platform adapters\n- G5 (140 lines): execute_rule/get_operation/get_table_row_count domain-specific manager protocols\n- G6 (132 lines): _load_base_queries/_load_etl_queries TPC-DI query dict builders\n- G7 (120 lines): get_queries_by_category/complexity/frequency in tpcdi/query_*.py\n- G8 (117 lines): cast_date/cast_string/col abstract stubs in dataframe context/protocols\n- G9 (116 lines): filter/limit/scalar_to_df abstract method declarations\n- G10 (100 lines): col/from_pandas/lit platform-specific thin wrappers\n- G11 (99 lines): can_provide/create_schema/date_add abstract method stubs across ABCs\n- G14 (91 lines): apply_platform_optimizations cloud platform stubs\n- G16 (90 lines): get_schema/get_csv_loading_config benchmark interface conformance\n- G18 (88 lines): first/floor/last multi-backend dispatch in unified_frame.py\n- G20 (87 lines): max_/mean/std multi-backend dispatch in expression_family.py\n- G22 (80 lines): _do_delete/delete_rows/generate_tuning_clauses Protocol+ABC stubs\n- G26 (75 lines): q33/q60 expression_impl TPC-DS spec-mandated query family\n- G28 (70 lines): q33/q60 pandas_impl TPC-DS spec-mandated query family\n\nAfter adding these rules, the threshold in pyproject.toml should be lowered to\nmatch the new effective duplicate count, preventing regressions.", "id": "dedup-ignore-rules-boilerplate", "preserves": ["Existing ignore_rules in pyproject.toml continue to work (do not break current suppressions)", "make duplicate-check exit code behavior unchanged"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Add ignore rules for 17 boilerplate duplicate groups identified by duplicate-check", "verifications": [{"command": "make duplicate-check", "description": "Duplicate check still passes with lowered threshold", "expected": "PASS: N duplicated lines within threshold of M", "item_id": "dedup-ignore-rules-boilerplate", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose --top-n 10", "description": "No false negatives - consolidation targets still appear in report", "expected": "Report still shows consolidation candidates (Groups 4,12,13,17,19,21,23-25,27,29,30)", "item_id": "dedup-ignore-rules-boilerplate", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: cast_date, cast_string, col, filter, limit, scalar_to_df,\ncan_provide, create_schema, date_add, date_sub, _do_delete, delete_rows,\ngenerate_tuning_clauses.\nPaths: benchbox/core/dataframe/, benchbox/core/operations/,\nbenchbox/platforms/dataframe/, benchbox/core/tuning/ddl_generator.py,\nbenchbox/platforms/base/\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for Protocol/ABC abstract stubs (G8,G9,G11,G22) - ~412 lines", "wid": "w1"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: _detect_format, col, from_pandas, lit, get_dataframe_operations,\napply_platform_optimizations (platform-level stubs).\nPaths: benchbox/platforms/dataframe/, benchbox/platforms/bigquery.py,\nbenchbox/platforms/databricks/, benchbox/platforms/snowflake.py, etc.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for platform adapter wrappers (G3,G10,G14) - ~366 lines", "wid": "w2"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: get_queries_by_category, get_query_info, get_operations_by_category,\nexecute_rule, get_operation, get_table_row_count, get_schema,\nget_csv_loading_config, apply_platform_optimizations (benchmark-level).\nPaths: benchbox/core/*/benchmark.py, benchbox/core/*/dataframe_operations.py,\nbenchbox/*.py (top-level modules)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for benchmark facade/delegation methods (G2,G5,G16) - ~410 lines", "wid": "w3"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: _fact_map, _role_aliases, _income_band_aliases, _load_base_queries,\n_load_etl_queries, get_queries_by_category, get_queries_by_complexity,\nget_queries_by_frequency, q33_expression_impl, q60_expression_impl,\nq33_pandas_impl, q60_pandas_impl.\nPaths: benchbox/core/tpcds_obt/etl/, benchbox/core/tpcdi/queries.py,\nbenchbox/core/tpcdi/query_*.py, benchbox/core/tpcds/dataframe_queries/\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for TPC-DS/TPC-DI domain patterns (G1,G6,G7,G26,G28) - ~741 lines", "wid": "w4"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": [], "notes": "Functions: first, floor, last, abs_, round_, ceil, sqrt, log, quantile,\nmax_, mean, std, sum, min_, count, var.\nPaths: benchbox/platforms/dataframe/unified_frame.py,\nbenchbox/platforms/dataframe/expression_family.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for multi-backend dispatch methods (G18,G20) - ~175 lines", "wid": "w5"}, {"evidence": null, "item_id": "dedup-ignore-rules-boilerplate", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Run make duplicate-check-verbose, verify suppressed groups are gone,\ncalculate new effective duplicate count, lower threshold to ~10% above\nthe new baseline. Current threshold: 13886, expect to drop significantly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify reduction and lower threshold in pyproject.toml", "wid": "w6"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": "For BaseQueryCatalog:\n- Check if ai_primitives, metadata_primitives, read_primitives query classes\n already share a base. If so, add get_query there. If not, create a mixin.\n- The shared get_query needs: self._entries dict, self._queries dict,\n ValueError on missing, skip_on check, variant selection.\n\nFor TranslatableQueryMixin:\n- Create a simple mixin with translate_query_text that delegates to\n benchbox.utils.dialect_utils.translate_sql_query\n- Parameterize source_dialect (currently hardcoded as \"netezza\" in all copies)\n as a class attribute with default \"netezza\"\n- Have amplab, coffeeshop, h2odb, read_primitives, ssb inherit from mixin\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Two groups from the duplicate-check report identify consolidation targets\nin the query catalog and translation layer:\n\n- Group 27 (74 lines): `get_query` is char-for-char identical across\n ai_primitives/queries.py, metadata_primitives/queries.py, and\n read_primitives/queries.py. The logic (lookup, skip_on check, variant\n selection, fallback) is a generic query catalog pattern.\n\n- Group 12 (95 lines): `translate_query_text` is byte-for-byte identical\n across 5 benchmark classes (amplab, coffeeshop, h2odb, read_primitives, ssb).\n All call `translate_sql_query(query=..., target_dialect=..., source_dialect=\"netezza\")`.\n\nThese should be extracted to:\n1. A `BaseQueryCatalog` class with the shared `get_query` implementation\n2. A `TranslatableQueryMixin` with the shared `translate_query_text`", "id": "dedup-query-catalog-base", "preserves": ["get_query(query_id, dialect) returns identical results for all query IDs", "translate_query_text(text, dialect) returns identical translations"], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Extract BaseQueryCatalog for shared get_query logic and TranslatableQueryMixin", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'primitives or amplab or coffeeshop or h2odb or ssb'", "description": "All affected benchmark tests pass", "expected": "all tests pass", "item_id": "dedup-query-catalog-base", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-query-catalog-base", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create BaseQueryCatalog with shared get_query (~37 lines per copy, 3 copies)", "wid": "w1"}, {"evidence": null, "item_id": "dedup-query-catalog-base", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor ai_primitives, metadata_primitives, read_primitives to use BaseQueryCatalog", "wid": "w2"}, {"evidence": null, "item_id": "dedup-query-catalog-base", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create TranslatableQueryMixin with shared translate_query_text", "wid": "w3"}, {"evidence": null, "item_id": "dedup-query-catalog-base", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor amplab, coffeeshop, h2odb, read_primitives, ssb to use TranslatableQueryMixin", "wid": "w4"}, {"evidence": null, "item_id": "dedup-query-catalog-base", "needs": ["w2", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run tests to verify no regressions", "wid": "w5"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "AMPLab, ClickBench, and SSB benchmarks had near-identical implementations of\nrun_benchmark (AMPLab/ClickBench) and _load_data/_load_table_data (AMPLab/SSB).\n\nCreated SimpleBenchmarkMixin in benchbox/core/simple_benchmark_mixin.py that\nextracts the shared logic, parameterized by _benchmark_label and _table_load_order\nclass attributes.\n\nSSB's run_benchmark was NOT extracted because it uses a different return format\n(query_results list vs queries dict).", "id": "dedup-simple-benchmark-base-methods", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Extract shared run_benchmark and _load_data to SimpleBenchmarkMixin", "verifications": [], "work": [{"evidence": null, "item_id": "dedup-simple-benchmark-base-methods", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create SimpleBenchmarkMixin with shared run_benchmark and _load_data", "wid": "w1"}, {"evidence": null, "item_id": "dedup-simple-benchmark-base-methods", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update AMPLab to use SimpleBenchmarkMixin (run_benchmark + _load_data)", "wid": "w2"}, {"evidence": null, "item_id": "dedup-simple-benchmark-base-methods", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update ClickBench to use SimpleBenchmarkMixin (run_benchmark only)", "wid": "w3"}, {"evidence": null, "item_id": "dedup-simple-benchmark-base-methods", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update SSB to use SimpleBenchmarkMixin (_load_data only, keep run_benchmark)", "wid": "w4"}, {"evidence": null, "item_id": "dedup-simple-benchmark-base-methods", "needs": ["w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run tests to verify no regressions", "wid": "w5"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "use a mixin that can be composed in", "dont": "create a deep inheritance chain", "why": "the adapter hierarchy is already 3+ levels"}], "approach": "1. Create `benchbox/platforms/base/spark_execution_mixin.py` with a\n `SparkExecutionMixin` class containing:\n - `_load_data_spark(self, benchmark, data_dir, connection)` - the shared\n load logic using DataSourceResolver, format detection, chunk loading\n - `_execute_query_spark(self, query, connection, **kwargs)` - the shared\n execution logic with dry-run, caching, validation, and error handling\n2. Have both LakeSail and Spark adapters inherit from the mixin\n3. Reduce each adapter's `load_data` and `execute_query` to thin wrappers\n or direct delegation to the mixin methods\n\nReference the existing `benchbox/platforms/base/cloud_spark/` directory\nwhich already demonstrates the mixin pattern for cloud Spark adapters.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Groups 13 and 30 from the duplicate-check report identify near-identical\nimplementations of `load_data` (~92 lines) and `execute_query` (~69 lines)\nbetween `benchbox/platforms/lakesail.py` and `benchbox/platforms/spark.py`.\n\nLakeSail is a Spark Connect-based platform that shares the same DataFrame\nloading and query execution patterns as standard PySpark. Both adapters\nalready inherit from a common base that provides helper methods like\n`_build_dry_run_result` and `_build_query_failure_result`.\n\nThe shared logic should be extracted to a mixin or intermediate base class,\neliminating ~161 duplicated lines.", "id": "dedup-spark-shared-base", "preserves": ["LakeSailAdapter.execute_query() and SparkAdapter.execute_query() return identical result dicts", "LakeSailAdapter.load_data() and SparkAdapter.load_data() public API unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Extract shared Spark base for LakeSail/Spark duplicated load_data and execute_query", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'spark or lakesail'", "description": "Platform adapter tests pass", "expected": "all tests pass", "item_id": "dedup-spark-shared-base", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose --top-n 30", "description": "Duplicate check shows reduction for groups 13 and 30", "expected": "load_data and execute_query lakesail/spark groups no longer appear", "item_id": "dedup-spark-shared-base", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-spark-shared-base", "needs": [], "notes": "Extract from spark.py:481. Both lakesail.py and spark.py use identical:\nDataSourceResolver, _normalize_and_validate_file_paths, _get_table_schema,\ndetect_file_format, parquet/CSV branching, row counting, timing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create SparkExecutionMixin with shared load_data logic (~92 lines)", "wid": "w1"}, {"evidence": null, "item_id": "dedup-spark-shared-base", "needs": ["w1"], "notes": "Extract from spark.py:631. Both files share identical: dry_run check,\nspark.catalog.clearCache(), spark.sql().collect(), validation block,\n_build_query_result_with_validation, _build_query_failure_result.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add shared execute_query logic to SparkExecutionMixin (~69 lines)", "wid": "w2"}, {"evidence": null, "item_id": "dedup-spark-shared-base", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor LakeSail and Spark adapters to use SparkExecutionMixin", "wid": "w3"}, {"evidence": null, "item_id": "dedup-spark-shared-base", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform adapter tests to verify no regressions", "wid": "w4"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "create a deep inheritance chain", "instead": "use a mixin that can be composed in", "why": "the adapter hierarchy is already 3+ levels"}], "approach": "1. Create `benchbox/platforms/base/spark_execution_mixin.py` with a\n `SparkExecutionMixin` class containing:\n - `_load_data_spark(self, benchmark, data_dir, connection)` - the shared\n load logic using DataSourceResolver, format detection, chunk loading\n - `_execute_query_spark(self, query, connection, **kwargs)` - the shared\n execution logic with dry-run, caching, validation, and error handling\n2. Have both LakeSail and Spark adapters inherit from the mixin\n3. Reduce each adapter's `load_data` and `execute_query` to thin wrappers\n or direct delegation to the mixin methods\n\nReference the existing `benchbox/platforms/base/cloud_spark/` directory\nwhich already demonstrates the mixin pattern for cloud Spark adapters.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Groups 13 and 30 from the duplicate-check report identify near-identical\nimplementations of `load_data` (~92 lines) and `execute_query` (~69 lines)\nbetween `benchbox/platforms/lakesail.py` and `benchbox/platforms/spark.py`.\n\nLakeSail is a Spark Connect-based platform that shares the same DataFrame\nloading and query execution patterns as standard PySpark. Both adapters\nalready inherit from a common base that provides helper methods like\n`_build_dry_run_result` and `_build_query_failure_result`.\n\nThe shared logic should be extracted to a mixin or intermediate base class,\neliminating ~161 duplicated lines.", "id": "dedup-spark-shared-base", "preserves": ["LakeSailAdapter.execute_query() and SparkAdapter.execute_query() return identical result dicts", "LakeSailAdapter.load_data() and SparkAdapter.load_data() public API unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/base/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Extract shared Spark base for LakeSail/Spark duplicated load_data and execute_query", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v -k 'spark or lakesail'", "description": "Platform adapter tests pass", "expected": "all tests pass", "item_id": "dedup-spark-shared-base", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose --top-n 30", "description": "Duplicate check shows reduction for groups 13 and 30", "expected": "load_data and execute_query lakesail/spark groups no longer appear", "item_id": "dedup-spark-shared-base", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-spark-shared-base", "needs": [], "notes": "Extract from spark.py:481. Both lakesail.py and spark.py use identical:\nDataSourceResolver, _normalize_and_validate_file_paths, _get_table_schema,\ndetect_file_format, parquet/CSV branching, row counting, timing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create SparkExecutionMixin with shared load_data logic (~92 lines)", "wid": "w1"}, {"evidence": null, "item_id": "dedup-spark-shared-base", "needs": ["w1"], "notes": "Extract from spark.py:631. Both files share identical: dry_run check,\nspark.catalog.clearCache(), spark.sql().collect(), validation block,\n_build_query_result_with_validation, _build_query_failure_result.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add shared execute_query logic to SparkExecutionMixin (~69 lines)", "wid": "w2"}, {"evidence": null, "item_id": "dedup-spark-shared-base", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor LakeSail and Spark adapters to use SparkExecutionMixin", "wid": "w3"}, {"evidence": null, "item_id": "dedup-spark-shared-base", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform adapter tests to verify no regressions", "wid": "w4"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": "1. Create or add to `benchbox/core/sql_utils.py` (or find an existing\n utilities module for SQL string manipulation)\n2. Extract the canonical 4-copy version as `normalize_table_name_in_sql(sql)`\n3. Handle the Redshift variant by parameterizing the regex pattern or having\n Redshift call the shared version with a flag\n4. Replace all 5 platform copies with imports from the shared module\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Group 25 from the duplicate-check report identifies 5 copies of\n`_normalize_table_name_in_sql` across platform adapters:\n- firebolt.py:1072\n- presto.py:956\n- redshift.py:1612\n- spark.py:722\n- trino.py:956\n\nFour of five copies are byte-for-byte identical (same two re.sub patterns\nwith identical regex and lambda replacements). Redshift has a minor variant\n(missing the IF NOT EXISTS group in its CREATE TABLE pattern).\n\nThis is a pure utility function with no platform-specific behavior - it just\nlowercases table names in SQL statements. It belongs in a shared module.", "id": "dedup-sql-utilities", "preserves": ["Redshift-specific variant still handles its CREATE TABLE pattern correctly", "SQL table name normalization produces identical output for all platforms"], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Extract _normalize_table_name_in_sql to shared SQL utility module", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -v", "description": "Platform adapter tests pass", "expected": "all tests pass", "item_id": "dedup-sql-utilities", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-sql-utilities", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract normalize_table_name_in_sql to benchbox/core/sql_utils.py", "wid": "w1"}, {"evidence": null, "item_id": "dedup-sql-utilities", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace 5 platform copies with imports from shared module", "wid": "w2"}, {"evidence": null, "item_id": "dedup-sql-utilities", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform tests to verify no regressions", "wid": "w3"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "the helper must accept an optional count and fall back to result extraction", "dont": "assume record count is always a pre-computed variable", "why": "generate_time_dimension extracts it from the query result"}, {"INSTEAD": "each caller computes its own count before invoking the helper", "dont": "hardcode count formula as base_X * scale_factor", "why": "generate_account_dimension uses num_customers * 1.5"}], "approach": "Extract a `_generate_table(self, conn, table_name, num_records, insert_sql)`\nhelper that handles: progress logging, timing via mono_time/elapsed_seconds,\nconn.execute(insert_sql), stats dict updates (records_generated, tables_generated,\ngeneration_times, sql_operations), completion logging.\n\nThe helper should accept an optional `num_records` parameter. When provided,\nit is used for stats and logging. When None, the helper extracts the count\nfrom the query result (as generate_time_dimension currently does).\n\nEach of the 6 generator methods reduces to: compute count (if known),\nbuild SQL string, call self._generate_table(conn, \"TableName\", count, sql).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Group 4 from the duplicate-check report identifies 6 methods with identical\nscaffolding in benchbox/core/tpcdi/generator/sql.py:\n- generate_time_dimension (line ~225)\n- generate_company_dimension (line 263)\n- generate_security_dimension (line 315)\n- generate_customer_dimension (line 359)\n- generate_account_dimension (line 427)\n- generate_trade_facts (line 467)\n\nAll 6 share the same pattern: progress log \u2192 mono_time() \u2192 conn.execute(sql)\n\u2192 elapsed_seconds \u2192 stats update (4 keys) \u2192 completion log \u2192 return count.\nOnly the SQL INSERT body and record count computation differ.\n\nNote: generate_time_dimension extracts its record count from the query result\nrather than pre-computing it. generate_account_dimension derives its count\nfrom a formula (num_customers * 1.5) rather than base_X * scale_factor.", "id": "dedup-tpcdi-dimension-generator", "preserves": ["Progress logging messages include table name and record count", "generate_time_dimension returns count extracted from query result, not pre-computed", "generation_stats dict populated identically after each dimension generation"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/generator/sql.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Extract _generate_dimension helper for TPC-DI SQL generator scaffolding", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k tpcdi", "description": "TPC-DI tests pass", "expected": "all tests pass", "item_id": "dedup-tpcdi-dimension-generator", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-tpcdi-dimension-generator", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _generate_table helper and refactor all 6 generator methods to use it", "wid": "w1"}, {"evidence": null, "item_id": "dedup-tpcdi-dimension-generator", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-DI tests to verify no regressions", "wid": "w2"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "assume record count is always a pre-computed variable", "instead": "the helper must accept an optional count and fall back to result extraction", "why": "generate_time_dimension extracts it from the query result"}, {"dont": "hardcode count formula as base_X * scale_factor", "instead": "each caller computes its own count before invoking the helper", "why": "generate_account_dimension uses num_customers * 1.5"}], "approach": "Extract a `_generate_table(self, conn, table_name, num_records, insert_sql)`\nhelper that handles: progress logging, timing via mono_time/elapsed_seconds,\nconn.execute(insert_sql), stats dict updates (records_generated, tables_generated,\ngeneration_times, sql_operations), completion logging.\n\nThe helper should accept an optional `num_records` parameter. When provided,\nit is used for stats and logging. When None, the helper extracts the count\nfrom the query result (as generate_time_dimension currently does).\n\nEach of the 6 generator methods reduces to: compute count (if known),\nbuild SQL string, call self._generate_table(conn, \"TableName\", count, sql).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Group 4 from the duplicate-check report identifies 6 methods with identical\nscaffolding in benchbox/core/tpcdi/generator/sql.py:\n- generate_time_dimension (line ~225)\n- generate_company_dimension (line 263)\n- generate_security_dimension (line 315)\n- generate_customer_dimension (line 359)\n- generate_account_dimension (line 427)\n- generate_trade_facts (line 467)\n\nAll 6 share the same pattern: progress log \u2192 mono_time() \u2192 conn.execute(sql)\n\u2192 elapsed_seconds \u2192 stats update (4 keys) \u2192 completion log \u2192 return count.\nOnly the SQL INSERT body and record count computation differ.\n\nNote: generate_time_dimension extracts its record count from the query result\nrather than pre-computing it. generate_account_dimension derives its count\nfrom a formula (num_customers * 1.5) rather than base_X * scale_factor.", "id": "dedup-tpcdi-dimension-generator", "preserves": ["Progress logging messages include table name and record count", "generate_time_dimension returns count extracted from query result, not pre-computed", "generation_stats dict populated identically after each dimension generation"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/generator/sql.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Extract _generate_dimension helper for TPC-DI SQL generator scaffolding", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k tpcdi", "description": "TPC-DI tests pass", "expected": "all tests pass", "item_id": "dedup-tpcdi-dimension-generator", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-tpcdi-dimension-generator", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _generate_table helper and refactor all 6 generator methods to use it", "wid": "w1"}, {"evidence": null, "item_id": "dedup-tpcdi-dimension-generator", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-DI tests to verify no regressions", "wid": "w2"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": "1. Create or add to `benchbox/core/tpcds/constants.py`:\n `TPCDS_TABLE_NAMES: list[str] = [\"call_center\", \"catalog_page\", ...]`\n2. Replace the 3 copies with imports from constants module\n3. For _empty_results, determine the canonical location (likely tpc_patterns.py)\n and have base.py import from there\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-22T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Group 15 from the duplicate-check report identifies two consolidation targets:\n\n1. TPC-DS canonical table names (~24-element list): duplicated identically in\n - benchbox/core/tpcds/c_tools.py:136\n - benchbox/core/tpcds/generator/manager.py:178\n - benchbox/core/tpcds/benchmark/runner.py:404\n This is a TPC-DS specification constant that should be defined once.\n\n2. _empty_results (zero-valued results dict): duplicated between\n benchbox/core/tpc_patterns.py:559 and benchbox/base.py:178.\n This is a shared result schema for stream/throughput result builders.", "id": "dedup-tpcds-constants-empty-results", "preserves": ["Table name list includes all 24 TPC-DS tables", "generator/manager.py copy includes 'dbgen_version' (25th entry) - handle separately"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpc_patterns.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Extract TPC-DS canonical table list and shared _empty_results to constants", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k tpcds", "description": "TPC-DS tests pass", "expected": "all tests pass", "item_id": "dedup-tpcds-constants-empty-results", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-tpcds-constants-empty-results", "needs": [], "notes": "constants.py already had TPCDS_TABLE_NAMES; c_tools.py and generator/manager.py\nalready imported it. Replaced 3 hardcoded lists in runner.py (get_available_tables,\nget_table_loading_order, load method). Added TPCDS_TABLE_LOADING_ORDER constant\nfor the two identical dependency-ordered lists.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create TPCDS_TABLE_NAMES constant and replace 3 copies", "wid": "w1"}, {"evidence": null, "item_id": "dedup-tpcds-constants-empty-results", "needs": [], "notes": "_empty_results only exists in tpc_patterns.py (no duplicate in base.py).\nNo consolidation needed; the reported duplication does not exist in the\ncurrent codebase.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate _empty_results to single canonical location", "wid": "w2"}, {"evidence": null, "item_id": "dedup-tpcds-constants-empty-results", "needs": ["w1", "w2"], "notes": "538 passed in 141s", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-DS tests", "wid": "w3"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": "Extract a `_insert_returns_generic(self, connection, sales_table, returns_table,\nsales_columns, returns_columns, row_generator_fn, estimated_rows)` method.\nEach channel-specific method becomes a 5-line call site passing its specific\ntable names, column lists, and row generator callback.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-21T00:00:00Z", "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Group 17 from the duplicate-check report identifies 3 copies of nearly\nidentical logic in benchbox/core/tpcds/maintenance_operations.py:\n- _insert_store_returns (line 470, ~44 lines)\n- _insert_catalog_returns (line 515, ~44 lines)\n- _insert_web_returns (line 564, ~44 lines)\n\nAll three follow the same algorithm:\n1. Log the insert count\n2. Get parameter placeholder\n3. Build SELECT ... FROM {sales_table} ORDER BY RANDOM() LIMIT {n}\n4. Execute query, fetch records\n5. Handle empty result / error cases identically\n6. Loop over records calling self._generate_{type}_returns_from_sale(sale)\n7. Call self._execute_batched_insert(connection, table, columns, rows, N)\n\nOnly the table/column names and the row generator callback differ.", "id": "dedup-tpcds-maintenance-returns", "preserves": ["Batched insert behavior identical (same batch size, same error handling)", "_insert_store_returns, _insert_catalog_returns, _insert_web_returns public API unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpcds/maintenance_operations.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Parameterize _insert_{store,catalog,web}_returns in TPC-DS maintenance operations", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k tpcds", "description": "TPC-DS tests pass", "expected": "all tests pass", "item_id": "dedup-tpcds-maintenance-returns", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-tpcds-maintenance-returns", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create _insert_returns_generic parameterized helper method", "wid": "w1"}, {"evidence": null, "item_id": "dedup-tpcds-maintenance-returns", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor _insert_{store,catalog,web}_returns to use generic helper", "wid": "w2"}, {"evidence": null, "item_id": "dedup-tpcds-maintenance-returns", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-DS tests to verify no regressions", "wid": "w3"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": "Create a `_rolling_average_impl(ctx, sales_table, date_sk_col, item_sk_col,\nsales_price_col, channel_table, channel_join_key_left, channel_join_key_right,\nchannel_cols, year)` helper function. Q47 and Q57 each become a ~5-line\ncall site passing their channel-specific parameters.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-21T00:00:00Z", "deferrals": [], "deps": [], "description": "Group 19 from the duplicate-check report identifies that q47_pandas_impl and\nq57_pandas_impl in benchbox/core/tpcds/dataframe_queries/queries.py (~87 lines\neach) implement the same rolling average algorithm. The differences are:\n- Sales table: store_sales vs catalog_sales\n- Column prefixes: ss_ vs cs_\n- Channel dimension: store (s_store_sk/s_store_name/s_company_name)\n vs call_center (cc_call_center_sk/cc_name)\n- Groupby keys: 4-key with company_name vs 3-key without\n\nThe TPC-DS specification intentionally mirrors Q47 and Q57 with the\nstore/catalog channel substitution. A parameterized helper is appropriate\nsince the algorithm is identical.\n\nNote: Lower priority because this is within TPC-DS spec-conforming query code\nwhere readability and auditability against the spec matters. The expression\nimplementations (Q33/Q60) are left as boilerplate for spec compliance reasons.", "id": "dedup-tpcds-rolling-average-queries", "preserves": ["Q47 and Q57 produce identical query results to current implementations"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/tpcds/dataframe_queries/queries.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Extract shared rolling average helper for TPC-DS Q47/Q57 pandas implementations", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k tpcds", "description": "TPC-DS tests pass", "expected": "all tests pass", "item_id": "dedup-tpcds-rolling-average-queries", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dedup-tpcds-rolling-average-queries", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create _rolling_average_pandas_impl parameterized helper", "wid": "w1"}, {"evidence": null, "item_id": "dedup-tpcds-rolling-average-queries", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor q47_pandas_impl and q57_pandas_impl to use helper", "wid": "w2"}, {"evidence": null, "item_id": "dedup-tpcds-rolling-average-queries", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-DS dataframe query tests", "wid": "w3"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change public method signatures", "why": "only extract internal dispatch logic"}, {"INSTEAD": "(not specified)", "dont": "merge expression_family and pandas_family into one class", "why": "only share the loading function"}, {"INSTEAD": "(not specified)", "dont": "use getattr-based dynamic dispatch for the aggregation helper", "why": "pass explicit function references to maintain IDE navigability and type safety"}], "approach": "1. G7 - Temporal extraction:\n Add `_extract_date_part(self, part_name: str)` private method to\n the relevant UnifiedExpr class that handles the platform dispatch.\n Each public method (year, month, day, hour, minute) becomes:\n `return self._extract_date_part(\"year\")`.\n\n2. G10 - Polars window functions:\n Add `_polars_window_rank(self, method: str, order_by, partition_by)`\n private method. Each public method becomes:\n `return self._polars_window_rank(\"min\", order_by, partition_by)`.\n\n3. G16 - Aggregation dispatch:\n Add `_apply_aggregation(self, pyspark_fn, datafusion_fn, polars_method)`\n private method that handles the 3-platform dispatch.\n Each aggregation method becomes a one-liner.\n\n4. G23 - Data loading:\n Extract shared logic into a standalone function\n `_load_tables_from_data_source_impl(ctx, data_dir, schema_info)`\n in benchbox/platforms/dataframe/shared_loading.py or as a method\n on a shared base. Both family classes call this implementation.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave2-ignore-rules"], "description": "Wave-2 duplicate-check triage (2026-03-12) identifies 4 consolidation-worthy\ngroups in the DataFrame platform layer (unified_frame.py, polars_df.py,\nexpression_family.py, pandas_family.py). These are identical dispatch patterns\nthat vary only by the underlying function name or method parameter.\n\nTargets (report rank \u2192 dup lines):\n\nG7 (60 dup lines): day/hour/minute/year/month temporal extraction in\n unified_frame.py:585-649. 5 methods with identical multi-platform\n dispatch (PySpark \u2192 DataFusion \u2192 Polars fallback), differing only\n in the extraction function name (e.g., F.year vs F.month).\n\nG10 (58 dup lines): window_rank/window_row_number/window_dense_rank in\n polars_df.py:532-591. 3 methods with identical pattern: unpack\n order_by \u2192 create Polars expression \u2192 apply rank with different\n method parameter (\"min\", \"ordinal\", \"dense\") \u2192 handle partition.\n\nG16 (55 dup lines): max/mean/min aggregation methods in\n unified_frame.py:1018-1127. 6 copies with identical multi-platform\n dispatch for aggregations, differing only by the function name\n (F.sum, F.avg, F.min, F.max, etc.).\n\nG23 (52 dup lines): load_tables_from_data_source in\n expression_family.py:1230 and pandas_family.py:1037. Identical\n data loading logic (resolve source \u2192 iterate tables \u2192 normalize\n paths \u2192 load) differing only by context type signature.", "id": "dedup-wave2-dataframe-platform", "preserves": ["Aggregation methods dispatch correctly across PySpark, DataFusion, and Polars", "All temporal extraction methods return identical UnifiedExpr types", "Data loading produces identical table_stats and loaded data for both families", "Polars window functions produce identical rank semantics (min/ordinal/dense)", "Type signatures on public methods remain compatible for callers"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/expression_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/polars_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/shared_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/unified_frame.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate DataFrame platform helpers: temporal extraction, window funcs, aggregation, data loading", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'dataframe or unified'", "description": "DataFrame platform tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-dataframe-platform", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k polars", "description": "Polars adapter tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-dataframe-platform", "last_result": null, "last_run": null, "seq": 2}, {"command": "make duplicate-check-verbose 2>&1 | grep -c 'day, hour, minute\\|window_dense_rank, window_rank\\|max, mean, min\\|load_tables_from_data'", "description": "Duplicate check shows reduction for these groups", "expected": "0 or reduced counts", "item_id": "dedup-wave2-dataframe-platform", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _extract_date_part helper in unified_frame.py, refactor 5 temporal methods", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _polars_window_rank helper in polars_df.py, refactor 3 window methods", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _apply_aggregation helper in unified_frame.py, refactor 6 aggregation methods", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract shared load_tables_from_data_source into shared_loading.py, update both families", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run DataFrame platform tests: pytest -k 'dataframe or unified_frame or polars'", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w6"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change the _generate_table helper from the prior dedup", "why": "the new work targets the SQL construction step above it"}, {"INSTEAD": "(not specified)", "dont": "create a deep inheritance hierarchy for source formats", "why": "a simple base method with a method_name parameter is sufficient"}, {"INSTEAD": "(not specified)", "dont": "over-generalize the manifest writer", "why": "it should accept the same parameters the current implementations use, not invent new abstractions"}], "approach": "1. G1 - TPC-DI dimension SQL pattern:\n Extract a `_build_dimension_insert_sql(table_name, num_records,\n column_specs, lookup_join_specs)` helper in sql.py that constructs\n the INSERT SELECT with CROSS JOINs. Each generate_* method reduces\n to: compute count, call _build_dimension_insert_sql, pass result\n to _generate_table.\n\n2. G3 - Manifest writing:\n Extract `write_generator_manifest(output_dir, benchmark_name,\n scale_factor, table_paths, compression_metadata, ...)` into\n benchbox/core/manifest_utils.py (new file). Each generator's\n _write_manifest becomes a one-liner calling this utility.\n\n3. G4 - Schema SQL collection:\n Extract `collect_create_table_sql(get_single_table_fn, table_order,\n dialect, enable_pks, enable_fks)` into benchbox/core/schema_utils.py\n (new file). Each benchmark's get_all_create_table_sql becomes a\n thin wrapper passing its table order and per-table function.\n\n4. G20 - TPC-DI source data generation:\n Extract a `_generate_source_data(self, gen_method_name, **kwargs)`\n base method in the SourceDataFormat ABC. Each subclass reduces to\n `return self._generate_source_data('_generate_oltp_data', **kwargs)`.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave2-ignore-rules"], "description": "Wave-2 duplicate-check triage (2026-03-12) identifies 4 consolidation-worthy\ngroups in the benchmark generator and schema layers. These are parallel\nimplementations that can be centralized into shared core utilities.\n\nTargets (report rank \u2192 dup lines):\n\nG1 (108 dup lines): TPC-DI dimension generators in core/tpcdi/generator/sql.py\n 4 methods (generate_company_dimension, generate_customer_dimension,\n generate_security_dimension, generate_account_dimension) at lines 266, 303,\n 332, 410. All share identical structure: compute scaled row count \u2192 build\n parameterized INSERT SELECT with CROSS JOINs \u2192 call _generate_table().\n Only column lists and join specs differ.\n NOTE: Prior TODO dedup-tpcdi-dimension-generator (completed 2026-02-21)\n already extracted _generate_table(). These 4 methods still share the\n inner SQL construction pattern that was NOT part of that extraction.\n\nG3 (60 dup lines): _write_manifest across 4 generators\n amplab/generator.py:390, clickbench/generator.py:383,\n h2odb/generator.py:299, joinorder/generator.py:635.\n All follow identical logic: check empty table_paths \u2192 create\n DataGenerationManifest \u2192 iterate table_paths adding entries \u2192 manifest.write().\n Only the benchmark_name string differs.\n\nG4 (60 dup lines): get_all_create_table_sql in 3 schema modules\n amplab/schema.py:107, read_primitives/schema.py:117, ssb/schema.py:175.\n All iterate a table_order list, call get_create_table_sql per table,\n collect and join SQL. Differs only in table ordering and the\n per-table SQL function called.\n\nG20 (52 dup lines): generate_data in 3 TPC-DI source format classes\n core/tpcdi/etl/sources.py:67, :128, :189 (CSVSourceFormat,\n XMLSourceFormat, FixedWidthSourceFormat). All extract scale_factor\n and output_dir from kwargs, init TPCDISourceDataGenerator, call\n format-specific _generate_*_data method.", "id": "dedup-wave2-generator-schema-utils", "preserves": ["DataGenerationManifest format and content identical after refactor", "Generated SQL for TPC-DI dimensions byte-for-byte identical", "TPC-DI source data generation produces same files and content", "get_all_create_table_sql output identical across all 3 schema modules"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/amplab/"}, {"kind": "only_modify", "path_glob": "benchbox/core/clickbench/"}, {"kind": "only_modify", "path_glob": "benchbox/core/h2odb/"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/"}, {"kind": "only_modify", "path_glob": "benchbox/core/manifest_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/schema_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate generator manifest writing, schema SQL helpers, and TPC-DI source generators", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k tpcdi", "description": "TPC-DI dimension generation tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k 'amplab or clickbench or h2odb or joinorder'", "description": "Generator manifest tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -v -k 'schema or create_table'", "description": "Schema SQL generation tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 3}, {"command": "make duplicate-check-verbose 2>&1 | head -5", "description": "Duplicate check shows reduction", "expected": "duplicated lines lower than 9326", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _build_dimension_insert_sql helper and refactor 4 TPC-DI dimension generators", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-DI tests to verify dimension generation unchanged", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract write_generator_manifest utility into benchbox/core/manifest_utils.py", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor amplab, clickbench, h2odb, joinorder generators to use shared manifest writer", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run generator tests: pytest -k 'amplab or clickbench or h2odb or joinorder'", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract collect_create_table_sql into benchbox/core/schema_utils.py", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor amplab, read_primitives, ssb schema modules to use shared helper", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _generate_source_data base method in TPC-DI SourceDataFormat ABC", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w2", "w5", "w7", "w8"], "notes": "1442 tests pass. Duplicate lines: 8614\u21928464 (-150).", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w9"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change public method signatures", "instead": "(not specified)", "why": "only extract internal dispatch logic"}, {"dont": "merge expression_family and pandas_family into one class", "instead": "(not specified)", "why": "only share the loading function"}, {"dont": "use getattr-based dynamic dispatch for the aggregation helper", "instead": "(not specified)", "why": "pass explicit function references to maintain IDE navigability and type safety"}], "approach": "1. G7 - Temporal extraction:\n Add `_extract_date_part(self, part_name: str)` private method to\n the relevant UnifiedExpr class that handles the platform dispatch.\n Each public method (year, month, day, hour, minute) becomes:\n `return self._extract_date_part(\"year\")`.\n\n2. G10 - Polars window functions:\n Add `_polars_window_rank(self, method: str, order_by, partition_by)`\n private method. Each public method becomes:\n `return self._polars_window_rank(\"min\", order_by, partition_by)`.\n\n3. G16 - Aggregation dispatch:\n Add `_apply_aggregation(self, pyspark_fn, datafusion_fn, polars_method)`\n private method that handles the 3-platform dispatch.\n Each aggregation method becomes a one-liner.\n\n4. G23 - Data loading:\n Extract shared logic into a standalone function\n `_load_tables_from_data_source_impl(ctx, data_dir, schema_info)`\n in benchbox/platforms/dataframe/shared_loading.py or as a method\n on a shared base. Both family classes call this implementation.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave2-ignore-rules"], "description": "Wave-2 duplicate-check triage (2026-03-12) identifies 4 consolidation-worthy\ngroups in the DataFrame platform layer (unified_frame.py, polars_df.py,\nexpression_family.py, pandas_family.py). These are identical dispatch patterns\nthat vary only by the underlying function name or method parameter.\n\nTargets (report rank \u2192 dup lines):\n\nG7 (60 dup lines): day/hour/minute/year/month temporal extraction in\n unified_frame.py:585-649. 5 methods with identical multi-platform\n dispatch (PySpark \u2192 DataFusion \u2192 Polars fallback), differing only\n in the extraction function name (e.g., F.year vs F.month).\n\nG10 (58 dup lines): window_rank/window_row_number/window_dense_rank in\n polars_df.py:532-591. 3 methods with identical pattern: unpack\n order_by \u2192 create Polars expression \u2192 apply rank with different\n method parameter (\"min\", \"ordinal\", \"dense\") \u2192 handle partition.\n\nG16 (55 dup lines): max/mean/min aggregation methods in\n unified_frame.py:1018-1127. 6 copies with identical multi-platform\n dispatch for aggregations, differing only by the function name\n (F.sum, F.avg, F.min, F.max, etc.).\n\nG23 (52 dup lines): load_tables_from_data_source in\n expression_family.py:1230 and pandas_family.py:1037. Identical\n data loading logic (resolve source \u2192 iterate tables \u2192 normalize\n paths \u2192 load) differing only by context type signature.", "id": "dedup-wave2-dataframe-platform", "preserves": ["Aggregation methods dispatch correctly across PySpark, DataFusion, and Polars", "All temporal extraction methods return identical UnifiedExpr types", "Data loading produces identical table_stats and loaded data for both families", "Polars window functions produce identical rank semantics (min/ordinal/dense)", "Type signatures on public methods remain compatible for callers"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/expression_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/polars_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/shared_loading.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/unified_frame.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate DataFrame platform helpers: temporal extraction, window funcs, aggregation, data loading", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'dataframe or unified'", "description": "DataFrame platform tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-dataframe-platform", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k polars", "description": "Polars adapter tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-dataframe-platform", "last_result": null, "last_run": null, "seq": 2}, {"command": "make duplicate-check-verbose 2>&1 | grep -c 'day, hour, minute\\|window_dense_rank, window_rank\\|max, mean, min\\|load_tables_from_data'", "description": "Duplicate check shows reduction for these groups", "expected": "0 or reduced counts", "item_id": "dedup-wave2-dataframe-platform", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _extract_date_part helper in unified_frame.py, refactor 5 temporal methods", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _polars_window_rank helper in polars_df.py, refactor 3 window methods", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _apply_aggregation helper in unified_frame.py, refactor 6 aggregation methods", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract shared load_tables_from_data_source into shared_loading.py, update both families", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": ["w1", "w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run DataFrame platform tests: pytest -k 'dataframe or unified_frame or polars'", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave2-dataframe-platform", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w6"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change the _generate_table helper from the prior dedup", "instead": "(not specified)", "why": "the new work targets the SQL construction step above it"}, {"dont": "create a deep inheritance hierarchy for source formats", "instead": "(not specified)", "why": "a simple base method with a method_name parameter is sufficient"}, {"dont": "over-generalize the manifest writer", "instead": "(not specified)", "why": "it should accept the same parameters the current implementations use, not invent new abstractions"}], "approach": "1. G1 - TPC-DI dimension SQL pattern:\n Extract a `_build_dimension_insert_sql(table_name, num_records,\n column_specs, lookup_join_specs)` helper in sql.py that constructs\n the INSERT SELECT with CROSS JOINs. Each generate_* method reduces\n to: compute count, call _build_dimension_insert_sql, pass result\n to _generate_table.\n\n2. G3 - Manifest writing:\n Extract `write_generator_manifest(output_dir, benchmark_name,\n scale_factor, table_paths, compression_metadata, ...)` into\n benchbox/core/manifest_utils.py (new file). Each generator's\n _write_manifest becomes a one-liner calling this utility.\n\n3. G4 - Schema SQL collection:\n Extract `collect_create_table_sql(get_single_table_fn, table_order,\n dialect, enable_pks, enable_fks)` into benchbox/core/schema_utils.py\n (new file). Each benchmark's get_all_create_table_sql becomes a\n thin wrapper passing its table order and per-table function.\n\n4. G20 - TPC-DI source data generation:\n Extract a `_generate_source_data(self, gen_method_name, **kwargs)`\n base method in the SourceDataFormat ABC. Each subclass reduces to\n `return self._generate_source_data('_generate_oltp_data', **kwargs)`.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave2-ignore-rules"], "description": "Wave-2 duplicate-check triage (2026-03-12) identifies 4 consolidation-worthy\ngroups in the benchmark generator and schema layers. These are parallel\nimplementations that can be centralized into shared core utilities.\n\nTargets (report rank \u2192 dup lines):\n\nG1 (108 dup lines): TPC-DI dimension generators in core/tpcdi/generator/sql.py\n 4 methods (generate_company_dimension, generate_customer_dimension,\n generate_security_dimension, generate_account_dimension) at lines 266, 303,\n 332, 410. All share identical structure: compute scaled row count \u2192 build\n parameterized INSERT SELECT with CROSS JOINs \u2192 call _generate_table().\n Only column lists and join specs differ.\n NOTE: Prior TODO dedup-tpcdi-dimension-generator (completed 2026-02-21)\n already extracted _generate_table(). These 4 methods still share the\n inner SQL construction pattern that was NOT part of that extraction.\n\nG3 (60 dup lines): _write_manifest across 4 generators\n amplab/generator.py:390, clickbench/generator.py:383,\n h2odb/generator.py:299, joinorder/generator.py:635.\n All follow identical logic: check empty table_paths \u2192 create\n DataGenerationManifest \u2192 iterate table_paths adding entries \u2192 manifest.write().\n Only the benchmark_name string differs.\n\nG4 (60 dup lines): get_all_create_table_sql in 3 schema modules\n amplab/schema.py:107, read_primitives/schema.py:117, ssb/schema.py:175.\n All iterate a table_order list, call get_create_table_sql per table,\n collect and join SQL. Differs only in table ordering and the\n per-table SQL function called.\n\nG20 (52 dup lines): generate_data in 3 TPC-DI source format classes\n core/tpcdi/etl/sources.py:67, :128, :189 (CSVSourceFormat,\n XMLSourceFormat, FixedWidthSourceFormat). All extract scale_factor\n and output_dir from kwargs, init TPCDISourceDataGenerator, call\n format-specific _generate_*_data method.", "id": "dedup-wave2-generator-schema-utils", "preserves": ["DataGenerationManifest format and content identical after refactor", "Generated SQL for TPC-DI dimensions byte-for-byte identical", "TPC-DI source data generation produces same files and content", "get_all_create_table_sql output identical across all 3 schema modules"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/amplab/"}, {"kind": "only_modify", "path_glob": "benchbox/core/clickbench/"}, {"kind": "only_modify", "path_glob": "benchbox/core/h2odb/"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/"}, {"kind": "only_modify", "path_glob": "benchbox/core/manifest_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/read_primitives/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/schema_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate generator manifest writing, schema SQL helpers, and TPC-DI source generators", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k tpcdi", "description": "TPC-DI dimension generation tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k 'amplab or clickbench or h2odb or joinorder'", "description": "Generator manifest tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -v -k 'schema or create_table'", "description": "Schema SQL generation tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 3}, {"command": "make duplicate-check-verbose 2>&1 | head -5", "description": "Duplicate check shows reduction", "expected": "duplicated lines lower than 9326", "item_id": "dedup-wave2-generator-schema-utils", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _build_dimension_insert_sql helper and refactor 4 TPC-DI dimension generators", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run TPC-DI tests to verify dimension generation unchanged", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract write_generator_manifest utility into benchbox/core/manifest_utils.py", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor amplab, clickbench, h2odb, joinorder generators to use shared manifest writer", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run generator tests: pytest -k 'amplab or clickbench or h2odb or joinorder'", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract collect_create_table_sql into benchbox/core/schema_utils.py", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor amplab, read_primitives, ssb schema modules to use shared helper", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _generate_source_data base method in TPC-DI SourceDataFormat ABC", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave2-generator-schema-utils", "needs": ["w2", "w5", "w7", "w8"], "notes": "1442 tests pass. Duplicate lines: 8614\u21928464 (-150).", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w9"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": "Add entries to the existing ignore_rules list in pyproject.toml under\n[tool.benchbox.duplicates]. Follow the established pattern.\n\nStrategy:\n1. For G2: Add supports_operation, is_setup to ignore_rules scoped to\n primitives dataframe_operations modules and platform dataframe adapters.\n2. For G5: Add to_dict, get_etl_config, get_performance_config as global\n ignore_function_names (they're structural coincidences across unrelated classes).\n3. For G6, G19, G30: Add TPC-DS query function names scoped to\n tpcds/dataframe_queries/ paths.\n4. For G14: Already partially covered. Add compress_file/decompress_file/\n open_for_read scoped to core/dataframe/ and utils/compression.py.\n5. For G17: Extend existing protocol stubs rule to include window_dense_rank,\n window_rank, window_row_number for context.py.\n6. For G21: Add channel helper functions scoped to channel_helper.py.\n7. For G22: Add __init__ to existing facade rule (lines 801-809).\n8. For G25: Add both function names scoped to their specific files.\n9. For G26: Extend existing cast_date/cast_string path_contains with\n file_parsers.py and ddl_generator.py.\n10. For G27: Add get_stream_info/get_table_info/get_table_tuning scoped\n to facade files.\n11. For G29: Add get_row_count to existing dataframe adapter rule (line 847).\n\nAfter adding, run `make duplicate-check-verbose` and lower threshold.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": [], "description": "The `make duplicate-check-verbose` report (2026-03-12) shows 363 duplicate groups /\n9,326 duplicated lines (threshold 10,284). Wave-2 triage of the top-30 groups\nidentifies 12 groups that are intentional boilerplate, protocol conformance, or\nTPC-DS spec-mandated query patterns. These should be suppressed via ignore_rules\nto reduce noise and focus the checker on genuine consolidation targets.\n\nGroups to mark as ignored (by report rank):\n\nG2 (60 lines): supports_operation/is_setup/sql - protocol operation support\n methods across ai_primitives, metadata_primitives, transaction_primitives,\n write_primitives dataframe_operations modules and datafusion_df/pyspark_df.\n\nG5 (60 lines): to_dict/get_etl_config/get_performance_config - dataclass-to-dict\n conversion methods on distinct config/artifact classes. Acceptable boilerplate.\n\nG6 (60 lines): q62_expression_impl/q99_expression_impl - TPC-DS Q62 and Q99\n are spec-mandated structurally similar delivery-time bucketing queries\n against different channels (web vs catalog).\n\nG14 (55 lines): compress_file/decompress_file/open_for_read - protocol stubs\n in context.py and protocols.py matching canonical impls in compression.py.\n\nG17 (54 lines): window_dense_rank/window_rank/window_row_number -\n Protocol/ABC abstract method stubs in core/dataframe/context.py.\n\nG19 (54 lines): q38_expression_impl/q87_expression_impl - TPC-DS spec pair\n using INTERSECT/EXCEPT patterns with 3-channel sales data.\n\nG21 (52 lines): union_sales_channels_pandas/union_returns_channels_pandas -\n domain-specific channel union operations in tpcds channel_helper.py.\n\nG22 (52 lines): __init__ on facade classes (datavault.py, tpcds.py, tpch.py) -\n API surface conformance constructors.\n\nG25 (51 lines): create_sample_test_results/get_extended_table_order - AST\n false positive; different purposes but similar nested dict/list structure.\n\nG26 (50 lines): cast_date/cast_string/col - partially covered by existing rule;\n extend path_contains to include tpcdi/tools/file_parsers.py and\n core/tuning/ddl_generator.py.\n\nG27 (50 lines): get_stream_info/get_table_info/get_table_tuning - thin\n delegation methods on facade classes.\n\nG29 (50 lines): _detect_format/col/get_row_count - extend existing rule to\n add get_row_count for platform adapters; partially already covered.\n\nG30 (49 lines): q62_pandas_impl/q99_pandas_impl - pandas implementations\n of the same TPC-DS Q62/Q99 spec pair.\n\nAfter adding rules, rebaseline: threshold = ceil(actual * 1.10).", "id": "dedup-wave2-ignore-rules", "preserves": ["Existing ignore_rules entries remain intact", "Threshold only lowered, never raised beyond current value", "make duplicate-check still passes in CI"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Add ignore rules for 12 boilerplate duplicate groups from wave-2 triage", "verifications": [{"command": "make duplicate-check", "description": "Duplicate check passes at lowered threshold", "expected": "PASS", "item_id": "dedup-wave2-ignore-rules", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose 2>&1 | grep -c 'supports_operation\\|union_sales_channels\\|q62_expression'", "description": "Suppressed groups no longer appear in verbose output", "expected": "0", "item_id": "dedup-wave2-ignore-rules", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-wave2-ignore-rules", "needs": [], "notes": "Added 3 global ignore_function_names (G5: to_dict, get_etl_config, get_performance_config).\nExtended 4 existing rules (G17: window funcs in protocol stubs, G26: cast_date/cast_string\nwith context.py, G29: get_row_count + pandas_family.py, G22: facade __init__ for\ndatavault/tpcds/tpch). Added 6 new path-scoped rules (G2, G6/G19/G30, G14, G21, G25, G27).\nNote: G26 original approach assumed cast_date/cast_string at file_parsers.py and\nddl_generator.py, but those files contain parse_file and supports_tuning_type respectively\n(structural false positives - different abstract method stubs sharing same AST hash).\nFixed to target context.py where cast_date/cast_string actually live.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add all new ignore rules to pyproject.toml [tool.benchbox.duplicates]", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave2-ignore-rules", "needs": ["w1"], "notes": "Before: 366 groups, 519 instances, 9,371 duplicated lines.\nAfter: 349 groups, 476 instances, 8,614 duplicated lines.\nReduction: 17 groups, 43 instances, 757 lines.\nThreshold rebaselined: 10,284 \u2192 9,476 (ceil(8614 * 1.10)).\nAll targeted function names verified absent from report output.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run make duplicate-check-verbose, verify groups suppressed, rebaseline threshold", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave2-ignore-rules", "needs": ["w2"], "notes": "PASS: 8614 duplicated lines within threshold of 9476", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run make duplicate-check to confirm check still passes with new threshold", "wid": "w3"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "create a PostgreSQL base class that TimescaleDB inherits for ALL methods", "why": "only share the CLI argument helper, not the full adapter"}, {"INSTEAD": "(not specified)", "dont": "create an abstract base for all SQL platforms' execute_query", "why": "only consolidate the specific Doris/PostgreSQL pair that is byte-for-byte identical"}, {"INSTEAD": "(not specified)", "dont": "merge Presto and Trino into a single adapter", "why": "only share the catalog validation utility"}, {"INSTEAD": "(not specified)", "dont": "use *args/**kwargs in the returns generation helper", "why": "use explicit field_mapping and channel_fields parameters for type safety"}], "approach": "1. G8 - PostgreSQL/TimescaleDB CLI args:\n Extract `_add_postgres_compatible_arguments(parser, prefix)` as\n a static/module-level function in postgresql.py. TimescaleDB calls\n this with prefix='timescale' then adds its own arguments.\n Alternatively, TimescaleDB can inherit and call super().\n\n2. G9 - Presto/Trino catalog validation:\n Extract `_validate_catalog_exists(connection, catalog, platform_name)`\n as a standalone function in benchbox/platforms/presto_trino_utils.py.\n Both platforms import and call it with their platform name.\n\n3. G11 - Benchmark get_queries:\n Extract a `get_queries_with_translation(query_manager, dialect,\n translate_fn)` utility in benchbox/core/query_utils.py or as a\n mixin method. All 4 benchmarks call this helper.\n\n4. G12 - TPC-DS returns generation:\n Extract `_generate_returns_from_sale(sale_record, field_mapping,\n channel_specific_fields)` parameterized helper. Catalog and web\n variants become thin wrappers passing their column mappings.\n\n5. G13 - SQL execute_query:\n Extract `execute_sql_query(cursor, query, timeout, validator, ...)`\n into benchbox/platforms/base/sql_execution.py or as a mixin.\n Doris and PostgreSQL delegate to this shared implementation.\n\n6. G18 - DataFrame manager factories:\n Extract `_get_dataframe_manager(platform_name, manager_class,\n supported_platforms, spark_session=None)` into\n benchbox/core/dataframe_manager_factory.py. Each factory function\n becomes a one-liner calling this generic dispatcher.\n\n7. G24 - Operation/query lookup:\n Extract `get_entry_by_id(entries, entry_id, entry_type_name)` into\n benchbox/core/primitives_utils.py. All 4 lookup methods delegate.\n\n8. G28 - H2O.db aggregation queries:\n Extract `aggregation_groupby_impl(ctx, table_name, group_cols,\n agg_specs)` parameterized helper within read_primitives/\n dataframe_queries.py. Each variant passes its column names.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave2-ignore-rules"], "description": "Wave-2 duplicate-check triage (2026-03-12) identifies 8 consolidation-worthy\ngroups across SQL platform adapters and the primitives layer. These are\nparallel implementations where platforms with shared heritage or identical\ndispatch patterns can share a base implementation.\n\nTargets (report rank \u2192 dup lines):\n\nG8 (60 dup lines): add_cli_arguments in postgresql.py:67 and\n timescaledb.py:77. TimescaleDB is a PostgreSQL extension; both\n platforms share identical connection arguments (host, port, database,\n username, password, schema, work_mem, maintenance_work_mem).\n Only argument prefix (--postgres-* vs --timescale-*) and\n TimescaleDB-specific additions (chunk_interval, compression) differ.\n\nG9 (60 dup lines): _validate_catalog_exists in presto.py:401 and\n trino.py:515. Byte-for-byte identical except for error message\n strings (\"Presto\" vs \"Trino\"). Textbook mixin candidate.\n\nG11 (57 dup lines): get_queries in amplab/benchmark.py:103,\n clickbench/benchmark.py:79, h2odb/benchmark.py:103,\n ssb/benchmark.py:109. All follow identical pattern: fetch queries\n from query_manager \u2192 optionally translate dialect \u2192 return dict.\n\nG12 (57 dup lines): _generate_catalog_returns_from_sale and\n _generate_web_returns_from_sale in tpcds/maintenance_operations.py\n :985 and :1043. Identical structure (11 mapped fields + 3\n random/calculated), differing only by tuple field names and\n column prefixes (CR_* vs WR_*).\n\nG13 (57 dup lines): execute_query in doris.py:819 and\n postgresql.py:548. Virtually identical: execute via cursor \u2192\n measure time \u2192 validate row count \u2192 build result with\n _build_query_result_with_validation.\n\nG18 (54 dup lines): get_dataframe_metadata_manager,\n get_dataframe_transaction_manager, get_dataframe_write_manager\n in metadata_primitives/dataframe_operations.py:1775,\n transaction_primitives/dataframe_operations.py:1202,\n write_primitives/dataframe_operations.py:874. Identical factory\n dispatch: normalize platform_name \u2192 check df_platforms tuple \u2192\n instantiate manager class.\n\nG24 (51 dup lines): get_operation/get_query_entry in\n ai_primitives/queries.py:47, metadata_primitives/queries.py:46,\n transaction_primitives/operations.py:54,\n write_primitives/operations.py:54. Identical lookup pattern:\n try entries[id] \u2192 except KeyError \u2192 raise ValueError with\n available keys.\n\nG28 (50 dup lines): aggregation_distinct_groupby_pandas_impl,\n aggregation_groupby_large_pandas_impl,\n aggregation_groupby_small_pandas_impl in\n read_primitives/dataframe_queries.py (lines 1363-2661).\n 6 instances of identical groupby\u2192agg\u2192sort pattern varying only\n in column names and cardinality.", "id": "dedup-wave2-sql-platform-primitives", "preserves": ["Manager factory dispatch maps identical platform names to identical manager classes", "Presto/Trino catalog validation error messages contain platform-specific name", "TPC-DS returns generation: catalog returns use CR_* columns, web returns use WR_* columns", "TimescaleDB CLI args include all PostgreSQL-compatible params plus chunk_interval and compression", "execute_query return format identical for Doris and PostgreSQL (timing, validation, error dict)", "get_entry_by_id raises ValueError with sorted available keys in error message"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/doris.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto_trino_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/timescaledb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/trino.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate SQL platform adapters and primitives: PostgreSQL family, Presto/Trino, query execution, manager factories", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'postgres or timescale'", "description": "PostgreSQL and TimescaleDB platform tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k 'presto or trino'", "description": "Presto and Trino platform tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -v -k 'maintenance or tpcds'", "description": "TPC-DS maintenance operations tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/ -v -k 'primitive'", "description": "Primitives layer tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 4}, {"command": "make duplicate-check-verbose 2>&1 | head -5", "description": "Duplicate check shows reduction", "expected": "duplicated lines lower than previous", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _add_postgres_compatible_arguments helper, refactor TimescaleDB to use it", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": ["w3", "w4", "w6", "w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run primitives and benchmark tests: pytest -k 'primitive or amplab or clickbench or h2odb or ssb or tpcds'", "wid": "w10"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": ["w10", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w11"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _validate_catalog_exists into presto_trino_utils.py, update both platforms", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract get_queries_with_translation helper, refactor amplab/clickbench/h2odb/ssb", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _generate_returns_from_sale base method in tpcds/maintenance_operations.py", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract execute_sql_query shared implementation for Doris and PostgreSQL", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _get_dataframe_manager generic factory in core/dataframe_manager_factory.py", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract get_entry_by_id into core/primitives_utils.py, refactor 4 lookup methods", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract aggregation_groupby_impl helper in read_primitives/dataframe_queries.py", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": ["w1", "w2", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform-specific tests: pytest -k 'postgres or timescale or presto or trino or doris'", "wid": "w9"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "only extract internal dispatch logic", "dont": "change public method signatures", "why": "callers across 20+ platform adapters and benchmark modules depend on the current signatures; changing them silently breaks the contract without a test failure"}, {"INSTEAD": "only extract the shared scaffolding, keep metric computation untouched", "dont": "combine the ranking refactor with ranking algorithm changes", "why": "the ranking algorithms are validated against TPC official QphH formulas; mixing structural refactoring with logic changes makes regressions harder to isolate"}, {"INSTEAD": "use a plain shared function", "dont": "create abstract base classes for the registries", "why": "4 registries with a 14-line shared pattern doesn't justify the cognitive overhead of an ABC; contributors adding a new benchmark would need to understand the class hierarchy instead of copying a simple function call"}, {"INSTEAD": "only share the specific _table_exists and get_create_tables_sql helper functions", "dont": "merge transaction_primitives and write_primitives benchmark modules", "why": "they represent distinct TPC benchmark categories with independent evolution paths; a merge couples unrelated feature work"}, {"INSTEAD": "pass string literals like 'float64', not Python type objects", "dont": "use getattr-based dynamic dispatch for _apply_cast", "why": "explicit type strings preserve IDE navigability and static analysis across the 3 cast methods"}], "approach": "1. G4+G10 -- Primitives benchmark shared helpers:\n Extract `_table_exists(connection, table_name, schema_name)` and\n `_get_create_tables_sql(table_order, get_single_table_fn, ...)`\n into a shared module (e.g., benchbox/core/primitives_benchmark_utils.py\n or as methods on a shared base). Both transaction_primitives/benchmark.py\n and write_primitives/benchmark.py delegate to these shared implementations.\n\n2. G13 -- Ranking parameterization:\n Extract `_rank_by_metric(results, metric_fn, metric_name)` private method\n in ranking.py. Both _rank_by_geometric_mean and _rank_by_total_time become\n thin wrappers passing their metric function (scipy.stats.gmean vs sum).\n\n3. G14 -- TPC audit trail:\n Extract `generate_audit_trail(benchmark_name, results, metadata, ...)`\n into benchbox/core/tpc_audit.py or benchbox/core/tpc_patterns.py (which\n already exists for shared TPC patterns). Both official_benchmark modules\n delegate to this shared function.\n\n4. G15 -- Type casting dispatch:\n Add `_apply_cast(self, target_type: str)` private method to the relevant\n class in unified_frame.py, following the same pattern as the wave-2\n `_extract_date_part` helper. Each cast method (cast_float64, cast_int32,\n cast_int64) becomes: `return self._apply_cast(\"float64\")`.\n\n5. G17 -- TPC-DI loader parameterization:\n Extract `_load_data(data_type, path, validation_rules, ...)` base method\n in tpcdi/loader.py. Both load_dimension_data and load_fact_data become\n thin wrappers passing their data type and rules.\n\n6. G25 -- DataFusion window functions:\n Add `_datafusion_window_rank(self, rank_method: str, order_by, partition_by)`\n private method in datafusion_df.py, following the same pattern as the\n wave-2 `_polars_window_rank` helper. Each window function becomes:\n `return self._datafusion_window_rank(\"rank\", order_by, partition_by)`.\n\n7. G29 -- CLI text formatting:\n Extract `format_text_table(headers, rows, ...)` into\n benchbox/cli/commands/text_utils.py or benchbox/cli/formatting.py.\n Both calculate_qphh.py and metrics.py import and use the shared formatter.\n\n8. G30 -- Registry listing:\n Extract `list_benchmark_queries(module_path, query_prefix)` into\n a shared registry utility (e.g., benchbox/core/dataframe/registry_utils.py).\n All 4 benchmark registries delegate to this shared function.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave3-ignore-rules"], "description": "Wave-3 duplicate-check triage (2026-03-12, post wave-2) identifies 9\nconsolidation-worthy groups in the core library layer, CLI commands, and\nDataFrame platform adapters. These are parallel implementations that can\nbe centralized into shared helpers or parameterized methods.\n\nTargets (report rank -> dup lines):\n\nG4 (49 dup lines): _table_exists in transaction_primitives/benchmark.py:342\n and write_primitives/benchmark.py:399. 2 copies with identical logic:\n query information_schema.tables -> check row count -> return bool.\n Same function in two primitives benchmark modules.\n\nG10 (48 dup lines): get_create_tables_sql in transaction_primitives/benchmark.py:741\n and write_primitives/benchmark.py:898. 2 copies with identical pattern:\n iterate table_order -> call per-table SQL builder -> collect and join.\n Related to G4 - both are in the same two primitives benchmark files.\n\nG13 (46 dup lines): _rank_by_geometric_mean and _rank_by_total_time in\n analysis/ranking.py:168 and :215. 2 methods in the same file with\n identical ranking algorithm structure, differing only by the metric\n computation (geometric mean vs sum of times).\n\nG14 (46 dup lines): generate_audit_trail in tpcds/official_benchmark.py:252\n and tpch/official_benchmark.py:215. 2 copies with identical audit trail\n generation logic for TPC official benchmark compliance reports. Only\n the benchmark name and metadata fields differ.\n\nG15 (46 dup lines): cast_float64, cast_int32, cast_int64 in\n unified_frame.py:1164, :1212, :1236. 3 methods with identical\n multi-platform dispatch (PySpark -> DataFusion -> Polars), differing\n only by the target type. Same pattern as the temporal extraction\n methods already consolidated in wave-2.\n\nG17 (45 dup lines): load_dimension_data and load_fact_data in\n tpcdi/loader.py:48 and :94. 2 methods in the same file with identical\n data loading pattern (resolve path -> read files -> validate -> load),\n differing only by data type classification and validation rules.\n\nG25 (44 dup lines): window_dense_rank, window_rank, window_row_number in\n datafusion_df.py:752, :775, :798. 3 methods with identical DataFusion\n window function dispatch, differing only by the rank method parameter\n (rank/dense_rank/row_number). Same pattern as the Polars window\n functions consolidated in wave-2.\n\nG29 (42 dup lines): _format_text_output in cli/commands/calculate_qphh.py:90\n and cli/commands/metrics.py:232. 2 copies with identical text table\n formatting logic for CLI output.\n\nG30 (42 dup lines): list_clickbench_queries, list_nyctaxi_queries,\n list_ssb_queries, list_tpcds_queries across 4 benchmark dataframe\n query registries: clickbench/dataframe_queries/registry.py:27,\n nyctaxi/dataframe_queries/registry.py:27,\n ssb/dataframe_queries/registry.py:31,\n tpcds/dataframe_queries/registry.py:33. 4 copies with identical\n registry listing pattern.", "id": "dedup-wave3-core-and-dataframe", "preserves": ["Audit trail output format and content identical for TPC-DS and TPC-H", "CLI text output formatting identical in both commands", "Cast methods return identical UnifiedExpr types across all platforms", "DataFusion window functions produce identical rank semantics", "Ranking algorithms produce identical rank orderings and scores", "Registry query listings return identical dictionaries", "TPC-DI loader produces identical loaded data for dimension and fact tables", "_table_exists returns identical bool for all table existence checks", "get_create_tables_sql produces identical DDL strings"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/calculate_qphh.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/analysis/ranking.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/clickbench/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/primitives_benchmark_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpc_patterns.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/official_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/official_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/datafusion_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/unified_frame.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate core, CLI, and DataFrame duplicates: primitives, rankings, audit trails, casts, window funcs, text output, registries", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'primitive or transaction or write_prim'", "description": "Primitives benchmark tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k 'official or audit'", "description": "TPC official benchmark tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -v -k 'dataframe or unified_frame or datafusion'", "description": "DataFrame platform tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/ -v -k 'qphh or metrics'", "description": "CLI command tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 4}, {"command": "make duplicate-check-verbose 2>&1 | grep -c '_table_exists\\|_rank_by_geometric_mean\\|generate_audit_trail\\|cast_float64\\|_format_text_output\\|list_clickbench_queries'", "description": "Duplicate check shows reduction for targeted groups", "expected": "0", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Both functions are in transaction_primitives/benchmark.py and\nwrite_primitives/benchmark.py. Consider placing in core/ alongside\nthe existing primitives_utils.py from wave-2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract shared _table_exists and _get_create_tables_sql into primitives_benchmark_utils.py", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w10"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Single-file refactor. Both ranking methods reduce to one-liners\npassing different metric functions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _rank_by_metric parameterized helper in ranking.py", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Both tpcds and tpch official benchmarks use identical audit trail logic.\nBenchmark name and metadata fields passed as parameters.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract generate_audit_trail into shared TPC utility", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Follows the same pattern as wave-2's _extract_date_part.\nMulti-platform dispatch (PySpark -> DataFusion -> Polars).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _apply_cast helper in unified_frame.py, refactor 3 cast methods", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Single-file refactor. Both loading methods differ by data type and\nvalidation rules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _load_data base method in tpcdi/loader.py, refactor load_dimension/fact_data", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Follows the same pattern as wave-2's _polars_window_rank.\nRank method parameter differs (rank/dense_rank/row_number).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _datafusion_window_rank helper in datafusion_df.py, refactor 3 window methods", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "The deprecated calculate_qphh command already delegates output emission to\nmetrics.py. Removing the dead local helper eliminates the duplicate\nformatting implementation without changing runtime behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove duplicate QphH text formatter from calculate_qphh and keep metrics as the single formatter", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Each registry now exports the bound QueryRegistry.list_queries method\ninstead of maintaining an identical wrapper function.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Collapse 4 registry list wrappers to shared QueryRegistry.list_queries bindings", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": ["w1", "w2", "w3", "w4", "w5", "w6", "w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run targeted tests: pytest -k 'primitive or ranking or audit or cast or tpcdi or dataframe or cli or registry'", "wid": "w9"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "always scope with path_contains", "dont": "add overly broad function name ignores", "why": "broad ignores mask future real duplicates that should be consolidated, defeating the purpose of the checker"}, {"INSTEAD": "extend existing rules by adding function names or paths", "dont": "duplicate existing rules", "why": "multiple rules matching the same function create maintenance burden when paths change and make it unclear which rule is authoritative"}, {"INSTEAD": "check the companion TODOs before classifying a group as ignore", "dont": "ignore groups that are genuine consolidation candidates", "why": "the wave-3 consolidation TODOs (sql-platform-adapters, core-and-dataframe) target groups where extraction will reduce real maintenance cost"}], "approach": "Add entries to the existing ignore_rules list in pyproject.toml under\n[tool.benchbox.duplicates]. Follow the established pattern and wave-2 style.\n\nStrategy by group:\n\n1. G1 (tpcdi dimension generators): Add generate_company_dimension,\n generate_customer_dimension, generate_security_dimension scoped to\n benchbox/core/tpcdi/generator/sql.py.\n\n2. G2 (H2O.db aggregation queries): Add aggregation_distinct_groupby_expression_impl,\n aggregation_distinct_groupby_pandas_impl, aggregation_groupby_large_expression_impl,\n aggregation_groupby_large_pandas_impl scoped to\n benchbox/core/read_primitives/dataframe_queries.py.\n\n3. G3 (facade delegation false positive): Add classify_error scoped to\n tpcdi/etl/error_recovery.py. For date_add/date_sub, check if already\n covered by existing rules; extend path_contains if not.\n\n4. G5 (primitives catalog methods): Add get_operations_by_category,\n get_queries_by_category scoped to primitives queries.py and operations.py files.\n\n5. G6 (schema wrappers): Add get_all_create_table_sql scoped to\n amplab/schema.py, read_primitives/schema.py, ssb/schema.py.\n\n6. G7 (capability checks): Add supports_acl_introspection,\n supports_column_grants, supports_role_hierarchy scoped to\n metadata_primitives/ddl.py.\n\n7. G8 (facade methods): Extend existing facade rule to add generate_data,\n get_all_aliases, get_benchmark_info for the relevant facade files.\n\n8. G9 (TPC-DI query catalog): Extend existing wave-2 TPC-DI rule (line 1005)\n to add get_queries_by_category for query_analytics, query_etl, query_validation.\n\n9. G19 (manager factory wrappers): Add get_dataframe_metadata_manager,\n get_dataframe_transaction_manager, get_dataframe_write_manager scoped to\n primitives dataframe_operations.py files.\n\n10. G20 (domain validators): Add validate_customer_tier, validate_gender,\n validate_trade_type scoped to tpcdi/etl/validation.py.\n\n11. G21 (__init__ constructors): Extend existing __init__ ignore rule to\n add the relevant file paths.\n\n12. G22 (false positive): Add calculate_official_metrics, execute_operation\n scoped to the specific false-positive file paths. Check if partially\n covered by existing rules.\n\n13. G27 (TPC-DS spec pair): Extend the existing Q62/Q99/Q38/Q87 rule to\n add q26_expression_impl, q7_expression_impl.\n\nAfter adding, run `make duplicate-check-verbose` and lower threshold.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": [], "description": "The `make duplicate-check-verbose` report (2026-03-12, post wave-2) shows 344\nduplicate groups / 8,051 duplicated lines (threshold 9,476). Wave-3 triage of\nthe top-30 groups identifies 13 groups that are intentional boilerplate, wave-2\nextraction residuals, protocol conformance, TPC spec-mandated query patterns,\nor AST structural false positives. These should be suppressed via ignore_rules\nto reduce noise and focus the checker on genuine consolidation targets.\n\nGroups to mark as ignored (by current report rank):\n\nG1 (68 lines): generate_company_dimension / generate_customer_dimension /\n generate_security_dimension in tpcdi/generator/sql.py. Wave-2 residual:\n these are thin wrappers calling _build_dimension_sql with domain-specific\n column expressions. The 34-line structural match is the wrapper pattern,\n not duplicated logic.\n\nG2 (55 lines): aggregation_distinct_groupby_expression_impl /\n aggregation_distinct_groupby_pandas_impl /\n aggregation_groupby_large_expression_impl in\n read_primitives/dataframe_queries.py. H2O.db benchmark spec-driven query\n variants. Wave-2 extracted aggregation_groupby_impl for some pandas variants;\n the remaining 6 copies (expression + large) are spec-mandated groupby patterns\n varying only by column names and cardinality.\n\nG3 (55 lines): classify_error / date_add / date_sub across 6 files.\n AST false positive - these are different functions with different return\n types and semantics (error classification vs datetime arithmetic) that\n happen to share the facade/delegation AST shape.\n\nG5 (48 lines): get_operations_by_category / get_queries_by_category across\n 5 primitives modules (ai_primitives, metadata_primitives, read_primitives,\n transaction_primitives, write_primitives). Query catalog lookup methods\n with type variance (dict[str, str] vs dict[str, Operation]). The 12-line\n pattern is intentional per-type catalog API. Not worth a shared base class\n for the type generics overhead.\n\nG6 (48 lines): get_all_create_table_sql in amplab, read_primitives, ssb.\n Wave-2 residual: all three now delegate to schema_utils.collect_create_table_sql.\n The structural match is the wrapper (passing table_order + per-table fn).\n\nG7 (48 lines): supports_acl_introspection / supports_column_grants /\n supports_role_hierarchy in metadata_primitives/ddl.py. Intentional\n capability-matrix methods. Each defines a different platform set.\n Consolidating would require a capability-type string parameter that\n obscures the intent.\n\nG8 (48 lines): generate_data / get_all_aliases / get_benchmark_info across\n platform_registry.py, datavault.py, nyctaxi.py, tpch_skew.py, tsbs_devops.py.\n Facade API conformance methods - same pattern as __init__ facades already\n ignored in wave-2.\n\nG9 (48 lines): get_queries_by_category in tpcdi/query_analytics.py,\n tpcdi/query_etl.py, tpcdi/query_validation.py. Same catalog lookup pattern\n as G5. Extend existing wave-2 rule for TPC-DI query modules.\n\nG19 (44 lines): get_dataframe_metadata_manager / get_dataframe_transaction_manager /\n get_dataframe_write_manager across 3 primitives dataframe_operations modules.\n Wave-2 residual: these now delegate to _get_dataframe_manager in\n dataframe_manager_factory.py. Structural match is the wrapper pattern.\n\nG20 (44 lines): validate_customer_tier / validate_gender / validate_trade_type\n in tpcdi/etl/validation.py. Domain-specific validators that intentionally\n follow the same isin(valid_set) pattern with different column names and\n valid value sets.\n\nG21 (44 lines): __init__ across 5 unrelated classes (tpcdi/tools/data_cleaners.py,\n base/data_loading.py x2, unified_frame.py x2). Constructor boilerplate:\n docstring + attribute assignment. Different classes, different attributes.\n\nG22 (44 lines): calculate_official_metrics / execute_operation / get across\n 5 files. AST false positive - three different function names with different\n purposes that share the delegation/accessor AST shape.\n\nG27 (43 lines): q26_expression_impl / q7_expression_impl in\n tpcds/dataframe_queries/queries.py. TPC-DS spec pair: Q7 and Q26 both\n compute aggregated metrics from item-joined fact tables with promotional\n filtering. Same structural pattern, different dimension tables and measures.\n\nAfter adding rules, rebaseline: threshold = ceil(actual * 1.10).", "id": "dedup-wave3-ignore-rules", "preserves": ["Existing ignore_rules entries remain intact", "Threshold only lowered, never raised beyond current value", "make duplicate-check still passes in CI"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Add ignore rules for 13 boilerplate/false-positive duplicate groups from wave-3 triage", "verifications": [{"command": "make duplicate-check", "description": "Duplicate check passes at lowered threshold", "expected": "PASS", "item_id": "dedup-wave3-ignore-rules", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose 2>&1 | grep -c 'generate_company_dimension\\|supports_acl_introspection\\|validate_customer_tier\\|q26_expression'", "description": "Suppressed groups no longer appear in verbose output", "expected": "0", "item_id": "dedup-wave3-ignore-rules", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": [], "notes": "These are thin wrappers left over after wave-2 extractions. The underlying\nlogic was already consolidated; the structural match is the delegation pattern.\nG1: generate_*_dimension scoped to tpcdi/generator/sql.py.\nG6: get_all_create_table_sql scoped to amplab/schema.py, read_primitives/schema.py, ssb/schema.py.\nG19: get_dataframe_*_manager scoped to primitives dataframe_operations.py files.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for wave-2 residual wrappers: G1, G6, G19", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": [], "notes": "These are structurally similar AST hashes across unrelated functions.\nG3: classify_error scoped to tpcdi/etl/error_recovery.py; check date_add/date_sub\ncoverage in existing rules, extend path_contains if needed.\nG22: calculate_official_metrics, execute_operation scoped to false-positive paths;\ncheck overlap with existing rules first.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for AST false positives: G3, G22", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": [], "notes": "G2: H2O.db aggregation query impls scoped to read_primitives/dataframe_queries.py.\nG5: primitives catalog methods scoped to queries.py and operations.py files.\nG7: capability checks scoped to metadata_primitives/ddl.py.\nG8: extend existing facade rule with generate_data, get_all_aliases, get_benchmark_info.\nG9: extend existing TPC-DI rule (line 1005) with get_queries_by_category.\nG20: domain validators scoped to tpcdi/etl/validation.py.\nG21: extend existing __init__ rule with new file paths.\nG27: extend existing Q62/Q99/Q38/Q87 rule with q26/q7_expression_impl.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for intentional boilerplate patterns: G2, G5, G7, G8, G9, G20, G21, G27", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": ["w1", "w2", "w3"], "notes": "Verify each targeted function name no longer appears in the report.\nRebaseline threshold: threshold = ceil(actual * 1.10).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run make duplicate-check-verbose, verify groups suppressed, rebaseline threshold", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run make duplicate-check to confirm check still passes with new threshold", "wid": "w5"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "use standalone utility functions or thin mixins", "dont": "create a deep inheritance hierarchy for platform adapters", "why": "platform adapters are frequently added/modified by contributors unfamiliar with the class hierarchy; utility functions are easier to discover and compose than deep class chains"}, {"INSTEAD": "extract only _execute_statement and _wait_for_statement", "dont": "merge Fabric and Synapse Spark adapters", "why": "Fabric has workspace-scoped Lakehouse/Warehouse semantics that Synapse lacks; only the Livy execution layer is shared"}, {"INSTEAD": "only share specific utilities via presto_trino_utils.py", "dont": "merge Presto and Trino into a single adapter class", "why": "they diverge on features (Trino adds fault-tolerant execution, Presto has different auth) and a merged class would accumulate platform-specific branches"}, {"INSTEAD": "only consolidate the specific 3 platforms (firebolt, lakesail, presto) that are 97% identical", "dont": "over-generalize the tuning utility", "why": "other platforms (BigQuery, Snowflake, DuckDB) use radically different tuning mechanisms (clustering, micro-partitions, pragmas) and a universal abstraction would be leaky"}, {"INSTEAD": "keep explicit parameters matching the current signatures", "dont": "use *args/**kwargs for the config builder", "why": "explicit parameters preserve type safety and IDE navigability across the 2 consuming adapters"}], "approach": "1. G11 -- apply_table_tunings:\n Extract `_apply_table_tunings_base(adapter, table_tuning, connection,\n platform_name)` into a shared module (e.g., benchbox/platforms/base/tuning_utils.py\n or as a mixin). The function handles the 97% common logic; platforms pass\n their name for log messages. ImportError handling can use a try/except\n with platform-specific fallback via a parameter or override.\n\n2. G12 -- Azure config builders:\n Extract `_build_azure_compatible_config(host, port, database, ...)` into\n a shared utility (e.g., benchbox/platforms/azure/config_utils.py).\n Both LakeSail and Azure Synapse call the shared builder and apply\n platform-specific overrides after.\n\n3. G16 -- Presto/Trino create_schema:\n Add `create_schema(connection, catalog, schema_name, platform_name)` to\n the existing benchbox/platforms/presto_trino_utils.py module from wave-2.\n Both platforms delegate to this shared function.\n\n4. G18+G28 -- Azure Spark execution:\n Extract `_execute_statement(session, code, ...)` and\n `_wait_for_statement(session, statement_id, ...)` into\n benchbox/platforms/azure/spark_execution_utils.py. Both Fabric and\n Synapse Spark adapters import and call the shared implementations.\n\n5. G23 -- get_query_plan:\n Extract `get_sql_query_plan(connection, query)` into\n benchbox/platforms/base/sql_execution.py (which already has the\n wave-2 execute_sql_query utility, or create a new plan_utils module).\n All 5 platforms delegate to this utility.\n\n6. G24 -- Cloud URL parsing:\n Extract `parse_cloud_url(url)` returning (bucket, path) into\n benchbox/utils/cloud_urls.py or similar. Both ClickHouse Cloud and\n Firebolt import and use the shared parser. Consider handling s3://,\n gs://, and az:// patterns in one function.\n\n7. G26 -- Table name extraction:\n Extract `extract_table_name(sql_statement)` into\n benchbox/platforms/base/sql_utils.py. All 6 identical platforms\n (firebolt, lakesail, presto, redshift, spark, trino) delegate to\n this shared utility. The 4 divergent platforms (azure_synapse, athena,\n fabric_warehouse, questdb) retain their platform-specific implementations\n due to different regex patterns for bracket-quoting, EXTERNAL TABLE\n support, schema-qualified names, and quote-handling respectively.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-23T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave3-ignore-rules"], "description": "Wave-3 duplicate-check triage (2026-03-12, post wave-2) identifies 7\nconsolidation-worthy groups in the SQL platform adapter layer. These are\nparallel implementations across platform adapters that can share a base\nimplementation via utilities, mixins, or extraction to existing shared modules.\n\nTargets (report rank -> dup lines):\n\nG11 (48 dup lines): apply_table_tunings in firebolt.py:1227, lakesail.py:604,\n presto.py:1051. 3 copies with 97% identical logic: check has_any_tuning() ->\n get partitioning columns -> sort by order -> log. Only ImportError handling\n and log messages differ across platforms.\n\nG12 (47 dup lines): _build_lakesail_config and _build_synapse_config in\n azure_synapse.py:1341 and lakesail.py:686. Identical Azure-compatible\n config builder pattern. Both construct connection configs from the same\n parameter set with platform-specific defaults.\n\nG16 (46 dup lines): create_schema in presto.py:635 and trino.py:647.\n Byte-for-byte identical schema creation logic. Textbook extension of\n the existing presto_trino_utils.py from wave-2 (which already holds\n the shared _validate_catalog_exists utility).\n\nG18+G28 (88 dup lines combined): _execute_statement in\n fabric_spark_adapter.py:419 and synapse_spark_adapter.py:402, plus\n _wait_for_statement in fabric_spark_adapter.py:465 and\n synapse_spark_adapter.py:448. Two pairs of identical methods in the\n Azure Spark adapter family. Both adapters use the same Livy-based\n statement execution and polling pattern.\n\nG23 (44 dup lines): get_query_plan across 5 platforms: athena.py:1193,\n azure_synapse.py:1200, firebolt.py:1124, presto.py:958, trino.py:956.\n Verbatim identical: cursor -> EXPLAIN query -> fetchall -> join rows.\n No platform-specific differences whatsoever.\n\nG24 (44 dup lines): _parse_gcs_url and _parse_s3_url in\n clickhouse_cloud.py:880/:904 and firebolt.py:999. Cloud storage URL\n parsing utilities duplicated across platforms. These extract bucket\n and path from gs:// and s3:// URLs using the same regex/split logic.\n\nG26 (44 dup lines): _extract_table_name across 6 platforms:\n firebolt.py:1063, lakesail.py:504, presto.py:913, redshift.py:1692,\n spark.py:539, trino.py:913. Identical SQL table name extraction from\n CREATE TABLE with shared regex pattern (IF NOT EXISTS variant) and\n try/except/pass structure. 4 additional platforms (azure_synapse:1102,\n athena:1176, fabric_warehouse:1223, questdb:437) have platform-specific\n regex variants (bracket-quoting, EXTERNAL TABLE, schema-qualified names,\n quote-handling) and are excluded from this consolidation.", "id": "dedup-wave3-sql-platform-adapters", "preserves": ["Azure Spark statement execution and polling semantics unchanged", "Azure config builders produce identical connection parameters", "Cloud URL parsing returns identical (bucket, path) tuples", "_extract_table_name returns identical table name strings from DDL", "apply_table_tunings behavior identical per platform (including ImportError handling)", "create_schema DDL output byte-for-byte identical for Presto and Trino", "get_query_plan output format identical (newline-joined plan rows)"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/utils/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate SQL platform adapter duplicates: tuning, config, schema, Azure Spark, query plans, URL parsing, table extraction", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'firebolt or presto or trino or athena or azure or clickhouse or redshift or spark or lakesail'", "description": "Platform adapter tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-sql-platform-adapters", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose 2>&1 | grep -c 'apply_table_tunings\\|_build_lakesail_config\\|_wait_for_statement\\|_extract_table_name'", "description": "Duplicate check shows reduction for targeted groups", "expected": "0", "item_id": "dedup-wave3-sql-platform-adapters", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "97% identical across platforms. Only ImportError handling differs.\nUse a platform_name parameter for log messages.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _apply_table_tunings_base into shared module, refactor firebolt/lakesail/presto", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Both build connection configs from the same parameters.\nKeep platform-specific defaults as post-build overrides.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _build_azure_compatible_config utility, refactor lakesail and azure_synapse", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Extends the existing wave-2 shared module.\nByte-for-byte identical logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add create_schema to presto_trino_utils.py, refactor both platforms", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Two functions, both identical between Fabric and Synapse adapters.\nBoth use the same Livy session API.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract Azure Spark _execute_statement and _wait_for_statement into spark_execution_utils.py", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Verbatim identical across athena, azure_synapse, firebolt, presto, trino.\nStandard EXPLAIN -> fetchall -> join pattern.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract get_sql_query_plan utility, refactor 5 platform adapters", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "URL parsing for s3:// and gs:// schemes.\nConsider a unified parser supporting both.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract parse_cloud_url utility, refactor clickhouse_cloud and firebolt", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Same SQL parsing logic across firebolt, lakesail, presto, redshift, spark, trino.\nAll 6 use identical regex (IF NOT EXISTS variant) and try/except/pass structure.\n4 divergent platforms (azure_synapse, athena, fabric_warehouse, questdb) have\nplatform-specific regex and are excluded - different enough to not consolidate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract extract_table_name utility, refactor 6 platform adapters", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": ["w1", "w2", "w3", "w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform-specific tests: pytest -k 'firebolt or lakesail or presto or trino or azure or spark or athena or clickhouse or redshift'", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w9"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "create a PostgreSQL base class that TimescaleDB inherits for ALL methods", "instead": "(not specified)", "why": "only share the CLI argument helper, not the full adapter"}, {"dont": "create an abstract base for all SQL platforms' execute_query", "instead": "(not specified)", "why": "only consolidate the specific Doris/PostgreSQL pair that is byte-for-byte identical"}, {"dont": "merge Presto and Trino into a single adapter", "instead": "(not specified)", "why": "only share the catalog validation utility"}, {"dont": "use *args/**kwargs in the returns generation helper", "instead": "(not specified)", "why": "use explicit field_mapping and channel_fields parameters for type safety"}], "approach": "1. G8 - PostgreSQL/TimescaleDB CLI args:\n Extract `_add_postgres_compatible_arguments(parser, prefix)` as\n a static/module-level function in postgresql.py. TimescaleDB calls\n this with prefix='timescale' then adds its own arguments.\n Alternatively, TimescaleDB can inherit and call super().\n\n2. G9 - Presto/Trino catalog validation:\n Extract `_validate_catalog_exists(connection, catalog, platform_name)`\n as a standalone function in benchbox/platforms/presto_trino_utils.py.\n Both platforms import and call it with their platform name.\n\n3. G11 - Benchmark get_queries:\n Extract a `get_queries_with_translation(query_manager, dialect,\n translate_fn)` utility in benchbox/core/query_utils.py or as a\n mixin method. All 4 benchmarks call this helper.\n\n4. G12 - TPC-DS returns generation:\n Extract `_generate_returns_from_sale(sale_record, field_mapping,\n channel_specific_fields)` parameterized helper. Catalog and web\n variants become thin wrappers passing their column mappings.\n\n5. G13 - SQL execute_query:\n Extract `execute_sql_query(cursor, query, timeout, validator, ...)`\n into benchbox/platforms/base/sql_execution.py or as a mixin.\n Doris and PostgreSQL delegate to this shared implementation.\n\n6. G18 - DataFrame manager factories:\n Extract `_get_dataframe_manager(platform_name, manager_class,\n supported_platforms, spark_session=None)` into\n benchbox/core/dataframe_manager_factory.py. Each factory function\n becomes a one-liner calling this generic dispatcher.\n\n7. G24 - Operation/query lookup:\n Extract `get_entry_by_id(entries, entry_id, entry_type_name)` into\n benchbox/core/primitives_utils.py. All 4 lookup methods delegate.\n\n8. G28 - H2O.db aggregation queries:\n Extract `aggregation_groupby_impl(ctx, table_name, group_cols,\n agg_specs)` parameterized helper within read_primitives/\n dataframe_queries.py. Each variant passes its column names.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave2-ignore-rules"], "description": "Wave-2 duplicate-check triage (2026-03-12) identifies 8 consolidation-worthy\ngroups across SQL platform adapters and the primitives layer. These are\nparallel implementations where platforms with shared heritage or identical\ndispatch patterns can share a base implementation.\n\nTargets (report rank \u2192 dup lines):\n\nG8 (60 dup lines): add_cli_arguments in postgresql.py:67 and\n timescaledb.py:77. TimescaleDB is a PostgreSQL extension; both\n platforms share identical connection arguments (host, port, database,\n username, password, schema, work_mem, maintenance_work_mem).\n Only argument prefix (--postgres-* vs --timescale-*) and\n TimescaleDB-specific additions (chunk_interval, compression) differ.\n\nG9 (60 dup lines): _validate_catalog_exists in presto.py:401 and\n trino.py:515. Byte-for-byte identical except for error message\n strings (\"Presto\" vs \"Trino\"). Textbook mixin candidate.\n\nG11 (57 dup lines): get_queries in amplab/benchmark.py:103,\n clickbench/benchmark.py:79, h2odb/benchmark.py:103,\n ssb/benchmark.py:109. All follow identical pattern: fetch queries\n from query_manager \u2192 optionally translate dialect \u2192 return dict.\n\nG12 (57 dup lines): _generate_catalog_returns_from_sale and\n _generate_web_returns_from_sale in tpcds/maintenance_operations.py\n :985 and :1043. Identical structure (11 mapped fields + 3\n random/calculated), differing only by tuple field names and\n column prefixes (CR_* vs WR_*).\n\nG13 (57 dup lines): execute_query in doris.py:819 and\n postgresql.py:548. Virtually identical: execute via cursor \u2192\n measure time \u2192 validate row count \u2192 build result with\n _build_query_result_with_validation.\n\nG18 (54 dup lines): get_dataframe_metadata_manager,\n get_dataframe_transaction_manager, get_dataframe_write_manager\n in metadata_primitives/dataframe_operations.py:1775,\n transaction_primitives/dataframe_operations.py:1202,\n write_primitives/dataframe_operations.py:874. Identical factory\n dispatch: normalize platform_name \u2192 check df_platforms tuple \u2192\n instantiate manager class.\n\nG24 (51 dup lines): get_operation/get_query_entry in\n ai_primitives/queries.py:47, metadata_primitives/queries.py:46,\n transaction_primitives/operations.py:54,\n write_primitives/operations.py:54. Identical lookup pattern:\n try entries[id] \u2192 except KeyError \u2192 raise ValueError with\n available keys.\n\nG28 (50 dup lines): aggregation_distinct_groupby_pandas_impl,\n aggregation_groupby_large_pandas_impl,\n aggregation_groupby_small_pandas_impl in\n read_primitives/dataframe_queries.py (lines 1363-2661).\n 6 instances of identical groupby\u2192agg\u2192sort pattern varying only\n in column names and cardinality.", "id": "dedup-wave2-sql-platform-primitives", "preserves": ["Manager factory dispatch maps identical platform names to identical manager classes", "Presto/Trino catalog validation error messages contain platform-specific name", "TPC-DS returns generation: catalog returns use CR_* columns, web returns use WR_* columns", "TimescaleDB CLI args include all PostgreSQL-compatible params plus chunk_interval and compression", "execute_query return format identical for Doris and PostgreSQL (timing, validation, error dict)", "get_entry_by_id raises ValueError with sorted available keys in error message"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/doris.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto_trino_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/timescaledb.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/trino.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate SQL platform adapters and primitives: PostgreSQL family, Presto/Trino, query execution, manager factories", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'postgres or timescale'", "description": "PostgreSQL and TimescaleDB platform tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k 'presto or trino'", "description": "Presto and Trino platform tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -v -k 'maintenance or tpcds'", "description": "TPC-DS maintenance operations tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/ -v -k 'primitive'", "description": "Primitives layer tests pass", "expected": "all tests pass", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 4}, {"command": "make duplicate-check-verbose 2>&1 | head -5", "description": "Duplicate check shows reduction", "expected": "duplicated lines lower than previous", "item_id": "dedup-wave2-sql-platform-primitives", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _add_postgres_compatible_arguments helper, refactor TimescaleDB to use it", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": ["w3", "w4", "w6", "w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run primitives and benchmark tests: pytest -k 'primitive or amplab or clickbench or h2odb or ssb or tpcds'", "wid": "w10"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": ["w10", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w11"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _validate_catalog_exists into presto_trino_utils.py, update both platforms", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract get_queries_with_translation helper, refactor amplab/clickbench/h2odb/ssb", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _generate_returns_from_sale base method in tpcds/maintenance_operations.py", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract execute_sql_query shared implementation for Doris and PostgreSQL", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _get_dataframe_manager generic factory in core/dataframe_manager_factory.py", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract get_entry_by_id into core/primitives_utils.py, refactor 4 lookup methods", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract aggregation_groupby_impl helper in read_primitives/dataframe_queries.py", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave2-sql-platform-primitives", "needs": ["w1", "w2", "w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform-specific tests: pytest -k 'postgres or timescale or presto or trino or doris'", "wid": "w9"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change public method signatures", "instead": "only extract internal dispatch logic", "why": "callers across 20+ platform adapters and benchmark modules depend on the current signatures; changing them silently breaks the contract without a test failure"}, {"dont": "combine the ranking refactor with ranking algorithm changes", "instead": "only extract the shared scaffolding, keep metric computation untouched", "why": "the ranking algorithms are validated against TPC official QphH formulas; mixing structural refactoring with logic changes makes regressions harder to isolate"}, {"dont": "create abstract base classes for the registries", "instead": "use a plain shared function", "why": "4 registries with a 14-line shared pattern doesn't justify the cognitive overhead of an ABC; contributors adding a new benchmark would need to understand the class hierarchy instead of copying a simple function call"}, {"dont": "merge transaction_primitives and write_primitives benchmark modules", "instead": "only share the specific _table_exists and get_create_tables_sql helper functions", "why": "they represent distinct TPC benchmark categories with independent evolution paths; a merge couples unrelated feature work"}, {"dont": "use getattr-based dynamic dispatch for _apply_cast", "instead": "pass string literals like 'float64', not Python type objects", "why": "explicit type strings preserve IDE navigability and static analysis across the 3 cast methods"}], "approach": "1. G4+G10 -- Primitives benchmark shared helpers:\n Extract `_table_exists(connection, table_name, schema_name)` and\n `_get_create_tables_sql(table_order, get_single_table_fn, ...)`\n into a shared module (e.g., benchbox/core/primitives_benchmark_utils.py\n or as methods on a shared base). Both transaction_primitives/benchmark.py\n and write_primitives/benchmark.py delegate to these shared implementations.\n\n2. G13 -- Ranking parameterization:\n Extract `_rank_by_metric(results, metric_fn, metric_name)` private method\n in ranking.py. Both _rank_by_geometric_mean and _rank_by_total_time become\n thin wrappers passing their metric function (scipy.stats.gmean vs sum).\n\n3. G14 -- TPC audit trail:\n Extract `generate_audit_trail(benchmark_name, results, metadata, ...)`\n into benchbox/core/tpc_audit.py or benchbox/core/tpc_patterns.py (which\n already exists for shared TPC patterns). Both official_benchmark modules\n delegate to this shared function.\n\n4. G15 -- Type casting dispatch:\n Add `_apply_cast(self, target_type: str)` private method to the relevant\n class in unified_frame.py, following the same pattern as the wave-2\n `_extract_date_part` helper. Each cast method (cast_float64, cast_int32,\n cast_int64) becomes: `return self._apply_cast(\"float64\")`.\n\n5. G17 -- TPC-DI loader parameterization:\n Extract `_load_data(data_type, path, validation_rules, ...)` base method\n in tpcdi/loader.py. Both load_dimension_data and load_fact_data become\n thin wrappers passing their data type and rules.\n\n6. G25 -- DataFusion window functions:\n Add `_datafusion_window_rank(self, rank_method: str, order_by, partition_by)`\n private method in datafusion_df.py, following the same pattern as the\n wave-2 `_polars_window_rank` helper. Each window function becomes:\n `return self._datafusion_window_rank(\"rank\", order_by, partition_by)`.\n\n7. G29 -- CLI text formatting:\n Extract `format_text_table(headers, rows, ...)` into\n benchbox/cli/commands/text_utils.py or benchbox/cli/formatting.py.\n Both calculate_qphh.py and metrics.py import and use the shared formatter.\n\n8. G30 -- Registry listing:\n Extract `list_benchmark_queries(module_path, query_prefix)` into\n a shared registry utility (e.g., benchbox/core/dataframe/registry_utils.py).\n All 4 benchmark registries delegate to this shared function.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave3-ignore-rules"], "description": "Wave-3 duplicate-check triage (2026-03-12, post wave-2) identifies 9\nconsolidation-worthy groups in the core library layer, CLI commands, and\nDataFrame platform adapters. These are parallel implementations that can\nbe centralized into shared helpers or parameterized methods.\n\nTargets (report rank -> dup lines):\n\nG4 (49 dup lines): _table_exists in transaction_primitives/benchmark.py:342\n and write_primitives/benchmark.py:399. 2 copies with identical logic:\n query information_schema.tables -> check row count -> return bool.\n Same function in two primitives benchmark modules.\n\nG10 (48 dup lines): get_create_tables_sql in transaction_primitives/benchmark.py:741\n and write_primitives/benchmark.py:898. 2 copies with identical pattern:\n iterate table_order -> call per-table SQL builder -> collect and join.\n Related to G4 - both are in the same two primitives benchmark files.\n\nG13 (46 dup lines): _rank_by_geometric_mean and _rank_by_total_time in\n analysis/ranking.py:168 and :215. 2 methods in the same file with\n identical ranking algorithm structure, differing only by the metric\n computation (geometric mean vs sum of times).\n\nG14 (46 dup lines): generate_audit_trail in tpcds/official_benchmark.py:252\n and tpch/official_benchmark.py:215. 2 copies with identical audit trail\n generation logic for TPC official benchmark compliance reports. Only\n the benchmark name and metadata fields differ.\n\nG15 (46 dup lines): cast_float64, cast_int32, cast_int64 in\n unified_frame.py:1164, :1212, :1236. 3 methods with identical\n multi-platform dispatch (PySpark -> DataFusion -> Polars), differing\n only by the target type. Same pattern as the temporal extraction\n methods already consolidated in wave-2.\n\nG17 (45 dup lines): load_dimension_data and load_fact_data in\n tpcdi/loader.py:48 and :94. 2 methods in the same file with identical\n data loading pattern (resolve path -> read files -> validate -> load),\n differing only by data type classification and validation rules.\n\nG25 (44 dup lines): window_dense_rank, window_rank, window_row_number in\n datafusion_df.py:752, :775, :798. 3 methods with identical DataFusion\n window function dispatch, differing only by the rank method parameter\n (rank/dense_rank/row_number). Same pattern as the Polars window\n functions consolidated in wave-2.\n\nG29 (42 dup lines): _format_text_output in cli/commands/calculate_qphh.py:90\n and cli/commands/metrics.py:232. 2 copies with identical text table\n formatting logic for CLI output.\n\nG30 (42 dup lines): list_clickbench_queries, list_nyctaxi_queries,\n list_ssb_queries, list_tpcds_queries across 4 benchmark dataframe\n query registries: clickbench/dataframe_queries/registry.py:27,\n nyctaxi/dataframe_queries/registry.py:27,\n ssb/dataframe_queries/registry.py:31,\n tpcds/dataframe_queries/registry.py:33. 4 copies with identical\n registry listing pattern.", "id": "dedup-wave3-core-and-dataframe", "preserves": ["Audit trail output format and content identical for TPC-DS and TPC-H", "CLI text output formatting identical in both commands", "Cast methods return identical UnifiedExpr types across all platforms", "DataFusion window functions produce identical rank semantics", "Ranking algorithms produce identical rank orderings and scores", "Registry query listings return identical dictionaries", "TPC-DI loader produces identical loaded data for dimension and fact tables", "_table_exists returns identical bool for all table existence checks", "get_create_tables_sql produces identical DDL strings"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/calculate_qphh.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/analysis/ranking.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/clickbench/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/nyctaxi/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/primitives_benchmark_utils.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpc_patterns.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/dataframe_queries/registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/official_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/official_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/datafusion_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/unified_frame.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate core, CLI, and DataFrame duplicates: primitives, rankings, audit trails, casts, window funcs, text output, registries", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'primitive or transaction or write_prim'", "description": "Primitives benchmark tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -v -k 'official or audit'", "description": "TPC official benchmark tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/ -v -k 'dataframe or unified_frame or datafusion'", "description": "DataFrame platform tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/ -v -k 'qphh or metrics'", "description": "CLI command tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 4}, {"command": "make duplicate-check-verbose 2>&1 | grep -c '_table_exists\\|_rank_by_geometric_mean\\|generate_audit_trail\\|cast_float64\\|_format_text_output\\|list_clickbench_queries'", "description": "Duplicate check shows reduction for targeted groups", "expected": "0", "item_id": "dedup-wave3-core-and-dataframe", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Both functions are in transaction_primitives/benchmark.py and\nwrite_primitives/benchmark.py. Consider placing in core/ alongside\nthe existing primitives_utils.py from wave-2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract shared _table_exists and _get_create_tables_sql into primitives_benchmark_utils.py", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": ["w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w10"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Single-file refactor. Both ranking methods reduce to one-liners\npassing different metric functions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _rank_by_metric parameterized helper in ranking.py", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Both tpcds and tpch official benchmarks use identical audit trail logic.\nBenchmark name and metadata fields passed as parameters.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract generate_audit_trail into shared TPC utility", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Follows the same pattern as wave-2's _extract_date_part.\nMulti-platform dispatch (PySpark -> DataFusion -> Polars).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _apply_cast helper in unified_frame.py, refactor 3 cast methods", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Single-file refactor. Both loading methods differ by data type and\nvalidation rules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _load_data base method in tpcdi/loader.py, refactor load_dimension/fact_data", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Follows the same pattern as wave-2's _polars_window_rank.\nRank method parameter differs (rank/dense_rank/row_number).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _datafusion_window_rank helper in datafusion_df.py, refactor 3 window methods", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "The deprecated calculate_qphh command already delegates output emission to\nmetrics.py. Removing the dead local helper eliminates the duplicate\nformatting implementation without changing runtime behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove duplicate QphH text formatter from calculate_qphh and keep metrics as the single formatter", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": [], "notes": "Each registry now exports the bound QueryRegistry.list_queries method\ninstead of maintaining an identical wrapper function.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Collapse 4 registry list wrappers to shared QueryRegistry.list_queries bindings", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave3-core-and-dataframe", "needs": ["w1", "w2", "w3", "w4", "w5", "w6", "w7", "w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run targeted tests: pytest -k 'primitive or ranking or audit or cast or tpcdi or dataframe or cli or registry'", "wid": "w9"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "add overly broad function name ignores", "instead": "always scope with path_contains", "why": "broad ignores mask future real duplicates that should be consolidated, defeating the purpose of the checker"}, {"dont": "duplicate existing rules", "instead": "extend existing rules by adding function names or paths", "why": "multiple rules matching the same function create maintenance burden when paths change and make it unclear which rule is authoritative"}, {"dont": "ignore groups that are genuine consolidation candidates", "instead": "check the companion TODOs before classifying a group as ignore", "why": "the wave-3 consolidation TODOs (sql-platform-adapters, core-and-dataframe) target groups where extraction will reduce real maintenance cost"}], "approach": "Add entries to the existing ignore_rules list in pyproject.toml under\n[tool.benchbox.duplicates]. Follow the established pattern and wave-2 style.\n\nStrategy by group:\n\n1. G1 (tpcdi dimension generators): Add generate_company_dimension,\n generate_customer_dimension, generate_security_dimension scoped to\n benchbox/core/tpcdi/generator/sql.py.\n\n2. G2 (H2O.db aggregation queries): Add aggregation_distinct_groupby_expression_impl,\n aggregation_distinct_groupby_pandas_impl, aggregation_groupby_large_expression_impl,\n aggregation_groupby_large_pandas_impl scoped to\n benchbox/core/read_primitives/dataframe_queries.py.\n\n3. G3 (facade delegation false positive): Add classify_error scoped to\n tpcdi/etl/error_recovery.py. For date_add/date_sub, check if already\n covered by existing rules; extend path_contains if not.\n\n4. G5 (primitives catalog methods): Add get_operations_by_category,\n get_queries_by_category scoped to primitives queries.py and operations.py files.\n\n5. G6 (schema wrappers): Add get_all_create_table_sql scoped to\n amplab/schema.py, read_primitives/schema.py, ssb/schema.py.\n\n6. G7 (capability checks): Add supports_acl_introspection,\n supports_column_grants, supports_role_hierarchy scoped to\n metadata_primitives/ddl.py.\n\n7. G8 (facade methods): Extend existing facade rule to add generate_data,\n get_all_aliases, get_benchmark_info for the relevant facade files.\n\n8. G9 (TPC-DI query catalog): Extend existing wave-2 TPC-DI rule (line 1005)\n to add get_queries_by_category for query_analytics, query_etl, query_validation.\n\n9. G19 (manager factory wrappers): Add get_dataframe_metadata_manager,\n get_dataframe_transaction_manager, get_dataframe_write_manager scoped to\n primitives dataframe_operations.py files.\n\n10. G20 (domain validators): Add validate_customer_tier, validate_gender,\n validate_trade_type scoped to tpcdi/etl/validation.py.\n\n11. G21 (__init__ constructors): Extend existing __init__ ignore rule to\n add the relevant file paths.\n\n12. G22 (false positive): Add calculate_official_metrics, execute_operation\n scoped to the specific false-positive file paths. Check if partially\n covered by existing rules.\n\n13. G27 (TPC-DS spec pair): Extend the existing Q62/Q99/Q38/Q87 rule to\n add q26_expression_impl, q7_expression_impl.\n\nAfter adding, run `make duplicate-check-verbose` and lower threshold.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-12T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": [], "description": "The `make duplicate-check-verbose` report (2026-03-12, post wave-2) shows 344\nduplicate groups / 8,051 duplicated lines (threshold 9,476). Wave-3 triage of\nthe top-30 groups identifies 13 groups that are intentional boilerplate, wave-2\nextraction residuals, protocol conformance, TPC spec-mandated query patterns,\nor AST structural false positives. These should be suppressed via ignore_rules\nto reduce noise and focus the checker on genuine consolidation targets.\n\nGroups to mark as ignored (by current report rank):\n\nG1 (68 lines): generate_company_dimension / generate_customer_dimension /\n generate_security_dimension in tpcdi/generator/sql.py. Wave-2 residual:\n these are thin wrappers calling _build_dimension_sql with domain-specific\n column expressions. The 34-line structural match is the wrapper pattern,\n not duplicated logic.\n\nG2 (55 lines): aggregation_distinct_groupby_expression_impl /\n aggregation_distinct_groupby_pandas_impl /\n aggregation_groupby_large_expression_impl in\n read_primitives/dataframe_queries.py. H2O.db benchmark spec-driven query\n variants. Wave-2 extracted aggregation_groupby_impl for some pandas variants;\n the remaining 6 copies (expression + large) are spec-mandated groupby patterns\n varying only by column names and cardinality.\n\nG3 (55 lines): classify_error / date_add / date_sub across 6 files.\n AST false positive - these are different functions with different return\n types and semantics (error classification vs datetime arithmetic) that\n happen to share the facade/delegation AST shape.\n\nG5 (48 lines): get_operations_by_category / get_queries_by_category across\n 5 primitives modules (ai_primitives, metadata_primitives, read_primitives,\n transaction_primitives, write_primitives). Query catalog lookup methods\n with type variance (dict[str, str] vs dict[str, Operation]). The 12-line\n pattern is intentional per-type catalog API. Not worth a shared base class\n for the type generics overhead.\n\nG6 (48 lines): get_all_create_table_sql in amplab, read_primitives, ssb.\n Wave-2 residual: all three now delegate to schema_utils.collect_create_table_sql.\n The structural match is the wrapper (passing table_order + per-table fn).\n\nG7 (48 lines): supports_acl_introspection / supports_column_grants /\n supports_role_hierarchy in metadata_primitives/ddl.py. Intentional\n capability-matrix methods. Each defines a different platform set.\n Consolidating would require a capability-type string parameter that\n obscures the intent.\n\nG8 (48 lines): generate_data / get_all_aliases / get_benchmark_info across\n platform_registry.py, datavault.py, nyctaxi.py, tpch_skew.py, tsbs_devops.py.\n Facade API conformance methods - same pattern as __init__ facades already\n ignored in wave-2.\n\nG9 (48 lines): get_queries_by_category in tpcdi/query_analytics.py,\n tpcdi/query_etl.py, tpcdi/query_validation.py. Same catalog lookup pattern\n as G5. Extend existing wave-2 rule for TPC-DI query modules.\n\nG19 (44 lines): get_dataframe_metadata_manager / get_dataframe_transaction_manager /\n get_dataframe_write_manager across 3 primitives dataframe_operations modules.\n Wave-2 residual: these now delegate to _get_dataframe_manager in\n dataframe_manager_factory.py. Structural match is the wrapper pattern.\n\nG20 (44 lines): validate_customer_tier / validate_gender / validate_trade_type\n in tpcdi/etl/validation.py. Domain-specific validators that intentionally\n follow the same isin(valid_set) pattern with different column names and\n valid value sets.\n\nG21 (44 lines): __init__ across 5 unrelated classes (tpcdi/tools/data_cleaners.py,\n base/data_loading.py x2, unified_frame.py x2). Constructor boilerplate:\n docstring + attribute assignment. Different classes, different attributes.\n\nG22 (44 lines): calculate_official_metrics / execute_operation / get across\n 5 files. AST false positive - three different function names with different\n purposes that share the delegation/accessor AST shape.\n\nG27 (43 lines): q26_expression_impl / q7_expression_impl in\n tpcds/dataframe_queries/queries.py. TPC-DS spec pair: Q7 and Q26 both\n compute aggregated metrics from item-joined fact tables with promotional\n filtering. Same structural pattern, different dimension tables and measures.\n\nAfter adding rules, rebaseline: threshold = ceil(actual * 1.10).", "id": "dedup-wave3-ignore-rules", "preserves": ["Existing ignore_rules entries remain intact", "Threshold only lowered, never raised beyond current value", "make duplicate-check still passes in CI"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "pyproject.toml"}], "state": "done", "title": "Add ignore rules for 13 boilerplate/false-positive duplicate groups from wave-3 triage", "verifications": [{"command": "make duplicate-check", "description": "Duplicate check passes at lowered threshold", "expected": "PASS", "item_id": "dedup-wave3-ignore-rules", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose 2>&1 | grep -c 'generate_company_dimension\\|supports_acl_introspection\\|validate_customer_tier\\|q26_expression'", "description": "Suppressed groups no longer appear in verbose output", "expected": "0", "item_id": "dedup-wave3-ignore-rules", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": [], "notes": "These are thin wrappers left over after wave-2 extractions. The underlying\nlogic was already consolidated; the structural match is the delegation pattern.\nG1: generate_*_dimension scoped to tpcdi/generator/sql.py.\nG6: get_all_create_table_sql scoped to amplab/schema.py, read_primitives/schema.py, ssb/schema.py.\nG19: get_dataframe_*_manager scoped to primitives dataframe_operations.py files.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for wave-2 residual wrappers: G1, G6, G19", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": [], "notes": "These are structurally similar AST hashes across unrelated functions.\nG3: classify_error scoped to tpcdi/etl/error_recovery.py; check date_add/date_sub\ncoverage in existing rules, extend path_contains if needed.\nG22: calculate_official_metrics, execute_operation scoped to false-positive paths;\ncheck overlap with existing rules first.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for AST false positives: G3, G22", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": [], "notes": "G2: H2O.db aggregation query impls scoped to read_primitives/dataframe_queries.py.\nG5: primitives catalog methods scoped to queries.py and operations.py files.\nG7: capability checks scoped to metadata_primitives/ddl.py.\nG8: extend existing facade rule with generate_data, get_all_aliases, get_benchmark_info.\nG9: extend existing TPC-DI rule (line 1005) with get_queries_by_category.\nG20: domain validators scoped to tpcdi/etl/validation.py.\nG21: extend existing __init__ rule with new file paths.\nG27: extend existing Q62/Q99/Q38/Q87 rule with q26/q7_expression_impl.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ignore rules for intentional boilerplate patterns: G2, G5, G7, G8, G9, G20, G21, G27", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": ["w1", "w2", "w3"], "notes": "Verify each targeted function name no longer appears in the report.\nRebaseline threshold: threshold = ceil(actual * 1.10).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run make duplicate-check-verbose, verify groups suppressed, rebaseline threshold", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave3-ignore-rules", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run make duplicate-check to confirm check still passes with new threshold", "wid": "w5"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "create a deep inheritance hierarchy for platform adapters", "instead": "use standalone utility functions or thin mixins", "why": "platform adapters are frequently added/modified by contributors unfamiliar with the class hierarchy; utility functions are easier to discover and compose than deep class chains"}, {"dont": "merge Fabric and Synapse Spark adapters", "instead": "extract only _execute_statement and _wait_for_statement", "why": "Fabric has workspace-scoped Lakehouse/Warehouse semantics that Synapse lacks; only the Livy execution layer is shared"}, {"dont": "merge Presto and Trino into a single adapter class", "instead": "only share specific utilities via presto_trino_utils.py", "why": "they diverge on features (Trino adds fault-tolerant execution, Presto has different auth) and a merged class would accumulate platform-specific branches"}, {"dont": "over-generalize the tuning utility", "instead": "only consolidate the specific 3 platforms (firebolt, lakesail, presto) that are 97% identical", "why": "other platforms (BigQuery, Snowflake, DuckDB) use radically different tuning mechanisms (clustering, micro-partitions, pragmas) and a universal abstraction would be leaky"}, {"dont": "use *args/**kwargs for the config builder", "instead": "keep explicit parameters matching the current signatures", "why": "explicit parameters preserve type safety and IDE navigability across the 2 consuming adapters"}], "approach": "1. G11 -- apply_table_tunings:\n Extract `_apply_table_tunings_base(adapter, table_tuning, connection,\n platform_name)` into a shared module (e.g., benchbox/platforms/base/tuning_utils.py\n or as a mixin). The function handles the 97% common logic; platforms pass\n their name for log messages. ImportError handling can use a try/except\n with platform-specific fallback via a parameter or override.\n\n2. G12 -- Azure config builders:\n Extract `_build_azure_compatible_config(host, port, database, ...)` into\n a shared utility (e.g., benchbox/platforms/azure/config_utils.py).\n Both LakeSail and Azure Synapse call the shared builder and apply\n platform-specific overrides after.\n\n3. G16 -- Presto/Trino create_schema:\n Add `create_schema(connection, catalog, schema_name, platform_name)` to\n the existing benchbox/platforms/presto_trino_utils.py module from wave-2.\n Both platforms delegate to this shared function.\n\n4. G18+G28 -- Azure Spark execution:\n Extract `_execute_statement(session, code, ...)` and\n `_wait_for_statement(session, statement_id, ...)` into\n benchbox/platforms/azure/spark_execution_utils.py. Both Fabric and\n Synapse Spark adapters import and call the shared implementations.\n\n5. G23 -- get_query_plan:\n Extract `get_sql_query_plan(connection, query)` into\n benchbox/platforms/base/sql_execution.py (which already has the\n wave-2 execute_sql_query utility, or create a new plan_utils module).\n All 5 platforms delegate to this utility.\n\n6. G24 -- Cloud URL parsing:\n Extract `parse_cloud_url(url)` returning (bucket, path) into\n benchbox/utils/cloud_urls.py or similar. Both ClickHouse Cloud and\n Firebolt import and use the shared parser. Consider handling s3://,\n gs://, and az:// patterns in one function.\n\n7. G26 -- Table name extraction:\n Extract `extract_table_name(sql_statement)` into\n benchbox/platforms/base/sql_utils.py. All 6 identical platforms\n (firebolt, lakesail, presto, redshift, spark, trino) delegate to\n this shared utility. The 4 divergent platforms (azure_synapse, athena,\n fabric_warehouse, questdb) retain their platform-specific implementations\n due to different regex patterns for bracket-quoting, EXTERNAL TABLE\n support, schema-qualified names, and quote-handling respectively.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-23T00:00:00Z", "completed_pr": null, "created_at": "2026-03-12T00:00:00Z", "deferrals": [], "deps": ["dedup-wave3-ignore-rules"], "description": "Wave-3 duplicate-check triage (2026-03-12, post wave-2) identifies 7\nconsolidation-worthy groups in the SQL platform adapter layer. These are\nparallel implementations across platform adapters that can share a base\nimplementation via utilities, mixins, or extraction to existing shared modules.\n\nTargets (report rank -> dup lines):\n\nG11 (48 dup lines): apply_table_tunings in firebolt.py:1227, lakesail.py:604,\n presto.py:1051. 3 copies with 97% identical logic: check has_any_tuning() ->\n get partitioning columns -> sort by order -> log. Only ImportError handling\n and log messages differ across platforms.\n\nG12 (47 dup lines): _build_lakesail_config and _build_synapse_config in\n azure_synapse.py:1341 and lakesail.py:686. Identical Azure-compatible\n config builder pattern. Both construct connection configs from the same\n parameter set with platform-specific defaults.\n\nG16 (46 dup lines): create_schema in presto.py:635 and trino.py:647.\n Byte-for-byte identical schema creation logic. Textbook extension of\n the existing presto_trino_utils.py from wave-2 (which already holds\n the shared _validate_catalog_exists utility).\n\nG18+G28 (88 dup lines combined): _execute_statement in\n fabric_spark_adapter.py:419 and synapse_spark_adapter.py:402, plus\n _wait_for_statement in fabric_spark_adapter.py:465 and\n synapse_spark_adapter.py:448. Two pairs of identical methods in the\n Azure Spark adapter family. Both adapters use the same Livy-based\n statement execution and polling pattern.\n\nG23 (44 dup lines): get_query_plan across 5 platforms: athena.py:1193,\n azure_synapse.py:1200, firebolt.py:1124, presto.py:958, trino.py:956.\n Verbatim identical: cursor -> EXPLAIN query -> fetchall -> join rows.\n No platform-specific differences whatsoever.\n\nG24 (44 dup lines): _parse_gcs_url and _parse_s3_url in\n clickhouse_cloud.py:880/:904 and firebolt.py:999. Cloud storage URL\n parsing utilities duplicated across platforms. These extract bucket\n and path from gs:// and s3:// URLs using the same regex/split logic.\n\nG26 (44 dup lines): _extract_table_name across 6 platforms:\n firebolt.py:1063, lakesail.py:504, presto.py:913, redshift.py:1692,\n spark.py:539, trino.py:913. Identical SQL table name extraction from\n CREATE TABLE with shared regex pattern (IF NOT EXISTS variant) and\n try/except/pass structure. 4 additional platforms (azure_synapse:1102,\n athena:1176, fabric_warehouse:1223, questdb:437) have platform-specific\n regex variants (bracket-quoting, EXTERNAL TABLE, schema-qualified names,\n quote-handling) and are excluded from this consolidation.", "id": "dedup-wave3-sql-platform-adapters", "preserves": ["Azure Spark statement execution and polling semantics unchanged", "Azure config builders produce identical connection parameters", "Cloud URL parsing returns identical (bucket, path) tuples", "_extract_table_name returns identical table name strings from DDL", "apply_table_tunings behavior identical per platform (including ImportError handling)", "create_schema DDL output byte-for-byte identical for Presto and Trino", "get_query_plan output format identical (newline-joined plan rows)"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/utils/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Consolidate SQL platform adapter duplicates: tuning, config, schema, Azure Spark, query plans, URL parsing, table extraction", "verifications": [{"command": "uv run -- python -m pytest tests/ -v -k 'firebolt or presto or trino or athena or azure or clickhouse or redshift or spark or lakesail'", "description": "Platform adapter tests pass", "expected": "exit code 0, no failures", "item_id": "dedup-wave3-sql-platform-adapters", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose 2>&1 | grep -c 'apply_table_tunings\\|_build_lakesail_config\\|_wait_for_statement\\|_extract_table_name'", "description": "Duplicate check shows reduction for targeted groups", "expected": "0", "item_id": "dedup-wave3-sql-platform-adapters", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "97% identical across platforms. Only ImportError handling differs.\nUse a platform_name parameter for log messages.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _apply_table_tunings_base into shared module, refactor firebolt/lakesail/presto", "wid": "w1"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Both build connection configs from the same parameters.\nKeep platform-specific defaults as post-build overrides.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _build_azure_compatible_config utility, refactor lakesail and azure_synapse", "wid": "w2"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Extends the existing wave-2 shared module.\nByte-for-byte identical logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add create_schema to presto_trino_utils.py, refactor both platforms", "wid": "w3"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Two functions, both identical between Fabric and Synapse adapters.\nBoth use the same Livy session API.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract Azure Spark _execute_statement and _wait_for_statement into spark_execution_utils.py", "wid": "w4"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Verbatim identical across athena, azure_synapse, firebolt, presto, trino.\nStandard EXPLAIN -> fetchall -> join pattern.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract get_sql_query_plan utility, refactor 5 platform adapters", "wid": "w5"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "URL parsing for s3:// and gs:// schemes.\nConsider a unified parser supporting both.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract parse_cloud_url utility, refactor clickhouse_cloud and firebolt", "wid": "w6"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": [], "notes": "Same SQL parsing logic across firebolt, lakesail, presto, redshift, spark, trino.\nAll 6 use identical regex (IF NOT EXISTS variant) and try/except/pass structure.\n4 divergent platforms (azure_synapse, athena, fabric_warehouse, questdb) have\nplatform-specific regex and are excluded - different enough to not consolidate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract extract_table_name utility, refactor 6 platform adapters", "wid": "w7"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": ["w1", "w2", "w3", "w4", "w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run platform-specific tests: pytest -k 'firebolt or lakesail or presto or trino or azure or spark or athena or clickhouse or redshift'", "wid": "w8"}, {"evidence": null, "item_id": "dedup-wave3-sql-platform-adapters", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run full test suite to verify no regressions: make test-all", "wid": "w9"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Benchmark Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "deferred-dataframe-benchmark-implementations", "id": 374, "reason": "These benchmarks test SQL optimizer behavior (syntax rewriting, join ordering) which is meaningless for DataFrame execution. Likely never applicable to DataFrame mode.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason states DataFrame execution bypasses SQL parsing entirely, so this is 'likely never applicable'; self-obsoleting.", "summary": "TPC-Havoc and JoinOrder DataFrame implementations"}], "deps": [], "description": "Consolidates 6 deferred benchmark implementations from\nimplement-dataframe-benchmarks (completed 2026-02-18) and 3 DataFusion-specific\ngaps from fix-read-primitives-datafusion-df (completed 2026-02-12).\n\nThese benchmarks were descoped because they don't map cleanly to the DataFrame\nread-query model. Each needs dedicated architecture work before implementation.\n\nDataFrame benchmark deferrals:\n- TPC-DI (38 ops): ETL pipeline with mutation-heavy SCD Type 2 operations\n- Write Primitives (109 ops): INSERT/UPDATE/DELETE/DDL - needs write-operation adapter layer\n- TPC-DS-OBT (89 queries): Fully denormalized table doesn't map to TPC-DS DataFrame schema\n- TPC-Havoc (220 queries): Tests SQL optimizer syntax rewriting - meaningless for DataFrames\n- JoinOrder (113 queries): Tests SQL optimizer join ordering - DataFrames handle joins programmatically\n- Read Primitives (136 queries): Catalog-driven architecture with DB-specific features\n\nDataFusion v50 gaps:\n- string_to_array returning actual list expressions (not split_part proxy)\n- array_sum / list.sum() (no built-in; needs unnest+aggregate workaround)\n- array_distinct Arrow type normalization (returns Dictionary instead of Utf8)", "id": "deferred-dataframe-benchmark-implementations", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Deferred DataFrame Benchmark Implementations - Architecturally Complex Benchmarks", "verifications": [], "work": [{"evidence": null, "item_id": "deferred-dataframe-benchmark-implementations", "needs": [], "notes": "Completed 2026-02-07 via dataframe-tpcds-obt-adaptation +\nimplement-tpcds-obt-dataframe-queries (DONE/dataframe-benchmarks/active/).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Evaluate TPC-DS-OBT feasibility - design denormalization step for DataFrame loading pipeline", "wid": "w1"}, {"evidence": null, "item_id": "deferred-dataframe-benchmark-implementations", "needs": [], "notes": "Completed 2026-02-07 via dataframe-write-primitives-model +\nintegrate-write-primitives-dataframe-mode + dataframe-write-primitives\n(DONE/dataframe-benchmarks/active/ and DONE/dataframe-expansion/active/).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design write-operation adapter layer for DataFrame platforms (INSERT/UPDATE/DELETE semantics)", "wid": "w2"}, {"evidence": null, "item_id": "deferred-dataframe-benchmark-implementations", "needs": [], "notes": "Completed via dataframe-read-primitives-architecture +\nintegrate-read-primitives-dataframe-mode + dataframe-read-primitives-complete\n(DONE/dataframe-benchmarks/active/ and DONE/dataframe-expansion/).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Evaluate Read Primitives DataFrame approach - catalog-driven query generation with different architecture", "wid": "w3"}, {"evidence": null, "item_id": "deferred-dataframe-benchmark-implementations", "needs": [], "notes": "Resolved 2026-02-12 via fix-read-primitives-datafusion-df\n(DONE/dataframe-benchmarks/active/). string_to_array and array_distinct fixed;\narray_sum confirmed genuinely unsupported in DataFusion v50 - skipped with xfail.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Monitor DataFusion releases for string_to_array, array_sum, array_distinct fixes - update adapter when resolved", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Performance Analysis", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "deferred-dataframe-performance-analysis", "id": 143, "reason": "Adapters for these platforms don't exist yet. Add to the comparison suite incrementally as adapters are implemented.", "resolution": "promoted", "resolved_item": "modin-cudf-and-dask-adapter-integration-into-cross-platform", "resolved_reason": null, "summary": "Modin, cuDF, and Dask adapter integration into cross-platform suite"}], "deps": [], "description": "Consolidates 3 deferred items from dataframe-performance-optimization-and-analysis\n(completed 2025-01-07). These represent the next tier of DataFrame performance\nwork beyond the initial profiling infrastructure.\n\nOriginally deferred 13+ months ago. The landscape has evolved - existing\nadapters (Polars, Pandas, DataFusion, PySpark) now provide enough coverage\nto start the cross-platform comparison without waiting for Modin/cuDF/Dask.\n\nDeferred items:\n1. Cross-platform benchmark suite - start with existing adapters (Polars,\n Pandas, DataFusion, PySpark), expand to Modin/cuDF/Dask later\n2. SQL vs DataFrame comparison benchmark (DuckDB SQL vs DataFrame API;\n DataFusion SQL vs DataFrame) - both platforms support dual execution modes\n3. Performance comparison report generation", "id": "deferred-dataframe-performance-analysis", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Deferred DataFrame Performance Cross-Platform Analysis", "verifications": [], "work": [{"evidence": null, "item_id": "deferred-dataframe-performance-analysis", "needs": [], "notes": "Start with existing adapters. Define: which TPC-H queries, which scale\nfactors, which metrics (wall time, memory, CPU). Design output format\nfor comparison tables.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design cross-platform DataFrame benchmark matrix - Polars vs Pandas vs DataFusion vs PySpark on TPC-H queries", "wid": "w1"}, {"evidence": null, "item_id": "deferred-dataframe-performance-analysis", "needs": ["w1"], "notes": "Both DuckDB and DataFusion support dual execution modes. Compare per-query\ntiming to quantify the overhead/advantage of each execution path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement SQL vs DataFrame comparison mode - run identical TPC-H queries via SQL and DataFrame on DuckDB and DataFusion", "wid": "w2"}, {"evidence": null, "item_id": "deferred-dataframe-performance-analysis", "needs": ["w1"], "notes": "Generate comparison tables, highlight fastest/slowest per query, compute\ngeometric means. Integrate with existing ASCII chart system.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build performance comparison report generator - aggregate multi-platform results into structured comparison output", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "require an explicit migration subcommand with a --confirm flag or dry-run mode", "dont": "auto-migrate data without explicit user confirmation", "why": "shared global caches may be intentionally cross-project"}, {"INSTEAD": "use the --global-cache flag as opt-in only", "dont": "change default storage path behavior", "why": "existing users have data at benchmark_runs/ and any default change would silently break their runs"}, {"INSTEAD": "verify that renaming dataframe-data/ does not invalidate any stored manifest references before committing the change", "dont": "rename datagen/ in place", "why": "existing runs have paths baked into result manifests"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Consolidates 3 deferred items from unify-dataframe-sql-default-storage-location\n(completed 2026-02-09). The core fix unified DataFrame and SQL storage to\nproject-local benchmark_runs/. These are follow-up refinements:\n\n1. CLI --global-cache flag to opt into ~/.benchbox/ location\n2. Auto-migration of existing ~/.benchbox/dataframe-data/ contents\n3. Unify cache subdirectory naming (datagen/ vs dataframe-data/)", "id": "deferred-storage-and-cache-unification", "preserves": ["Default project-local storage (benchmark_runs/) continues to work without any flags", "Existing DataFrame and SQL data loading paths produce identical results"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "do_not_modify", "path_glob": "benchbox/results/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/"}], "state": "done", "title": "Deferred Storage Location and Cache Unification Enhancements", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/dataframe/ -v --tb=short", "description": "Existing tests pass - storage paths unchanged by default", "expected": "all tests pass, 0 failures", "item_id": "deferred-storage-and-cache-unification", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/ -k global_cache -v --tb=short", "description": "--global-cache flag routes to ~/.benchbox/ location", "expected": "tests pass, flag resolves to ~/.benchbox/", "item_id": "deferred-storage-and-cache-unification", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.core.dataframe import data_loader; print(data_loader.get_data_path())\"", "description": "Default path unchanged when flag absent", "expected": "path contains benchmark_runs/", "item_id": "deferred-storage-and-cache-unification", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "deferred-storage-and-cache-unification", "needs": [], "notes": "Wire as a Click option in run.py, pass through the config/context object, thread\ninto benchmark_mixin.py and down to data_loader.py. Keep the default branch\n(no flag) identical to current behavior - only the opt-in path changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --global-cache CLI flag to opt into ~/.benchbox/ storage location for shared cross-project caching", "wid": "w1"}, {"evidence": null, "item_id": "deferred-storage-and-cache-unification", "needs": ["w1"], "notes": "No backward compatibility required. Dropped per owner decision 2026-02-18.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add explicit migration command for existing ~/.benchbox/dataframe-data/ contents to project-local", "wid": "w2"}, {"evidence": null, "item_id": "deferred-storage-and-cache-unification", "needs": [], "notes": "Independent of w1 - rename applies to the project-local path regardless of whether\n--global-cache exists. Removed needs: [w1] dependency since local naming and the\nglobal cache flag are orthogonal concerns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unify cache subdirectory naming between SQL (datagen/) and DataFrame (dataframe-data/) paths", "wid": "w3"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "define comparison eligibility first", "dont": "allow ranking before compatibility cohorts are defined", "why": "mixed cohorts create false leaderboards"}, {"INSTEAD": "make trust labels explicit in the contract", "dont": "treat every public upload as equally trustworthy", "why": "self-reported and curated results carry different weight"}], "approach": "Treat the canonical schema-v2 bundle as input, not as the full hosted public\ncontract. Add hosted result identity, visibility, and trust semantics above\nit without mutating the raw bundle format into a service-shaped payload.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-29T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-results-platform-product-and-launch-strategy"], "description": "A hosted results platform is not just an upload endpoint. BenchBox needs a\ndurable contract for what a hosted result is, how it relates to the canonical\nschema-v2 bundle, who can see it, how trustworthy it is, and which runs are\nvalid to compare or rank together.\n\nThis item defines the canonical hosted object model and the governance rules\naround visibility, trust promotion, compatibility cohorts, privacy, and\npublic claims. Without this layer, the API, UI, and ranking surfaces will\ndrift into misleading or contradictory behavior.\n\n**Phase assignment:** Phase 2-3 preparation. NOT required for Phase 1 (all\nresults are maintainer-curated, so trust labels and governance are trivial).\nThe cohort compatibility rules (w3) are partially relevant to Phase 1 compare\nviews but can be handled with simple same-benchmark/same-scale validation\nwithout a full governance model.", "id": "define-hosted-results-contract-and-governance-model", "preserves": ["BenchBox does not imply TPC certification or equivalent official standing for ordinary hosted submissions", "Canonical BenchBox result bundles remain downloadable and distinguishable from any derived hosted metadata"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}], "state": "done", "title": "Define hosted results contract, trust labels, and governance model", "verifications": [], "work": [{"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": [], "notes": "Separate three distinct identifiers: (1) bundle hash - content-addressable,\nimmutable; (2) submission ID - generated on ingest, survives re-upload;\n(3) public result ID - stable permalink, may be minted after curation.\nDefine allowed ingest status transitions: pending \u2192 validated \u2192 published,\nplus rejected and withdrawn. Output: identity-model.md draft.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define hosted result identity, immutable bundle contract, and ingest status model", "wid": "w1"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w1"], "notes": "Cover private, unlisted, public self-reported, public curated, and public\nverified states or an explicitly revised equivalent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define visibility states and trust labels for hosted results", "wid": "w2"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w2"], "notes": "Prevent incompatible runs from being mixed into misleading comparisons.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define cohort and ranking eligibility rules for comparison and leaderboards", "wid": "w3"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w2"], "notes": "Decide what system metadata may be public, redactable, or removable after submission.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define privacy, redaction, retention, and data-removal policy", "wid": "w4"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w3", "w4"], "notes": "Produce a single hosted-results-contract.md that specifies:\n- What fields the submission endpoint accepts and returns\n- Which visibility states and trust labels the explorer must render\n- Which cohort fields must match for a compare view to be valid\n- What privacy/redaction/removal operations are supported\nThis doc is the authoritative input for design-results-ingest (Phase 3\nAPI design) and the explorer frontend trust-label implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the contract for CLI, API, and frontend consumers", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "pick metrics that can't be measured cheaply", "why": "a dashboard nobody can build is the same as no dashboard; favor `gh` CLI queries and a flat notes file over building a metrics service."}, {"INSTEAD": "(not specified)", "dont": "set the thresholds so low that they trip on noise", "why": "false promotion of Phase 3 wastes the largest engineering investment in the project."}, {"INSTEAD": "(not specified)", "dont": "start any Phase 3 implementation work as part of this TODO", "why": "that defeats the deferral the strategy doc explicitly requires; this is measurement and decision plumbing only."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-27T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["implement-pr-based-result-submission-workflow"], "description": "The strategy doc explicitly defers Phase 3 (hosted submission API) until\nPhase 2 PR volume justifies the operational cost. It cites indicators\n(\"submission volume exceeds ~50/month sustained\", \"maintainer review\nbecomes a sustained bottleneck\", etc.) but no one is currently measuring\nthose indicators, and there is no decision moment where someone is asked\nto look.\n\nWithout measurement and a forcing function, the Phase 3 promotion\ndecision will be made by whoever feels frustrated first - which is\nexactly the kind of ad-hoc engineering decision the strategy doc is\ntrying to prevent. This item operationalizes the trigger: pick the\nmetrics, build the cheapest possible dashboard, and put a review on\nsomeone's calendar.", "id": "define-phase-3-promotion-metrics", "preserves": ["Existing Phase 3 planning TODOs (`design-results-ingest-storage-and-derived-read-model`, `integrate-benchbox-cli-submit-and-service-auth`, `operate-results-platform-security-observability-and-abuse-controls`) remain Not Started until a review explicitly promotes them.", "Strategy doc's deferral of Phase 3 - this TODO measures the trigger, doesn't pull it."], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/phase-3-promotion-metrics.md"}, {"kind": "only_modify", "path_glob": "_project/handoffs/phase-3-review-baseline-*.md"}, {"kind": "only_modify", "path_glob": "_project/notes/phase-2-requests.md"}, {"kind": "only_modify", "path_glob": "scripts/phase2_metrics.py"}], "state": "done", "title": "Define Phase 3 promotion metrics, dashboard, and review cadence", "verifications": [{"command": "test -f _project/analysis/phase-3-promotion-metrics.md && grep -c 'threshold' _project/analysis/phase-3-promotion-metrics.md", "description": "Promotion metrics doc exists with concrete thresholds", "expected": ">= 4", "item_id": "define-phase-3-promotion-metrics", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run python scripts/phase2_metrics.py", "description": "Metrics script runs and produces a report", "expected": "exit 0, markdown output with one section per metric", "item_id": "define-phase-3-promotion-metrics", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": [], "notes": "From the strategy doc indicators, choose 4-6 measurable signals.\nSuggested starting set:\n - Merged community submissions / month (threshold 50 sustained)\n - Median PR review latency in hours (threshold 72h)\n - PR backlog (open >7 days) (threshold 5 sustained)\n - Requests for private/unlisted results (threshold 3 distinct\n requests)\n - Submissions blocked by maintainer-only constraints (threshold 5)\n - Org-account / team-space requests (threshold 3 distinct)\nDocument each metric with: query/source, threshold, what it implies\nif breached. Save as\n`_project/analysis/phase-3-promotion-metrics.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pick concrete promotion metrics and thresholds", "wid": "w1"}, {"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": ["w1"], "notes": "Inputs:\n - GitHub API (PRs against `published-results`, labels, merge dates,\n review timestamps) via `gh` CLI subprocess or PyGithub.\n - Optional: user-feedback file `_project/notes/phase-2-requests.md`\n for the qualitative metrics.\nOutput: markdown report to stdout (or `--output FILE`), one section\nper metric, current value vs threshold, simple Y/N \"breached?\" flag.\nRead-only; no API writes; no auth beyond `gh auth status`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a minimal `scripts/phase2_metrics.py` that computes the metrics", "wid": "w2"}, {"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": ["w2"], "notes": "Add to `_project/analysis/phase-3-promotion-metrics.md`:\n - Review cadence: quarterly (first Monday of each quarter).\n - Owner: BenchBox maintainer rotation.\n - Procedure: run `phase2_metrics.py`, paste output to handoff file\n `_project/handoffs/phase-3-review-.md`, decide whether any\n threshold breach justifies promoting\n `design-results-ingest-storage-and-derived-read-model` and\n `integrate-benchbox-cli-submit-and-service-auth` to Active.\nAdd a calendar reminder (or repo-level reminder issue) for the next\nreview date. Calendar = source of truth; this TODO doesn't try to\ninvent a scheduler.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Schedule a quarterly review and document the promotion procedure", "wid": "w3"}, {"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": ["w3"], "notes": "Run the script today against the (currently small) Phase 2 dataset\nand commit the baseline report to\n`_project/handoffs/phase-3-review-baseline-.md`. Even if all\nthresholds are far from breached, the baseline shows the trajectory\nover time.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the first baseline review and capture the starting state", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "choose the site topology explicitly", "dont": "assume the static docs stack can absorb a rich results explorer unchanged", "why": "interactive compare/search/auth workflows have different needs"}, {"INSTEAD": "define `publish` and `submit` separately", "dont": "collapse local artifact publication and hosted submission into one command surface", "why": "the storage contract and service contract differ materially"}], "approach": "Start with product shape, not implementation. The result explorer, hosted\nAPI, and local artifact publishing should each have a separate contract and\nsuccess metric. Use the existing BenchBox export path as the anchor, then map\nexternal reference patterns onto the repo's real deployment constraints.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-29T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox currently blurs together three different concerns:\n1) local/cloud artifact publication,\n2) hosted result submission,\n3) public result browsing and analysis on benchbox.dev.\n\nThat ambiguity makes the existing publishing TODO too large and too vague to\nexecute safely. This item defines the product boundary between those concerns,\nchooses the initial site topology, and sequences the launch so BenchBox does\nnot jump directly from local result files to an open public submission portal.\n\nThe target output is a high-confidence strategy that distinguishes\n`benchbox publish`, `benchbox submit`, and `benchbox.dev/results/`, plus a\ndelivery sequence the rest of the TODO cluster can depend on.\n\n**Phase assignment:** This is a prerequisite planning item. Must complete before\nany Phase 1 implementation begins. The refined strategy memo at\n`_project/analysis/benchbox-results-platform-strategy.md` contains the current\nrecommended phasing: Phase 1 (static explorer MVP with repo-local\n`results-data/`), Phase 2 (PR-based community contributions against that\ncorpus), Phase 3 (hosted API, only if demand warrants).", "id": "define-results-platform-product-and-launch-strategy", "preserves": ["The existing static GitHub Pages deployment for landing/docs/blog remains a valid baseline during planning", "The schema-v2 result bundle remains the single canonical artifact for all downstream publishing and submission flows"], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}], "state": "done", "title": "Define BenchBox results platform product boundary and launch strategy", "verifications": [], "work": [{"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": [], "notes": "Capture the current canonical result path, static-site deployment model,\nand any pre-existing hints of hosted service design.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit current BenchBox result export, results CLI, and public-site constraints", "wid": "w1"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w1"], "notes": "Classify what each reference teaches BenchBox about submission, browsing,\ncomparison, static hosting, and trust labels.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a reference matrix from Geekbench, CloudSpecs, OpenBenchmarking, and ASV", "wid": "w2"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w2"], "notes": "Cover at least maintainers/curators, result submitters, and result readers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define personas, workflows, trust levels, and phase-by-phase success metrics", "wid": "w3"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w3"], "notes": "Confirm the launch topology: explorer at `/results/`, seed corpus stored\nunder repo-local `results-data/`, and a single GitHub Pages workflow for\ndeployment. Explicitly separate local/cloud publication from hosted\nsubmission. A dedicated data repo is a later optimization, not the default.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose the initial topology and command boundary for publish vs submit", "wid": "w4"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Publish the strategy artifact and update companion TODO dependencies", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "auto-migrate data without explicit user confirmation", "instead": "require an explicit migration subcommand with a --confirm flag or dry-run mode", "why": "shared global caches may be intentionally cross-project"}, {"dont": "change default storage path behavior", "instead": "use the --global-cache flag as opt-in only", "why": "existing users have data at benchmark_runs/ and any default change would silently break their runs"}, {"dont": "rename datagen/ in place", "instead": "verify that renaming dataframe-data/ does not invalidate any stored manifest references before committing the change", "why": "existing runs have paths baked into result manifests"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-18T00:00:00Z", "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Consolidates 3 deferred items from unify-dataframe-sql-default-storage-location\n(completed 2026-02-09). The core fix unified DataFrame and SQL storage to\nproject-local benchmark_runs/. These are follow-up refinements:\n\n1. CLI --global-cache flag to opt into ~/.benchbox/ location\n2. Auto-migration of existing ~/.benchbox/dataframe-data/ contents\n3. Unify cache subdirectory naming (datagen/ vs dataframe-data/)", "id": "deferred-storage-and-cache-unification", "preserves": ["Default project-local storage (benchmark_runs/) continues to work without any flags", "Existing DataFrame and SQL data loading paths produce identical results"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "do_not_modify", "path_glob": "benchbox/results/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/data_loader.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/"}], "state": "done", "title": "Deferred Storage Location and Cache Unification Enhancements", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/dataframe/ -v --tb=short", "description": "Existing tests pass - storage paths unchanged by default", "expected": "all tests pass, 0 failures", "item_id": "deferred-storage-and-cache-unification", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/ -k global_cache -v --tb=short", "description": "--global-cache flag routes to ~/.benchbox/ location", "expected": "tests pass, flag resolves to ~/.benchbox/", "item_id": "deferred-storage-and-cache-unification", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"from benchbox.core.dataframe import data_loader; print(data_loader.get_data_path())\"", "description": "Default path unchanged when flag absent", "expected": "path contains benchmark_runs/", "item_id": "deferred-storage-and-cache-unification", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "deferred-storage-and-cache-unification", "needs": [], "notes": "Wire as a Click option in run.py, pass through the config/context object, thread\ninto benchmark_mixin.py and down to data_loader.py. Keep the default branch\n(no flag) identical to current behavior - only the opt-in path changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --global-cache CLI flag to opt into ~/.benchbox/ storage location for shared cross-project caching", "wid": "w1"}, {"evidence": null, "item_id": "deferred-storage-and-cache-unification", "needs": ["w1"], "notes": "No backward compatibility required. Dropped per owner decision 2026-02-18.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add explicit migration command for existing ~/.benchbox/dataframe-data/ contents to project-local", "wid": "w2"}, {"evidence": null, "item_id": "deferred-storage-and-cache-unification", "needs": [], "notes": "Independent of w1 - rename applies to the project-local path regardless of whether\n--global-cache exists. Removed needs: [w1] dependency since local naming and the\nglobal cache flag are orthogonal concerns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unify cache subdirectory naming between SQL (datagen/) and DataFrame (dataframe-data/) paths", "wid": "w3"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "allow ranking before compatibility cohorts are defined", "instead": "define comparison eligibility first", "why": "mixed cohorts create false leaderboards"}, {"dont": "treat every public upload as equally trustworthy", "instead": "make trust labels explicit in the contract", "why": "self-reported and curated results carry different weight"}], "approach": "Treat the canonical schema-v2 bundle as input, not as the full hosted public\ncontract. Add hosted result identity, visibility, and trust semantics above\nit without mutating the raw bundle format into a service-shaped payload.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-29T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-results-platform-product-and-launch-strategy"], "description": "A hosted results platform is not just an upload endpoint. BenchBox needs a\ndurable contract for what a hosted result is, how it relates to the canonical\nschema-v2 bundle, who can see it, how trustworthy it is, and which runs are\nvalid to compare or rank together.\n\nThis item defines the canonical hosted object model and the governance rules\naround visibility, trust promotion, compatibility cohorts, privacy, and\npublic claims. Without this layer, the API, UI, and ranking surfaces will\ndrift into misleading or contradictory behavior.\n\n**Phase assignment:** Phase 2-3 preparation. NOT required for Phase 1 (all\nresults are maintainer-curated, so trust labels and governance are trivial).\nThe cohort compatibility rules (w3) are partially relevant to Phase 1 compare\nviews but can be handled with simple same-benchmark/same-scale validation\nwithout a full governance model.", "id": "define-hosted-results-contract-and-governance-model", "preserves": ["BenchBox does not imply TPC certification or equivalent official standing for ordinary hosted submissions", "Canonical BenchBox result bundles remain downloadable and distinguishable from any derived hosted metadata"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}], "state": "done", "title": "Define hosted results contract, trust labels, and governance model", "verifications": [], "work": [{"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": [], "notes": "Separate three distinct identifiers: (1) bundle hash - content-addressable,\nimmutable; (2) submission ID - generated on ingest, survives re-upload;\n(3) public result ID - stable permalink, may be minted after curation.\nDefine allowed ingest status transitions: pending \u2192 validated \u2192 published,\nplus rejected and withdrawn. Output: identity-model.md draft.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define hosted result identity, immutable bundle contract, and ingest status model", "wid": "w1"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w1"], "notes": "Cover private, unlisted, public self-reported, public curated, and public\nverified states or an explicitly revised equivalent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define visibility states and trust labels for hosted results", "wid": "w2"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w2"], "notes": "Prevent incompatible runs from being mixed into misleading comparisons.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define cohort and ranking eligibility rules for comparison and leaderboards", "wid": "w3"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w2"], "notes": "Decide what system metadata may be public, redactable, or removable after submission.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define privacy, redaction, retention, and data-removal policy", "wid": "w4"}, {"evidence": null, "item_id": "define-hosted-results-contract-and-governance-model", "needs": ["w3", "w4"], "notes": "Produce a single hosted-results-contract.md that specifies:\n- What fields the submission endpoint accepts and returns\n- Which visibility states and trust labels the explorer must render\n- Which cohort fields must match for a compare view to be valid\n- What privacy/redaction/removal operations are supported\nThis doc is the authoritative input for design-results-ingest (Phase 3\nAPI design) and the explorer frontend trust-label implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the contract for CLI, API, and frontend consumers", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "pick metrics that can't be measured cheaply", "instead": "(not specified)", "why": "a dashboard nobody can build is the same as no dashboard; favor `gh` CLI queries and a flat notes file over building a metrics service."}, {"dont": "set the thresholds so low that they trip on noise", "instead": "(not specified)", "why": "false promotion of Phase 3 wastes the largest engineering investment in the project."}, {"dont": "start any Phase 3 implementation work as part of this TODO", "instead": "(not specified)", "why": "that defeats the deferral the strategy doc explicitly requires; this is measurement and decision plumbing only."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-27T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["implement-pr-based-result-submission-workflow"], "description": "The strategy doc explicitly defers Phase 3 (hosted submission API) until\nPhase 2 PR volume justifies the operational cost. It cites indicators\n(\"submission volume exceeds ~50/month sustained\", \"maintainer review\nbecomes a sustained bottleneck\", etc.) but no one is currently measuring\nthose indicators, and there is no decision moment where someone is asked\nto look.\n\nWithout measurement and a forcing function, the Phase 3 promotion\ndecision will be made by whoever feels frustrated first - which is\nexactly the kind of ad-hoc engineering decision the strategy doc is\ntrying to prevent. This item operationalizes the trigger: pick the\nmetrics, build the cheapest possible dashboard, and put a review on\nsomeone's calendar.", "id": "define-phase-3-promotion-metrics", "preserves": ["Existing Phase 3 planning TODOs (`design-results-ingest-storage-and-derived-read-model`, `integrate-benchbox-cli-submit-and-service-auth`, `operate-results-platform-security-observability-and-abuse-controls`) remain Not Started until a review explicitly promotes them.", "Strategy doc's deferral of Phase 3 - this TODO measures the trigger, doesn't pull it."], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/phase-3-promotion-metrics.md"}, {"kind": "only_modify", "path_glob": "_project/handoffs/phase-3-review-baseline-*.md"}, {"kind": "only_modify", "path_glob": "_project/notes/phase-2-requests.md"}, {"kind": "only_modify", "path_glob": "scripts/phase2_metrics.py"}], "state": "done", "title": "Define Phase 3 promotion metrics, dashboard, and review cadence", "verifications": [{"command": "test -f _project/analysis/phase-3-promotion-metrics.md && grep -c 'threshold' _project/analysis/phase-3-promotion-metrics.md", "description": "Promotion metrics doc exists with concrete thresholds", "expected": ">= 4", "item_id": "define-phase-3-promotion-metrics", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run python scripts/phase2_metrics.py", "description": "Metrics script runs and produces a report", "expected": "exit 0, markdown output with one section per metric", "item_id": "define-phase-3-promotion-metrics", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": [], "notes": "From the strategy doc indicators, choose 4-6 measurable signals.\nSuggested starting set:\n - Merged community submissions / month (threshold 50 sustained)\n - Median PR review latency in hours (threshold 72h)\n - PR backlog (open >7 days) (threshold 5 sustained)\n - Requests for private/unlisted results (threshold 3 distinct\n requests)\n - Submissions blocked by maintainer-only constraints (threshold 5)\n - Org-account / team-space requests (threshold 3 distinct)\nDocument each metric with: query/source, threshold, what it implies\nif breached. Save as\n`_project/analysis/phase-3-promotion-metrics.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pick concrete promotion metrics and thresholds", "wid": "w1"}, {"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": ["w1"], "notes": "Inputs:\n - GitHub API (PRs against `published-results`, labels, merge dates,\n review timestamps) via `gh` CLI subprocess or PyGithub.\n - Optional: user-feedback file `_project/notes/phase-2-requests.md`\n for the qualitative metrics.\nOutput: markdown report to stdout (or `--output FILE`), one section\nper metric, current value vs threshold, simple Y/N \"breached?\" flag.\nRead-only; no API writes; no auth beyond `gh auth status`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a minimal `scripts/phase2_metrics.py` that computes the metrics", "wid": "w2"}, {"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": ["w2"], "notes": "Add to `_project/analysis/phase-3-promotion-metrics.md`:\n - Review cadence: quarterly (first Monday of each quarter).\n - Owner: BenchBox maintainer rotation.\n - Procedure: run `phase2_metrics.py`, paste output to handoff file\n `_project/handoffs/phase-3-review-.md`, decide whether any\n threshold breach justifies promoting\n `design-results-ingest-storage-and-derived-read-model` and\n `integrate-benchbox-cli-submit-and-service-auth` to Active.\nAdd a calendar reminder (or repo-level reminder issue) for the next\nreview date. Calendar = source of truth; this TODO doesn't try to\ninvent a scheduler.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Schedule a quarterly review and document the promotion procedure", "wid": "w3"}, {"evidence": null, "item_id": "define-phase-3-promotion-metrics", "needs": ["w3"], "notes": "Run the script today against the (currently small) Phase 2 dataset\nand commit the baseline report to\n`_project/handoffs/phase-3-review-baseline-.md`. Even if all\nthresholds are far from breached, the baseline shows the trajectory\nover time.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the first baseline review and capture the starting state", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "assume the static docs stack can absorb a rich results explorer unchanged", "instead": "choose the site topology explicitly", "why": "interactive compare/search/auth workflows have different needs"}, {"dont": "collapse local artifact publication and hosted submission into one command surface", "instead": "define `publish` and `submit` separately", "why": "the storage contract and service contract differ materially"}], "approach": "Start with product shape, not implementation. The result explorer, hosted\nAPI, and local artifact publishing should each have a separate contract and\nsuccess metric. Use the existing BenchBox export path as the anchor, then map\nexternal reference patterns onto the repo's real deployment constraints.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-03-29T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox currently blurs together three different concerns:\n1) local/cloud artifact publication,\n2) hosted result submission,\n3) public result browsing and analysis on benchbox.dev.\n\nThat ambiguity makes the existing publishing TODO too large and too vague to\nexecute safely. This item defines the product boundary between those concerns,\nchooses the initial site topology, and sequences the launch so BenchBox does\nnot jump directly from local result files to an open public submission portal.\n\nThe target output is a high-confidence strategy that distinguishes\n`benchbox publish`, `benchbox submit`, and `benchbox.dev/results/`, plus a\ndelivery sequence the rest of the TODO cluster can depend on.\n\n**Phase assignment:** This is a prerequisite planning item. Must complete before\nany Phase 1 implementation begins. The refined strategy memo at\n`_project/analysis/benchbox-results-platform-strategy.md` contains the current\nrecommended phasing: Phase 1 (static explorer MVP with repo-local\n`results-data/`), Phase 2 (PR-based community contributions against that\ncorpus), Phase 3 (hosted API, only if demand warrants).", "id": "define-results-platform-product-and-launch-strategy", "preserves": ["The existing static GitHub Pages deployment for landing/docs/blog remains a valid baseline during planning", "The schema-v2 result bundle remains the single canonical artifact for all downstream publishing and submission flows"], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}], "state": "done", "title": "Define BenchBox results platform product boundary and launch strategy", "verifications": [], "work": [{"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": [], "notes": "Capture the current canonical result path, static-site deployment model,\nand any pre-existing hints of hosted service design.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit current BenchBox result export, results CLI, and public-site constraints", "wid": "w1"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w1"], "notes": "Classify what each reference teaches BenchBox about submission, browsing,\ncomparison, static hosting, and trust labels.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build a reference matrix from Geekbench, CloudSpecs, OpenBenchmarking, and ASV", "wid": "w2"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w2"], "notes": "Cover at least maintainers/curators, result submitters, and result readers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define personas, workflows, trust levels, and phase-by-phase success metrics", "wid": "w3"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w3"], "notes": "Confirm the launch topology: explorer at `/results/`, seed corpus stored\nunder repo-local `results-data/`, and a single GitHub Pages workflow for\ndeployment. Explicitly separate local/cloud publication from hosted\nsubmission. A dedicated data repo is a later optimization, not the default.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose the initial topology and command boundary for publish vs submit", "wid": "w4"}, {"evidence": null, "item_id": "define-results-platform-product-and-launch-strategy", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Publish the strategy artifact and update companion TODO dependencies", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:07Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #394 of execution-results-dataframe-routing-and-query-resolution.\nDeferred: Delete `benchbox/core/runner/dataframe_runner.py` completely\nReason deferred: The current public/backward-compat registry describes it as a deprecated internal compatibility runner; deletion needs a separate removal-window decision after helper imports migrate.", "id": "delete-benchbox-core-runner-dataframe-runner-py-completely", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Delete `benchbox/core/runner/dataframe_runner.py` completely", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "mix psycopg2 and psycopg3 imports across modules", "why": "pick one and complete the migration fully"}, {"INSTEAD": "(not specified)", "dont": "use psycopg3 async API in the synchronous adapter path without a separate design decision", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dependency-migration-psycopg3", "id": 310, "reason": "Async usage is additive and out of scope for the synchronous driver swap; file as a follow-on if async PostgreSQL connections are desired.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Async psycopg3 interface filed as optional follow-on 'if desired'; no evidence of demand or a concrete plan.", "summary": "Migrate to psycopg3 async interface for concurrent query paths"}], "deps": [], "description": "`psycopg2-binary` is a legacy PostgreSQL driver with known packaging\nfragility (ships pre-compiled C extensions in a binary wheel, no async\nsupport, soft-deprecated in many CI environments). `psycopg3` (package:\n`psycopg[binary]`) is the successor \u2014 async-compatible, PEP 249 v3 compliant,\nand actively maintained.\n\nSurfaced as a deferred item in the DONE file for `tighten-dependencies-phase-plan`\n(completed 2026-04-15): \"Migration off psycopg2-binary to psycopg3 \u2014 driver\nmodernization \u2014 out of scope for an *elimination* audit. File separately when\nproduct is ready.\"\n\nThis is not an elimination audit item \u2014 it is a deliberate driver swap that\nmay require updating connection-string parameters, cursor usage, and any\ncode that imports psycopg2 directly. Scope must be confirmed before\ncommitting to the swap.", "id": "dependency-migration-psycopg3", "preserves": ["All Redshift and PostgreSQL benchmark runs produce identical results before and after the swap", "Connection factory interface remains unchanged for any callers outside benchbox/platforms/"], "prior_art": [{"concept": "benchbox/platforms/postgres*.py, benchbox/platforms/redshift*.py \u2014 likely primary psycopg2 consumers", "decision": "reuse", "path": "benchbox/platforms/postgres*.py"}, {"concept": "pyproject.toml \u2014 replace psycopg2-binary here", "decision": "reuse", "path": "pyproject.toml"}, {"concept": "tighten-dependencies-phase-plan DONE file \u2014 deferred this item explicitly", "decision": "reuse", "path": "tighten-dependencies-phase-plan"}], "priority": "low", "scope": [], "state": "done", "title": "Migrate off psycopg2-binary to psycopg3 (asyncpg-compatible driver modernisation)", "verifications": [{"command": "grep -rn 'import psycopg2' benchbox/ | wc -l", "description": "No psycopg2 imports remain after migration", "expected": "0", "item_id": "dependency-migration-psycopg3", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'redshift or postgres' -q", "description": "PostgreSQL/Redshift unit tests pass", "expected": "all tests pass", "item_id": "dependency-migration-psycopg3", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": [], "notes": "grep -rn 'psycopg2\\|import psycopg' benchbox/ tests/\nClassify each site: direct import, connection factory, cursor type\nassumption, exception type catch. Estimate the change surface.\nAlso check if any tests mock psycopg2 internals that would need updating.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit all psycopg2 import sites and connection usage patterns", "wid": "w0"}, {"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": ["w0"], "notes": "psycopg3 is largely API-compatible with psycopg2 for synchronous use, but\nthere are known differences: parameter placeholders (%s vs %(name)s vs ?),\ncopy API, server-side cursors, and exception hierarchy. For each call site\nfound in w0, note whether it works unchanged or needs updating.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Assess psycopg3 API compatibility at each identified call site", "wid": "w1"}, {"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": ["w1"], "notes": "In pyproject.toml, replace psycopg2-binary with psycopg[binary] (or\npsycopg[c] for the C extension build). Update any `import psycopg2` to\n`import psycopg`. Keep a compatibility shim if the change surface from w1\nis large \u2014 prefer a single incremental swap over a big-bang refactor.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Swap psycopg2-binary for psycopg[binary] in pyproject.toml and update imports", "wid": "w2"}, {"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": ["w2"], "notes": "uv run -- python -m pytest tests/ -k 'redshift or postgres or postgresql' -q\nAlso run the full unit suite to catch any indirect psycopg2 dependency\nvia transitive imports. Fix any failures before marking this done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the full Redshift/PostgreSQL test suite and confirm zero regressions", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "mix psycopg2 and psycopg3 imports across modules", "instead": "(not specified)", "why": "pick one and complete the migration fully"}, {"dont": "use psycopg3 async API in the synchronous adapter path without a separate design decision", "instead": "(not specified)", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dependency-migration-psycopg3", "id": 310, "reason": "Async usage is additive and out of scope for the synchronous driver swap; file as a follow-on if async PostgreSQL connections are desired.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Async psycopg3 interface filed as optional follow-on 'if desired'; no evidence of demand or a concrete plan.", "summary": "Migrate to psycopg3 async interface for concurrent query paths"}], "deps": [], "description": "`psycopg2-binary` is a legacy PostgreSQL driver with known packaging\nfragility (ships pre-compiled C extensions in a binary wheel, no async\nsupport, soft-deprecated in many CI environments). `psycopg3` (package:\n`psycopg[binary]`) is the successor \u2014 async-compatible, PEP 249 v3 compliant,\nand actively maintained.\n\nSurfaced as a deferred item in the DONE file for `tighten-dependencies-phase-plan`\n(completed 2026-04-15): \"Migration off psycopg2-binary to psycopg3 \u2014 driver\nmodernization \u2014 out of scope for an *elimination* audit. File separately when\nproduct is ready.\"\n\nThis is not an elimination audit item \u2014 it is a deliberate driver swap that\nmay require updating connection-string parameters, cursor usage, and any\ncode that imports psycopg2 directly. Scope must be confirmed before\ncommitting to the swap.", "id": "dependency-migration-psycopg3", "preserves": ["All Redshift and PostgreSQL benchmark runs produce identical results before and after the swap", "Connection factory interface remains unchanged for any callers outside benchbox/platforms/"], "prior_art": [{"concept": "benchbox/platforms/postgres*.py, benchbox/platforms/redshift*.py \u2014 likely primary psycopg2 consumers", "decision": "reuse", "path": "benchbox/platforms/postgres*.py"}, {"concept": "pyproject.toml \u2014 replace psycopg2-binary here", "decision": "reuse", "path": "pyproject.toml"}, {"concept": "tighten-dependencies-phase-plan DONE file \u2014 deferred this item explicitly", "decision": "reuse", "path": "tighten-dependencies-phase-plan"}], "priority": "low", "scope": [], "state": "done", "title": "Migrate off psycopg2-binary to psycopg3 (asyncpg-compatible driver modernisation)", "verifications": [{"command": "grep -rn 'import psycopg2' benchbox/ | wc -l", "description": "No psycopg2 imports remain after migration", "expected": "0", "item_id": "dependency-migration-psycopg3", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'redshift or postgres' -q", "description": "PostgreSQL/Redshift unit tests pass", "expected": "all tests pass", "item_id": "dependency-migration-psycopg3", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": [], "notes": "grep -rn 'psycopg2\\|import psycopg' benchbox/ tests/\nClassify each site: direct import, connection factory, cursor type\nassumption, exception type catch. Estimate the change surface.\nAlso check if any tests mock psycopg2 internals that would need updating.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit all psycopg2 import sites and connection usage patterns", "wid": "w0"}, {"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": ["w0"], "notes": "psycopg3 is largely API-compatible with psycopg2 for synchronous use, but\nthere are known differences: parameter placeholders (%s vs %(name)s vs ?),\ncopy API, server-side cursors, and exception hierarchy. For each call site\nfound in w0, note whether it works unchanged or needs updating.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Assess psycopg3 API compatibility at each identified call site", "wid": "w1"}, {"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": ["w1"], "notes": "In pyproject.toml, replace psycopg2-binary with psycopg[binary] (or\npsycopg[c] for the C extension build). Update any `import psycopg2` to\n`import psycopg`. Keep a compatibility shim if the change surface from w1\nis large \u2014 prefer a single incremental swap over a big-bang refactor.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Swap psycopg2-binary for psycopg[binary] in pyproject.toml and update imports", "wid": "w2"}, {"evidence": null, "item_id": "dependency-migration-psycopg3", "needs": ["w2"], "notes": "uv run -- python -m pytest tests/ -k 'redshift or postgres or postgresql' -q\nAlso run the full unit suite to catch any indirect psycopg2 dependency\nvia transitive imports. Fix any failures before marking this done.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the full Redshift/PostgreSQL test suite and confirm zero regressions", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-14T00:00:00Z", "deferrals": [], "deps": ["deployment-factory-integration"], "description": "Integrate deployment mode awareness into the BenchBox CLI, providing clear\ndocumentation, help text, platform listing with deployment modes, and\nenhanced error messages for deployment-related issues.", "id": "deployment-cli-integration", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Platform Deployment CLI Integration", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-14T00:00:00Z", "deferrals": [], "deps": ["deployment-registry-extensions"], "description": "Implement configuration inheritance system allowing child platforms to inherit\nSQL dialect, benchmark compatibility, and data type mappings from parent platforms.\n\nThis enables MotherDuck to automatically inherit DuckDB's SQL dialect and\nbenchmark query translations, while having its own connection configuration.", "id": "deployment-config-inheritance", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Platform Configuration Inheritance", "verifications": [], "work": [], "worktree": "platform-expansion"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-14T00:00:00Z", "deferrals": [], "deps": ["deployment-registry-extensions"], "description": "Integrate deployment mode awareness into the adapter factory, enabling routing\nto appropriate adapters based on deployment mode (local, self-hosted, managed).\n\nThis extends the existing _normalize_platform_name() pattern that handles the\n-df suffix for DataFrame mode, adding support for :deployment suffix parsing\n(e.g., clickhouse:cloud, firebolt:core).", "id": "deployment-factory-integration", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Platform Deployment Factory Integration", "verifications": [], "work": [], "worktree": "platform-expansion"} @@ -365,22 +365,22 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:15Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #433 of landing-prompts-all-supported-platforms.\nDeferred: Derive benchmark compatibility per platform\nReason deferred: This TODO must fix platform availability. Benchmark-specific compatibility filtering can be added separately unless registry `unsupported_benchmarks` makes it cheap and safe.", "id": "derive-benchmark-compatibility-per-platform", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Derive benchmark compatibility per platform", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:45Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #512 of uat-single-connection-registry.\nDeferred: Derive credential/secondary-port literals (password=, http_port=, be_http_port=) from canonical sources\nReason deferred: Scoped out by decision (2026-05-31): host reachability + adapter port= now derive from compose; credential and secondary-port literals were relocated into the single docker_assets registry but not yet derived. Env keys vary per platform (ROOT_PASSWORD/POSTGRES_PASSWORD/...) and wrong derivation risks the must_preserve 'no new skips'. Tackle in a dedicated pass.", "id": "derive-credential-secondary-port-literals-password-http-port", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Derive credential/secondary-port literals (password=, http_port=, be_http_port=) from canonical sources", "verifications": [], "work": [], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-02T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Implement the foundational abstractions for DataFrame benchmarking including\nDataFrameQuery protocol, DataFrameContext, and DataFrameOps protocol that\nwill enable 95%+ code reuse across 8 dataframe platforms using a family-based\narchitecture.", "id": "design-and-implement-core-dataframe-abstractions", "preserves": [], "prior_art": [], "priority": "critical", "scope": [], "state": "done", "title": "Design and Implement Core DataFrame Abstractions", "verifications": [], "work": [], "worktree": "dataframe-core"} -{"anti_patterns": [{"INSTEAD": "separate them explicitly", "dont": "collapse object storage keys, submission IDs, and public result URLs into one identifier", "why": "retries, rebuilds, and public permalinks have different lifecycles"}, {"INSTEAD": "serve derived read models", "dont": "make the explorer query arbitrary raw uploads as its primary data source", "why": "that couples UX latency and safety to uncurated artifacts"}], "approach": "Use a split write/read architecture. Store raw bundles once, normalize into\ndurable metadata, and publish rebuildable read models for the explorer.\nPrefer static-friendly public reads and keep authenticated mutations behind\nthe hosted service boundary.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-27T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-hosted-results-contract-and-governance-model", "define-results-platform-product-and-launch-strategy"], "description": "Once BenchBox decides what a hosted result means, it still needs an\narchitecture that separates the mutable write path from the public read path.\nThe write path must ingest immutable raw bundles reliably; the read path must\nserve fast public exploration without forcing the browser to query arbitrary\nuploads directly.\n\nThis item defines the service boundary, storage tiers, async ingest model,\nrebuild strategy, and the derived public read models consumed by the results\nexplorer.\n\n**Phase assignment:** This TODO covers the full hosted ingest architecture\n(Phase 3). The Phase 1 static build pipeline (bundles \u2192 manifests \u2192 DuckDB\nsnapshot) is simpler and is tracked by dedicated Phase 1 implementation TODOs.\nThis item remains relevant for Phase 3 API design but is NOT a blocker for\nPhase 1 or Phase 2.", "id": "design-results-ingest-storage-and-derived-read-model", "preserves": ["Raw schema-v2 bundles and companion files remain immutable source artifacts", "The public explorer can be rebuilt from stored source artifacts plus durable metadata"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}], "state": "done", "title": "Design results ingest, storage, and derived read-model architecture", "verifications": [{"command": "test -f _project/analysis/ingest-architecture-design.md && echo PRESENT", "description": "Architecture design doc committed to _project/analysis/", "expected": "PRESENT", "item_id": "design-results-ingest-storage-and-derived-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -c 'submission API\\|storage model\\|metadata schema\\|read model\\|deployment\\|backfill\\|failure' _project/analysis/ingest-architecture-design.md", "description": "Design doc addresses all required topic areas", "expected": "7", "item_id": "design-results-ingest-storage-and-derived-read-model", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": [], "notes": "Output: 1-page reference matrix documenting each platform's bundle format,\nstorage tier, idempotency model, and public read surface. Use as a\ngrounding input for w2 and w3 design choices.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Catalog 3 reference ingest architectures - survey Geekbench, OpenBenchmarking, and CloudSpecs submission \u2192 storage \u2192 projection pipelines", "wid": "w1"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w1"], "notes": "Define the hosted submission interface: what the CLI POSTs, how the service\nacknowledges receipt, and what guarantees are given on idempotent re-submit.\nOutput: submission-api.md with endpoint sketch and idempotency rules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write submission API surface proposal: endpoints, idempotency rules, and bundle upload contract", "wid": "w2"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w1"], "notes": "Define where and how raw schema-v2 bundles are stored after receipt. Key\ndecisions: key naming (must survive rename/rebrand), retention policy,\nintegrity verification (hash check on ingest), and access control\nboundaries. Output: storage-model.md with object key scheme and retention\nrules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write raw bundle storage model: object key scheme, retention tiers, and integrity properties", "wid": "w3"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w2", "w3"], "notes": "Capture the normalized data layer that separates the write path from the\npublic read path. Include submission-level fields (actor, timestamp, trust\ntier, visibility) and how they map to cohort membership and compare views.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define durable metadata schema for runs, submissions, actors, trust, and cohorts", "wid": "w4"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w4"], "notes": "Include static manifests, leaderboard projections, and whether a Parquet or\nDuckDB snapshot is worth shipping for browser-side analysis. Document the\nrebuild trigger and incremental update strategy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define derived public read models: manifests, projections, and DuckDB-vs-Parquet tradeoffs", "wid": "w5"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w2", "w3", "w5"], "notes": "Document what runs as a long-lived service vs. a triggered worker vs. a\nstatic asset. Include deployment topology (e.g., Fly.io + R2 or equivalent)\nand cost model at low submission volume.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose deployment boundaries: API service, async workers, object storage, and static explorer assets", "wid": "w6"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w4", "w5", "w6"], "notes": "How does the platform recover from a corrupted read model, a failed\nsubmission, or a schema migration? Define rebuild triggers, retention\nwindows, and failure escalation paths.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define backfill, rebuild, retention, and failure-recovery procedures", "wid": "w7"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w2", "w3", "w4", "w5", "w6", "w7"], "notes": "Consolidate w1-w7 outputs into a single DESIGN.md under _project/analysis/.\nThis doc is the authoritative input for implementation TODOs that follow.\nPhase 3 cannot start without this review passing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write architecture design doc and get peer review before Phase 3 begins", "wid": "w8"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep them as four named acceptance criteria so reviewers can trace each one back to its source.", "dont": "collapse the four guardrails into a single bullet on Step 6", "why": "each one cites a specific blind-spot and a specific failure mode"}, {"INSTEAD": "wait for the verdict.", "dont": "pre-author a steward-queue implementation TODO 'just in case'", "why": "that primes the Step 5 decision and violates the Step 5 anti-pattern list"}, {"INSTEAD": "leave todo_id on this TODO until the queue spec demonstrably owns them.", "dont": "re-point the blind-spots' todo_id at dev-loop-step-6-queue-decision-gate before w2 has actually folded the guardrails into it", "why": "Step 6 may resolve as DO NOT BUILD and never carry them"}], "approach": "This is a watcher-and-handoff TODO, not an implementation TODO. The\nonly state changes happen at two well-defined trigger points:\n 1. Step 5 publishes its decision record (w1 branches).\n 2. If BUILD QUEUE: edit Step 6 (w2) and promote the blind-spots\n (w3) in a single PR.\nKeep the PR scoped to the four files: Step 6 YAML and the four\nblind-spot files. Do not touch Step 5, the dev-loop infrastructure,\nor any other architecture in the same change.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Process and Workflow", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-08T00:00:00Z", "deferrals": [], "deps": ["dev-loop-step-5-measurement-window"], "description": "The 2026-04-30 L2 merge-loop audit produced four blind-spot findings\nthat all gate any future steward-queue implementation. They were\npromoted to `merged-to-todo` -> this TODO on 2026-05-11, so the\nblind-spot log records that they are integrated into a planned item\nrather than sitting in the `actionable` re-triage queue:\n\n - 2026-04-30-114721-global-test-lock-cpu-protection \u2014 keep the\n global lock default until measurement shows concurrent xdist\n runs are CPU-safe.\n - 2026-04-30-114722-queue-single-point-failure \u2014 require explicit\n failure-mode analysis before adding a steward queue; prefer\n native GitHub features if they appear.\n - 2026-04-30-114723-queue-dwell-contract \u2014 treat dwell-time\n targets as part of any queue acceptance criteria; update agent\n handoff docs if dwell becomes materially longer than CI.\n - 2026-04-30-114724-post-merge-safety-tension \u2014 define the exact\n failure that develop-post-merge.yml handles; do not add a queue\n unless Step 5 metrics show a pre-merge ordering problem.\n\nAll four are blocked on the same gate:\n`dev-loop-step-5-measurement-window` \u2014 a 30-day measurement window\nthat started 2026-04-30 and closes ~2026-05-30. Until Step 5\npublishes its decision record, none of these findings are\nindividually actionable, which is exactly why they belong to one\nhandoff TODO instead of four `actionable` log entries.\n\nThis TODO does NOT pre-build the queue (that anti-pattern is\nexplicit in Step 5's `do_not_modify` list). It exists to ensure\nthat, IF Step 5 returns BUILD QUEUE, Step 6 inherits these four\nfindings as named acceptance criteria \u2014 not as four ad-hoc reviewer\ncomments that get lost in the queue spec.\n\nIf Step 5 returns DO NOT BUILD or REASSESS, this TODO closes as\nsuperseded with a one-line note pointing to the Step 5 decision\nrecord; the four blind-spots stay `merged-to-todo` pointing here\n(this TODO's DONE record carries the outcome) \u2014 no further\nstatus flip is needed.\n", "id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "preserves": ["All four blind-spots' historical record \u2014 the only mutations are the `status`/`todo_id` frontmatter line and append-only triage-log entries; the `## Finding` / `## Why this matters` body text is immutable.", "The 'do not pre-build the queue during measurement' anti-pattern in Step 5.", "dev-loop-step-5-measurement-window's calendar-gated Blocked status \u2014 this TODO must not push Step 5 to publish early."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml"}, {"kind": "do_not_modify", "path_glob": "tests/conftest.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114721-global-test-lock-cpu-protection.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114722-queue-single-point-failure.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114723-queue-dwell-contract.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114724-post-merge-safety-tension.md"}], "state": "active", "title": "Pre-register four 2026-04-30 blind-spots as Step 6 queue acceptance criteria", "verifications": [{"command": "ls _project/decisions/dev-loop-measurement-*.md", "description": "Step 5 decision record exists before this TODO begins w2", "expected": "at least one file", "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -E '2026-04-30-114(721|722|723|724)-' _project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml", "description": "If BUILD QUEUE: Step 6 YAML lists the four guardrails by blind-spot ID", "expected": "four matching lines, one per blind-spot id", "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "last_result": null, "last_run": null, "seq": 2}, {"command": "for id in 2026-04-30-114721-global-test-lock-cpu-protection 2026-04-30-114722-queue-single-point-failure 2026-04-30-114723-queue-dwell-contract 2026-04-30-114724-post-merge-safety-tension; do uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py show $id | grep -E 'status:|todo_id:'; done", "description": "Four blind-spots show status: merged-to-todo with todo_id populated", "expected": "all four show status: merged-to-todo and a non-null todo_id", "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "needs": [], "notes": "Trigger: `dev-loop-step-5-measurement-window` publishes\n`_project/decisions/dev-loop-measurement-.md` with a\nBUILD QUEUE / DO NOT BUILD / REASSESS verdict.\n - BUILD QUEUE -> proceed to w2.\n - DO NOT BUILD -> close this TODO as completed-via-supersede\n with a one-line note citing the Step 5 decision SHA. The\n four blind-spots stay `merged-to-todo` -> this TODO; the\n DONE record is their disposition. (Optionally append a\n one-line triage entry to each pointing at the decision SHA.)\n - REASSESS -> stay Blocked; let Step 5's REASSESS path drive\n the next iteration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Watch Step 5 decision record; branch on outcome", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "needs": ["w1"], "notes": "Edit `_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml`\n(only after Step 5 unblocks it) so its `success_metrics` /\n`must_preserve` explicitly include:\n - \"CPU-protection default preserved: queue must not silently\n flip the global test-lock default before concurrent-xdist\n measurement clears it as safe.\"\n Cites: 2026-04-30-114721-global-test-lock-cpu-protection.\n - \"Failure-mode analysis is a prerequisite: a written analysis\n of steward-queue failure modes (single-point-of-failure,\n recovery, fallback commands) lands before the steward\n implementation TODO is authored.\"\n Cites: 2026-04-30-114722-queue-single-point-failure.\n - \"Dwell-time SLO is a queue acceptance criterion: queue spec\n names a target P50/P95 dwell-time and a triggering\n measurement source; agent handoff docs update if dwell\n becomes materially longer than current CI.\"\n Cites: 2026-04-30-114723-queue-dwell-contract.\n - \"Post-merge safety tension is resolved before queue lands:\n queue spec states explicitly which failures\n develop-post-merge.yml handles vs which the queue handles,\n and does not duplicate safety machinery without measurable\n recovery improvement.\"\n Cites: 2026-04-30-114724-post-merge-safety-tension.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Fold the four guardrails into Step 6's acceptance criteria", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "needs": ["w2"], "notes": "The four blind-spots are already `merged-to-todo` ->\n`dev-loop-queue-acceptance-criteria-from-blind-spots` (promoted\n2026-05-11). On the BUILD QUEUE path, once w2 has folded the\nguardrails into Step 6, re-run the promote action so the link\npoints at the queue spec that now owns them:\n `uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py triage --action promote --todo-id dev-loop-step-6-queue-decision-gate`\nThe triage-log entry should cite Step 5's decision SHA and name\nthis TODO as the prior merge path. On any non-BUILD path this\nwork unit is a no-op \u2014 leave todo_id pointing here.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-point the four blind-spots' todo_id to Step 6 (BUILD QUEUE path only)", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "wait for the GO signal in the decision record", "dont": "begin Step 1-4 work in parallel with this TODO", "why": "the verification may invalidate downstream plans"}, {"INSTEAD": "paste verbatim JSON", "dont": "paraphrase API responses in the decision record", "why": "future re-plans need raw values"}, {"INSTEAD": "checking only one surface gives a false answer", "dont": "skip w2 (rulesets)", "why": "GitHub repos commonly encode protection in rulesets rather than classic branch protection"}], "approach": "Pure read-only investigation. Run `gh api` calls in sequence,\npaste raw JSON into the decision record, then write conclusions.\nDo not modify branch protection, rulesets, or any workflow during\nthis TODO. The single artifact is a markdown file documenting\nwhat is true today.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-0-verify-assumptions", "id": 375, "reason": "Each downstream step is a separate TODO with its own scope and verification", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Refers to dev-loop Steps 1-4 implementation, already superseded \u2014 project has progressed to Steps 5/6, both already open tracked items.", "summary": "Acting on the findings (Steps 1-4 implementation)"}], "deps": [], "description": "Decision-gate item that precedes Steps 1-4 of the dev-loop simplification\nplan. The plan rests on three load-bearing assumptions: (a) \"Require\nbranches to be up to date before merging\" is enabled on `develop`,\n(b) PR-tier CI on `develop` is heavier than necessary, (c) native\nGitHub Merge Queue is unavailable on this user-owned repo.\n\nThis TODO produces a written decision record. If any assumption is\nwrong, downstream TODOs (Step 1-4) must be re-planned BEFORE shipping.\n\nNo commits are produced by this TODO except the decision record itself.\n\nContext (for future readers): see `_project/blind-spots/` for the L2\naudit findings that motivated the dev-loop simplification, captured\nin Step 1.", "id": "dev-loop-step-0-verify-assumptions", "preserves": ["All API outputs recorded verbatim, not paraphrased \u2014 future agents may need the raw values", "No code or workflow changes during this step \u2014 verification only"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml (audited but not edited)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": "CLAUDE.md, AGENTS.md (no edits \u2014 Steps 1-4 update these)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 0 reads via gh api; never PATCH)"}, {"kind": "do_not_modify", "path_glob": "Makefile (no edits \u2014 read-only investigation)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-verification-2026-04-30.md (new file, the decision record)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 0: verify assumptions before changing anything", "verifications": [{"command": "test -f _project/decisions/dev-loop-verification-2026-04-30.md && grep -cE '^## (W[1-6]|Conclusion)' _project/decisions/dev-loop-verification-2026-04-30.md", "description": "Decision record exists and contains all six findings", "expected": ">= 7", "item_id": "dev-loop-step-0-verify-assumptions", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE 'Step [1-6]:' _project/decisions/dev-loop-verification-2026-04-30.md", "description": "Decision record explicitly addresses each downstream step", "expected": ">= 4", "item_id": "dev-loop-step-0-verify-assumptions", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Run: `gh api repos/joeharris76/BenchBox/branches/develop/protection\n--jq '.required_status_checks'` and record the value of `strict`.\nIf strict is already false, the staleness diagnosis is wrong and\nStep 3.4 (disable strict-base) is a no-op. Re-plan before Step 3.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Check classic branch protection on develop for `strict: true`", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Required checks may be encoded in a ruleset, not in classic\nbranch protection. Run: `gh api repos/joeharris76/BenchBox/rulesets`\nand inspect each ruleset that targets `develop`. Record:\n - Ruleset names targeting develop\n - Required status checks per ruleset\n - Whether `require_branches_up_to_date` is set in any of them\nThe Step 3.4 toggle must be applied wherever it's encoded.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Check GitHub rulesets targeting develop for required-checks and strict-base", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Read `.github/workflows/test.yml` `on:` block; identify which jobs\nrun on `pull_request` to develop. Cross-reference with recent run\ndurations: `gh run list --workflow test.yml --branch develop --limit 20\n--json conclusion,startedAt,updatedAt,name`. Record P50/P95 CI\nduration on develop PRs today. This is the baseline that Step 3.1\n(cut to lint + Ubuntu 3.12 fast tests) reduces.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit current PR-tier CI cost on develop", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": ["w3"], "notes": "Run: `gh run list --workflow develop-post-merge.yml --branch develop\n--limit 30 --json conclusion`. Record current red/green ratio. If\nalready noisy, Step 4 needs to handle false positives carefully.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit develop-post-merge.yml health", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Use a concrete proxy rather than free-text grep. For the last 50\nmerged PRs targeting develop, compute the gap between the last\nsuccessful CI run start and the PR merge time. A gap > 30 minutes\nwith intervening pushes is a strong signal of a strict-base re-CI\ncycle (the PR was green, develop advanced, CI re-ran):\n - `gh pr list --state merged --base develop --limit 50 \\\n --json number,mergedAt,headRefOid,labels,statusCheckRollup`\n - For each PR, compute `merged_at - last_successful_check_started_at`\n across `statusCheckRollup` entries\n - Count PRs where that gap > 30 min AND statusCheckRollup has\n >= 2 distinct startedAt timestamps for the same check name\n (proxy for re-CI after a base push)\nSecondary signal (lower confidence): run\n`gh pr list --state merged --base develop --limit 50 --json number,comments`\nand grep PR comments for explicit `pr-refresh`, `make pr-refresh`,\nor `merged develop into branch`. Use only as corroboration.\n\nRecord:\n - Strict-base re-CI rate (primary proxy, %)\n - Explicit pr-refresh mentions (secondary, count)\nIf the primary proxy is < 5%, the queue is overkill even if Step 4\nmetrics later say otherwise \u2014 Step 6 should reflect that.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Estimate actual pr-refresh / staleness pain rate", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Run: `gh api repos/joeharris76/BenchBox --jq '{visibility,\nowner_type: .owner.type, owner_login: .owner.login,\nfull_name, default_branch, private}'`. Also inspect the branch\nprotection / ruleset surfaces from w1/w2 for any native merge-queue\nrule. Do NOT infer merge-queue availability from unrelated repo\nfields such as `.has_issues`; that field only says whether Issues\nare enabled.\n\nNative MQ generally requires org-owned repos (with plan-dependent\navailability for private repos). User-owned repos should be treated\nas \"native MQ unavailable\" unless GitHub's protection/ruleset API\nshows merge-queue rules can be applied. Record the evidence so Step 6\npicks the right path (native vs steward) without re-investigation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm native GitHub Merge Queue availability", "wid": "w6"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": "Single markdown file with: timestamp, raw outputs of w1-w6,\nconclusions, and explicit GO/NO-GO/RE-PLAN signal for each of\nSteps 1-4. Future agents read this before starting any downstream\nstep.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write decision record at _project/decisions/dev-loop-verification-2026-04-30.md", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep global default and offer override only", "dont": "localize the test lock to per-worktree by default", "why": "10 concurrent xdist runs (auto-workers) would exhaust CPU and produce timing-flake"}, {"INSTEAD": "that target stays useful until then", "dont": "remove pr-refresh target in this PR", "why": "branch protection still requires up-to-date until Step 3 lands"}, {"INSTEAD": "they are 6 small files", "dont": "skip the blind-spot capture (w5)", "why": "the protocol explicitly requires file-first capture and this is the natural home"}], "approach": "Order the work units to minimize risk: hooks first (w1) is the\nload-bearing change; fanout (w2) depends on hooks being opt-in;\nlock (w3) is independent but read tests/README.md:257 first to\nunderstand the existing rationale; docs (w4) reflect the new\nreality; blind-spots (w5) are the protocol-debt cleanup; PR (w6)\nships everything as one cohesive unit.\n\nFor w1, prefer wrapping hook commands in `bash -c` with the env-var\nguard \u2014 it's smaller than rewriting hook definitions and easier to\nrevert if something breaks.\n\nFor w2, the `xargs -P` approach is simpler than GNU parallel\n(parallel may not be installed). Keep the per-worktree output\nprefixed with the worktree name so logs remain readable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [], "deps": ["dev-loop-step-0-verify-assumptions"], "description": "First implementation step of the dev-loop simplification plan. Removes\nlocal-loop friction so agents can push and run `pr-fanout` without\nhook serialization, while preserving the global test lock that protects\nthe workstation from concurrent xdist storms.\n\nThis step also captures the consolidated L2 blind-spot findings from\nthe four-round merge-loop audit to `_project/blind-spots/`, satisfying\nthe protocol-debt accumulated during the planning rounds.\n\nSingle PR via `make pr-open`. Independently shippable; no dependency\non Steps 2-4 landing first.", "id": "dev-loop-step-1-local-loop-cleanup", "preserves": ["Existing pr-fanout output structure (one section per worktree) \u2014 only parallelism changes, not the report shape", "Existing test runtime when BENCHBOX_PREPUSH is set \u2014 no functional change to the hook itself", "Global test lock as the DEFAULT \u2014 agents that don't set the env var continue to be CPU-protected", "make pr-preflight as a working manual command (it is the human-facing escape hatch)"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 3 manual deployment)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-refresh target (Step 3 deprecates this; here it stays)"}, {"kind": "do_not_modify", "path_glob": "Makefile worktree-add, worktree-prune, worktree-pool-* targets (Step 2 owns these)"}, {"kind": "only_modify", "path_glob": ".pre-commit-config.yaml"}, {"kind": "only_modify", "path_glob": "AGENTS.md (manual-coordination and pre-push hook paragraphs)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (manual-coordination and pre-push hook paragraphs)"}, {"kind": "only_modify", "path_glob": "Makefile (pr-fanout target only)"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-*.md (6 new files)"}, {"kind": "only_modify", "path_glob": "tests/README.md (around line 257)"}, {"kind": "only_modify", "path_glob": "tests/conftest.py (lock-path logic only)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 1: opt-in pre-push, bounded fanout, conservative test lock", "verifications": [{"command": "BENCHBOX_PREPUSH= uv run -- pre-commit run --hook-stage pre-push --all-files 2>&1 | grep -cE '(Skipped|Passed)'", "description": "All pre-push hooks no-op when BENCHBOX_PREPUSH is unset", "expected": "matches the count of pre-push hooks in .pre-commit-config.yaml", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": "BENCHBOX_PREPUSH=1 uv run -- pre-commit run pr-preflight-fast-tests --hook-stage pre-push --all-files 2>&1 | tail -3", "description": "Pre-push hooks execute when BENCHBOX_PREPUSH=1", "expected": "hook runs (success or expected failure on uncommitted state); does not skip", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -E '^PR_FANOUT_JOBS \\?= ' Makefile", "description": "pr-fanout default parallelism is 4", "expected": "PR_FANOUT_JOBS ?= 4", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 3}, {"command": "BENCHBOX_TEST_LOCK_DIR=/tmp/bb-lock-test uv run -- python -m pytest -m fast -q tests/unit/ -x 2>&1 | tail -5 && ls /tmp/bb-lock-test/test.lock", "description": "Test lock honors BENCHBOX_TEST_LOCK_DIR override", "expected": "lock file present at /tmp/bb-lock-test/test.lock; tests run normally", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -c \"from pathlib import Path; from tests.conftest import _get_test_lock_path; print(_get_test_lock_path() == Path.home() / '.benchbox' / 'test.lock')\"", "description": "Default test lock still uses ~/.benchbox/", "expected": "True", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -cE '(Drain one at a time|One PR per file area|Sequential by design|pre-push hook.*runs the fast lane|re-runs the fast lane automatically)' CLAUDE.md AGENTS.md", "description": "Stale guidance removed from agent-facing docs", "expected": "0", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 6}, {"command": "grep -cE 'BENCHBOX_PREPUSH=1|pre-push hooks.*opt-in|make pr-preflight.*explicit|explicit.*make pr-preflight' CLAUDE.md AGENTS.md", "description": "Agent docs say pre-push fast lane is opt-in and pr-preflight is explicit", "expected": ">= 2", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 7}, {"command": "ls _project/blind-spots/2026-04-30-*.md | wc -l", "description": "Six blind-spot findings recorded", "expected": "6", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 8}], "work": [{"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Audit `.pre-commit-config.yaml` for every hook with `stages:\n[pre-push]`. Currently includes at minimum:\n - `pr-preflight-fast-tests`\n - `timing-policy-fast-lane`\nWrap each entry's command in a guard:\n `if [ \"$BENCHBOX_PREPUSH\" = \"1\" ]; then ; else exit 0; fi`\nOr use the `args` / `entry` pattern that pre-commit supports.\nDefault behavior: hooks no-op unless env var is set. `make\npr-preflight` remains the explicit human-facing invocation\n(does not depend on the env var).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate all expensive pre-push hooks behind BENCHBOX_PREPUSH=1", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": ["w1"], "notes": "Change `pr-fanout` target in `Makefile` from sequential to parallel.\nAdd `PR_FANOUT_JOBS ?= 4` near the top. Use `xargs -P \"$(PR_FANOUT_JOBS)\"`\nor GNU `parallel -j \"$(PR_FANOUT_JOBS)\"` to fan out\n`make pr-open` calls across worktrees. Drop the\n\"Sequential by design\" comment block. Document the env var in\nthe help target.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make pr-fanout bounded-parallel with PR_FANOUT_JOBS=4 default", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Read `tests/conftest.py` and locate the lock acquisition logic\n(currently `~/.benchbox/test.lock`). Do NOT change the default \u2014\na global lock is what prevents 10 concurrent xdist runs from\nsaturating the workstation. Add an env-var-aware helper, e.g.\n`_get_test_lock_path()`, that returns:\n `Path(os.environ.get(\"BENCHBOX_TEST_LOCK_DIR\",\n Path.home() / \".benchbox\")) / \"test.lock\"`\nUpdate `Makefile` `test-unlock` target to honor the same env var.\nUpdate `tests/README.md:257` area to document the override and\nexplain WHY the default stays global.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep global test lock; add BENCHBOX_TEST_LOCK_DIR override", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Remove from both files:\n - \"Drain one at a time, oldest first\" (and surrounding pr-refresh prose)\n - \"One PR per file area at a time\"\n - \"Sequential by design\" notes around pr-fanout\n - Any claim that the pre-push hook reruns the fast lane\n automatically on every push\nReplace with a one-paragraph summary of the new model: pool\nworktrees + auto-merge + post-merge auto-revert as safety net.\nUpdate the pre-push-hook guidance to say expensive pre-push hooks\nare opt-in via `BENCHBOX_PREPUSH=1`; `make pr-preflight` is the\nexplicit local gate for humans/agents that want the fast lane before\npushing.\nNote that this paragraph will be expanded by Steps 2 and 4 once\nthose land; here, just remove the now-incorrect guidance.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Strip manual-coordination guidance from CLAUDE.md and AGENTS.md", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Six findings from the four-round merge-loop audit (2026-04-30 session):\n 1. Original plan bundled \"stop babysitting PRs\" with \"promote\n develop to release-grade gating\" \u2014 separable decisions\n 2. PR CI on develop is already heavy today; reducing it is the\n highest-leverage standalone move\n 3. Test-lock localization is not a pure win at scale; the global\n lock has a real CPU-protection job (10 worktrees \u00d7 xdist auto)\n 4. A queue creates a single point of failure with no analog in\n current setup\n 5. Dwell time silently changes the agent contract (5 min today\n vs hours under a queue)\n 6. develop-post-merge.yml is the actual safety net; queue plan\n kept it as belt-and-suspenders without resolving the tension\nOne file per finding under `_project/blind-spots/2026-04-30-HHMMSS-.md`\nusing the frontmatter schema from `_project/blind-spots/README.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture consolidated L2 blind-spot findings to _project/blind-spots/", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Run `make lint format typecheck`, then\n`BENCHBOX_PREPUSH=1 make pr-preflight` to confirm the opt-in path\nstill works. Then `make pr-open`. Do NOT enable the new env var\nglobally; the point is that absence-of-var is the new default.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run validation locally and open the PR", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "surface as 'stale' in status, require explicit `worktree-pool-reset` to free", "dont": "auto-reap stale claims", "why": "an agent crash mid-session may leave valuable uncommitted work"}, {"INSTEAD": "use detached origin/develop as the Free state", "dont": "check out the local develop branch in pool worktrees", "why": "the main clone owns develop and Git forbids the same branch in multiple worktrees"}, {"INSTEAD": "prune is for legacy non-pool worktrees only", "dont": "delete pool-NN worktrees in worktree-prune", "why": "the pool is the contract"}, {"INSTEAD": "read POOL_SIZE ?= 10 from the environment", "dont": "hardcode pool size to 10", "why": "the workstation may want a different cap"}, {"INSTEAD": "deprecate-with-notice now, remove next release", "dont": "remove make worktree-add in this PR", "why": "in-flight work in non-pool worktrees needs continuity"}, {"INSTEAD": "atomic selection is required", "dont": "skip portable locking on claim selection", "why": "two concurrent agents racing to claim would both pick pool-01 and one would clobber the other's branch"}], "approach": "Build the targets in dependency order: pool-init (w1) is foundational,\nthen claim (w2) and release (w3) are the core lifecycle, then status\n(w4) and reset (w5) are operational tooling, then docs (w6) and\ndeprecation (w7-w8) update the rest of the surface.\n\nUse a portable lock wrapper for atomic claim. `flock(1)` is not present\non the macOS workstation, so the Makefile cannot assume it. Prefer\n`lockf` on macOS, `flock` on Linux when available, or a small Python\nhelper that calls `fcntl.flock`. Lock file stays at\n`/.git/pool.lock`. Single shared lock across all pool\nworktrees \u2014 claim is fast (seconds) so contention is fine.\n\nFor state detection in worktree-pool-status:\n - free: `git symbolic-ref -q --short HEAD` is empty AND\n `git rev-parse HEAD` == `git rev-parse origin/develop` AND\n `git status --porcelain` is empty\n - claimed: not on develop, working tree may or may not be clean\n - stale: claimed AND `gh pr view --json state` returns MERGED\n - dirty: claimed AND `git status --porcelain` non-empty\n\nRefer to existing `make worktree-add` and `make worktree-prune`\ntargets for patterns; reuse their git-worktree command structure.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-2-worktree-pool", "id": 376, "reason": "Reasonable enhancement but out of scope; claim already updates at claim-time. Revisit if pool-status shows worktrees consistently lagging develop.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Auto-update-while-idle was conditioned on pool-status showing worktrees lagging develop; no evidence that trigger occurred; claim-time update already mitigates.", "summary": "Auto-update of pool worktrees while idle (background pull of develop)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-2-worktree-pool", "id": 377, "reason": "Pool size of 10 + global test lock is sufficient for now. Add only if metrics in Step 5 show resource-contention failures.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Gated on resource-contention data from Step 5 (open, active); measurement window hasn't concluded so trigger unmet. (tracked by dev-loop-step-5-measurement-window)", "summary": "Per-worktree resource quotas (CPU/memory limits)"}], "deps": ["dev-loop-step-0-verify-assumptions"], "description": "Replace the create-and-destroy worktree model with a retained pool of\nworktrees. Default pool size is 10 (`POOL_SIZE ?= 10`), but the targets\nmust keep the size configurable so the workstation can be tuned without\nediting Makefile logic. New agent sessions claim an available worktree,\nupdate it from current `develop`, work there until merge, then release\nit back to the pool.\n\nThis formalizes how concurrent agents share the workstation. The default\nsize of 10 bounds routine parallelism (which is also why Step 1 keeps\nthe test lock global \u2014 even with the pool bounded, 10 concurrent xdist\nruns would saturate the machine).\n\nSingle PR via `make pr-open`.\n\n## Post-merge runbook (not part of the PR diff)\n\nAfter this PR merges to develop, perform on Joe's workstation:\n\n make worktree-pool-init\n\nThen verify:\n - 10 worktrees present at `BenchBox.pool-01..10`\n - Each is detached at `origin/develop` and clean\n - Each has `.venv/` populated\n - `make worktree-pool-status` shows all 10 free\n - Each free worktree is detached and clean, NOT on the local\n `develop` branch; claim refreshes free slots to current\n `origin/develop`\n\nTracked as a follow-up checklist item in the PR description, not as a\nwork unit, because the work unit DAG is for code changes that can be\nmarked done by `todo-cli done ` against a commit.", "id": "dev-loop-step-2-worktree-pool", "preserves": ["Atomic claim semantics \u2014 two concurrent `make worktree-claim` calls must always pick different pool-NN worktrees, never collide", "Existing make worktree-add target keeps working \u2014 only adds deprecation notice; do not break legacy non-pool worktree workflows mid-session", "Pool worktrees retain their .venv/ across release/claim cycles \u2014 uv sync only runs at claim time after a fetch, and only if pyproject.toml/uv.lock changed", "make worktree-pool-init is idempotent \u2014 re-running must not destroy or reset existing pool worktrees"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": ".pre-commit-config.yaml (Step 1 owns pre-push gating)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 3 manual deployment)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-* targets (Steps 1 and 3 own these)"}, {"kind": "do_not_modify", "path_glob": "tests/conftest.py (Step 1 owns the test-lock helper)"}, {"kind": "only_modify", "path_glob": ".gitignore (add .git/pool.lock if needed)"}, {"kind": "only_modify", "path_glob": "AGENTS.md (Session start / Worktree section)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (Session start, Worktrees, and Pre-approved Commands sections)"}, {"kind": "only_modify", "path_glob": "Makefile (worktree-* targets only)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 2: fixed pool of 10 retained worktrees with claim/release", "verifications": [{"command": "tmp=$(mktemp -d); WORKTREE_POOL_PARENT=$tmp POOL_SIZE=2 make worktree-pool-init && find \"$tmp\" -maxdepth 1 -type d -name 'BenchBox.pool-*' | wc -l && WORKTREE_POOL_PARENT=$tmp POOL_SIZE=2 make worktree-pool-init; for wt in \"$tmp\"/BenchBox.pool-*; do git worktree remove --force \"$wt\" >/dev/null 2>&1 || true; done; rm -rf \"$tmp\"", "description": "make worktree-pool-init creates 10 worktrees idempotently", "expected": "2 (first run) and second run completes without error; disposable worktrees are removed", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 1}, {"command": "make worktree-claim BRANCH=bad-name 2>&1 | grep -c 'must match'", "description": "worktree-claim refuses invalid branch names", "expected": ">= 1", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 2}, {"command": "tmp=$(mktemp -d); WORKTREE_POOL_PARENT=$tmp POOL_SIZE=1 make worktree-pool-init >/dev/null && WORKTREE_POOL_PARENT=$tmp POOL_SIZE=1 make worktree-claim BRANCH=chore/test-pool-claim 2>&1 | tee /tmp/bb-pool-claim.out | grep -E 'WORKTREE_PATH=.*pool-[0-9]+'; wt=$(sed -n 's/^WORKTREE_PATH=//p' /tmp/bb-pool-claim.out); git worktree remove --force \"$wt\" >/dev/null 2>&1 || true; git branch -D chore/test-pool-claim >/dev/null 2>&1 || true; rm -rf \"$tmp\" /tmp/bb-pool-claim.out", "description": "worktree-claim selects an available pool worktree", "expected": "WORKTREE_PATH=", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 3}, {"command": "make worktree-pool-status 2>&1 | grep -cE '^pool-[0-9]+'", "description": "worktree-pool-status reports all pool worktrees", "expected": "10", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 4}, {"command": "tmp=$(mktemp -d); WORKTREE_POOL_PARENT=$tmp POOL_SIZE=3 make worktree-pool-init >/dev/null; for i in 1 2 3; do (WORKTREE_POOL_PARENT=$tmp POOL_SIZE=3 make worktree-claim BRANCH=chore/race-test-$i >\"$tmp/out-$i\" 2>&1 &); done; wait; cat \"$tmp\"/out-* | grep -E '^WORKTREE_PATH=' | sort -u | wc -l; for wt in \"$tmp\"/BenchBox.pool-*; do git worktree remove --force \"$wt\" >/dev/null 2>&1 || true; done; for i in 1 2 3; do git branch -D chore/race-test-$i >/dev/null 2>&1 || true; done; rm -rf \"$tmp\"", "description": "Concurrent claims do not race", "expected": "3 distinct claims, each on a different pool-NN worktree", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 5}, {"command": "make worktree-prune 2>&1 | grep -c 'Skipping pool worktree'", "description": "worktree-prune skips pool worktrees", "expected": ">= 0 (skips logged for any pool worktrees on develop)", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 6}, {"command": "make worktree-add BRANCH=chore/legacy-test 2>&1 | grep -c 'DEPRECATED'", "description": "worktree-add prints deprecation notice", "expected": ">= 1", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 7}, {"command": "grep -cE 'worktree-claim|worktree-release|worktree-pool-status|worktree-pool-init|worktree-pool-reset' CLAUDE.md", "description": "Claude command permissions list new pool commands explicitly", "expected": ">= 5", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 8}, {"command": "grep -cE 'worktree-claim|worktree-release|worktree-pool-status|worktree-pool-reset' CLAUDE.md AGENTS.md", "description": "Agent-facing docs describe claim/release pool lifecycle", "expected": ">= 8", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 9}], "work": [{"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": [], "notes": "One-time bootstrap. Add `POOL_SIZE ?= 10` and\n`WORKTREE_POOL_PARENT ?= ..` so verification can create disposable\npools outside the normal sibling paths. With defaults, creates\n`BenchBox.pool-01` through `BenchBox.pool-10` as siblings of the\nmain clone, each in detached HEAD at `origin/develop`, each with\n`uv sync --group dev` run, each with `pre-commit install`.\nIdempotent: if a pool-NN worktree already exists, leave it alone\n(do not touch its branch state).\nPrint summary table at end: pool-NN | path | branch.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-pool-init target \u2014 creates pool-01..POOL_SIZE from develop", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Selects the first **Free** pool worktree (detached HEAD, working\ntree clean, no local feature branch checked out). Claim fetches\nand resets the selected worktree to current `origin/develop`\nbefore creating the branch. Pool worktrees must not check out the\nlocal `develop` branch because the main clone keeps `develop`\nchecked out and Git does not allow the same branch in multiple\nworktrees.\nSteps:\n 1. Acquire a portable file lock on `/.git/pool.lock`\n (timeout 30s). Use `lockf` on macOS, `flock` on Linux if\n available, or a tiny Python helper using `fcntl.flock` so the\n target works on Joe's macOS workstation and CI.\n 2. Iterate pool-01..POOL_SIZE; first match is \"Free\"\n 3. In that worktree: `git fetch origin`, `git reset --hard\n origin/develop`, `git checkout -b $BRANCH`, `uv sync --group dev`\n 4. Release flock\n 5. Print: `WORKTREE_PATH=`\nRefuses (exit 1) if pool is fully claimed; suggests\n`make worktree-pool-status` to investigate.\nRefuses if BRANCH is empty or doesn't match `^(chore|fix|feat|docs)/.+`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-claim BRANCH= \u2014 atomic claim under portable lock", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Run from inside a pool worktree (refuses if cwd is not a pool-NN\npath). Steps:\n 1. Identify current branch\n 2. Verify PR for this branch is in MERGED state via\n `gh pr view --json state` (refuse if not, with explicit\n \"open or close PR first\" message; FORCE=1 escapes)\n 3. `git checkout --detach origin/develop`\n 4. `git fetch origin`\n 5. `git reset --hard origin/develop`\n 6. `git branch -D ` (the one just released)\n 7. `git remote prune origin`\nWorktree returns to **Free** state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-release \u2014 verify merged, return to develop", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Prints table with columns: pool-NN | path | branch | state |\nclaim_age. State is one of:\n - free: detached HEAD, working tree clean, no untracked files\n outside expected (`.benchbox/`, `.venv/`, etc.); claim\n refreshes to current origin/develop before checkout\n - claimed: HEAD on a non-develop branch\n - stale: claimed branch has been merged upstream (gh pr view\n returns MERGED) but worktree still on the branch\n - dirty: claimed with uncommitted changes \u2014 needs review\nclaim_age is `git log -1 --format=%ar` of the most recent commit\non the current branch (or \"\u2014\" for free).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-pool-status \u2014 tabular state listing", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Forcibly returns pool-NN to Free state. Refuses if working tree\nhas uncommitted changes UNLESS `FORCE=1` is set. Even with FORCE,\nprints what it's about to discard and requires interactive\nconfirmation (read -p). Useful when an agent died mid-session\nleaving the worktree in an inconsistent state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-pool-reset POOL=NN \u2014 manual escape hatch", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w2", "w3", "w4"], "notes": "Replace the existing \"Session start\" instructions:\n - Old: `make worktree-add BRANCH=/`\n - New: `make worktree-claim BRANCH=/`\nAdd a paragraph explaining the pool model: 10 worktrees,\nretained indefinitely, claim/release lifecycle. Document the\nStale recovery flow (run `worktree-pool-status`, then\n`worktree-pool-reset POOL=NN` if needed).\nNote that `make worktree-add` remains as a deprecated alias for\none release with a stderr deprecation notice.\n\nUpdate Claude-specific pre-approved command buckets in CLAUDE.md:\n - Read-only: `make worktree-pool-status`, `make worktree-list`,\n `git worktree list*`\n - Write, feature/pool worktrees only: `make worktree-claim\n BRANCH=...`, `make worktree-release`\n - Manual/admin escape hatches, not broad auto-allow:\n `make worktree-pool-init`, `make worktree-pool-reset POOL=NN`\nAGENTS.md should describe the same lifecycle in prose even though it\ndoes not have Claude's pre-approved-command table.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update Session-start docs in CLAUDE.md and AGENTS.md", "wid": "w6"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w2"], "notes": "`worktree-add` keeps current semantics but prints to stderr:\n\"DEPRECATED: use `make worktree-claim BRANCH=...` instead. The\npool model retains worktrees rather than creating new ones.\n`worktree-add` will be removed in the next release.\" Then\nproceeds normally. Pool-NN worktrees are never created via\nworktree-add; they only come from worktree-pool-init.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Mark worktree-add as deprecated alias of worktree-claim", "wid": "w7"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Pool worktrees are retained indefinitely; `worktree-prune` should\nno longer be a routine end-of-session command. Modify the target\nto:\n 1. Detect pool-NN paths and skip them\n 2. Print: \"Skipping pool worktree pool-NN (retained)\"\n 3. Continue pruning legacy non-pool worktrees as today\nUpdate help text to note: `worktree-prune` is for legacy\nworktrees only; pool worktrees are released via `worktree-release`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update worktree-prune to skip pool-NN worktrees", "wid": "w8"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep Step 3 scoped to non-required job cost", "dont": "block Step 3 on Step 3a's path umbrella", "why": "content-only skip is a separate follow-up"}, {"INSTEAD": "keep them in nightly/workflow_dispatch or an explicit opt-in lane", "dont": "delete heavy checks outright", "why": "moved checks still catch compatibility regressions"}, {"INSTEAD": "keep it as a rare branch-drift escape hatch", "dont": "deprecate or remove `make pr-refresh` in Step 3", "why": "the old rationale was stale strict-base churn"}, {"INSTEAD": "prefer split workflows or a small documented condition pattern", "dont": "introduce many copy-pasted branch conditions without a design note", "why": "future jobs will drift"}, {"INSTEAD": "preserve release-grade coverage there", "dont": "lighten main PR or tag-release validation", "why": "main is release-only and tags publish artifacts"}, {"INSTEAD": "record the live state and leave it unchanged", "dont": "patch develop branch protection or rulesets in Step 3", "why": "Step 0 already found strict-base false and required checks reduced"}], "approach": "Treat Step 3 as a workflow routing change. Required checks are already\nminimal, so the implementation should not touch GitHub protection/ruleset\nsettings. The highest-risk design point is check-name stability: GitHub\nrequired contexts are name-sensitive, and Step 3a may later add an umbrella\nresult check. Keep the existing required context names stable during Step 3\nunless the implementation deliberately leaves rulesets untouched and proves\nGitHub still reports the required checks.\n\nPrefer moving whole non-required lanes to a nightly/manual workflow over\nsprinkling branch conditions through every job. A split workflow makes\nownership visible: routine develop PR checks prove fast code health, while\nnightly/manual checks preserve compatibility and release confidence. If a\njob must stay in `test.yml` for status-name compatibility, gate it narrowly\nand document the reason next to the condition.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "id": 378, "reason": "Separate TODO already exists and must preserve the required contexts once Step 3's CI routing is stable.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: _project/DONE/main/active/dev-loop-step-3a-path-based-ci-skip.yaml confirms the referenced separate TODO exists and is already Completed.", "summary": "Step 3a path-aware required-check umbrella"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "id": 379, "reason": "Step 6 territory, and Step 0 found no current strict-base re-CI pain plus no exposed merge-queue rule for this user-owned repository.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Merge Queue migration is explicitly 'Step 6 territory', which is the open dev-loop-step-6-queue-decision-gate item.", "summary": "Native GitHub Merge Queue migration"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "id": 380, "reason": "Could reduce cost, but adds operational burden. Revisit only if Step 5 runner-minute metrics justify it.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Self-hosted runner adoption is explicitly gated on dev-loop Step 5 runner-minute metrics justifying it; Step 5 measurement is still in progress (dev-loop-step-5-measurement-window).", "summary": "Self-hosted runner adoption"}], "deps": ["dev-loop-step-0-verify-assumptions"], "description": "Step 0 invalidated the original strict-base diagnosis. `develop` has no\nclassic branch protection object, and the active `develop-squash-only`\nruleset already requires only `lint` and `test (ubuntu-latest, 3.12)`;\n`strict_required_status_checks_policy` is already `false`.\n\nThe remaining load-bearing Step 3 work is CI-cost reduction, not branch\nprotection. `.github/workflows/test.yml` still runs a broad non-required\nPR-tier workflow on pull requests to `develop`: a multi-version/OS test\nmatrix, security, smoke/integration lanes, table-format checks, package\ninstall validation, parity, and non-blocking PySpark. The Step 0 baseline\nmeasured recent `Tests` runs at P50 16.70 minutes and P95 19.47 minutes.\n\nReplanned Step 3 should keep the existing required contexts unchanged and\nmove or gate only non-required heavy lanes so ordinary `develop` PRs stop\nspending runner time on release-grade validation. Full validation must\nremain available through nightly and workflow_dispatch, and `main` PR/tag\nrelease coverage must remain release-grade.\n\nDesigned-together follow-up: Step 3a (`dev-loop-step-3a-path-based-ci-skip`)\nremains valid after this replan. Step 3 reduces non-required job breadth for\ncode PRs; Step 3a separately adds path-aware behavior for content-only PRs\nwhile preserving the already-minimal required contexts.", "id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "preserves": ["Full validation for main PRs and tag releases remains release-grade and must not be lightened as a side effect of reducing develop PR cost", "Heavy checks removed from routine develop PR execution remain runnable through nightly/workflow_dispatch or an explicit opt-in label", "No Step 3 branch-protection or ruleset PATCH; Step 0 already found the intended strict-base and required-check state live", "The develop required contexts remain exactly `lint` and `test (ubuntu-latest, 3.12)` unless a separate Step 3a deployment explicitly changes the ruleset required-check list", "`make pr-refresh` remains available as an escape hatch for real branch drift; Step 3 must not remove or deprecate it based on the stale strict-base premise"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/lint.yml unless w1 proves required-check naming requires it"}, {"kind": "do_not_modify", "path_glob": ".pre-commit-config.yaml (Step 1 owns pre-push gating)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (read-only verification only)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-refresh, pr-open, pr-fanout, and worktree targets"}, {"kind": "do_not_modify", "path_glob": "tests/ (no product test changes for CI routing)"}, {"kind": "only_modify", "path_glob": ".github/workflows/nightly.yml (new or updated)"}, {"kind": "only_modify", "path_glob": ".github/workflows/release.yml (audit/comment only if needed)"}, {"kind": "only_modify", "path_glob": ".github/workflows/test.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md (CI guidance only)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (CI guidance only)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml (replan only)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 3: reduce non-required develop PR CI cost", "verifications": [{"command": "repo=$(gh repo view --json nameWithOwner --jq .nameWithOwner)\nfound=0\nfor ruleset_id in $(gh api \"repos/${repo}/rulesets\" --jq '.[] | select(.target == \"branch\" and ((.conditions.ref_name.include // []) | any(. == \"refs/heads/develop\"))) | .id'); do\n found=1\n gh api \"repos/${repo}/rulesets/${ruleset_id}\" --jq '.rules[] | select(.type == \"required_status_checks\") | .parameters'\ndone\ntest \"$found\" = \"1\"\n", "description": "Develop ruleset still has strict-base off with only the two lightweight required contexts", "expected": "strict_required_status_checks_policy is false; required_status_checks are the lightweight develop PR contexts", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 1}, {"command": "gh api repos/joeharris76/BenchBox/branches/develop/protection --jq '.required_status_checks' 2>/dev/null || echo classic-protection-not-configured", "description": "Classic branch protection is absent or has no conflicting required-check policy", "expected": "classic-protection-not-configured OR no broader/strict required checks", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 2}, {"command": "branch=$(git branch --show-current)\npr_base=$(gh pr view \"$branch\" --json baseRefName --jq .baseRefName)\n[ \"$pr_base\" = \"develop\" ] || { echo \"PR for $branch targets $pr_base, not develop\"; exit 1; }\nrun_id=$(gh run list --workflow test.yml --branch \"$branch\" --event pull_request --limit 1 --json databaseId --jq '.[0].databaseId')\ngh run view \"$run_id\" --json jobs --jq '[.jobs[] | select(.conclusion != \"skipped\") | .name] | sort[]'\n", "description": "Routine develop PR Tests workflow no longer queues broad non-required lanes", "expected": "Only non-skipped required/lightweight develop PR job names, plus any explicitly documented lightweight umbrella/content jobs", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 3}, {"command": "gh workflow list --json name,state,path --jq '.[] | select((.name | test(\"nightly|Nightly\")) or (.path | test(\"nightly\")))'", "description": "Nightly/manual workflow exists and is active for broad validation", "expected": "nightly workflow exists and state is active", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 4}, {"command": "rg -n 'pull_request:|branches:|main|tags:|matrix:|macos-latest|windows-latest|test-package|release' .github/workflows/test.yml .github/workflows/nightly.yml .github/workflows/release.yml", "description": "Main PR and release paths still expose full validation coverage", "expected": "main/tag release validation paths and full validation lanes are present outside nightly-only execution", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 5}, {"command": "rg -n 'strict-base|strict|required checks|nightly|workflow_dispatch|release-grade|pr-refresh' CLAUDE.md AGENTS.md", "description": "Agent docs describe the current CI contract without stale strict-base/deprecation claims", "expected": "docs state strict-base is already off, required checks are minimal, nightly/manual carries heavy validation, and pr-refresh remains an escape hatch", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": [], "notes": "Start from `_project/decisions/dev-loop-verification-2026-04-30.md`,\nthen re-read live GitHub state before editing workflows. Record:\n - Classic branch protection for `develop` remains absent or has no\n required-check drift.\n - Ruleset `develop-squash-only` still requires exactly `lint` and\n `test (ubuntu-latest, 3.12)`.\n - `strict_required_status_checks_policy` remains `false`.\n - Every job in `.github/workflows/test.yml` is classified as required,\n non-required PR-tier, push/main, label-gated, or manual/nightly.\n\nThis is a gate for the rest of Step 3. If required contexts changed\nafter Step 0, stop and re-evaluate rather than editing CI around stale\nassumptions.\n\nImplementation record (2026-04-30):\n - `gh api repos/joeharris76/BenchBox/branches/develop/protection`\n returned HTTP 404, so classic branch protection remains absent.\n - `gh api repos/joeharris76/BenchBox/rulesets/15611785` returned\n required status checks `lint` and `test (ubuntu-latest, 3.12)`;\n `strict_required_status_checks_policy` is `false`.\n - Required PR contexts: `lint` from `.github/workflows/lint.yml` and\n `test (ubuntu-latest, 3.12)` from the `Tests` matrix.\n - Non-required routine PR contexts in `.github/workflows/test.yml`:\n `test (ubuntu-latest, 3.10)`, `test (ubuntu-latest, 3.13)`,\n `test (macos-latest, 3.12)`, `test (windows-latest, 3.12)`,\n `test (windows-latest, 3.10)`, `check-test-patterns`, `security`,\n `integration-smoke` on Ubuntu and Windows, `integration-table-formats`,\n `test-package` on Ubuntu and Windows, `parity`, and `pyspark-tests`.\n - Label-gated/push-main context: `integration` runs on `push` or PRs\n carrying `run-integration`; it is skipped for ordinary develop PRs.\n - Manual/nightly broad validation does not exist yet; Step 3 must add\n it before removing non-required routine PR lanes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconfirm protection/ruleset state and classify PR-tier jobs by requirement status", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w1"], "notes": "Choose the smallest maintainable workflow shape that keeps the two\nrequired contexts stable. The default recommendation is:\n - Keep `lint` in its existing workflow.\n - Keep `test (ubuntu-latest, 3.12)` reporting under the existing Tests\n workflow/job identity for code PRs.\n - Remove, split, or job-gate non-required develop PR lanes from the\n routine `pull_request` path.\n - Preserve label-gated/manual execution for heavier checks when a PR\n intentionally needs them.\n\nPrefer a clear workflow split (`test.yml` for PR-tier required checks,\n`nightly.yml` for broad validation) over repeating fragile job-level\nconditions everywhere. If a split would break required check names,\nuse targeted conditions instead and document why.\n\nImplementation record (2026-04-30):\n - Kept `.github/workflows/lint.yml` unchanged so the required `lint`\n context stays stable.\n - Replaced the broad `test` matrix with one explicit required job named\n `test (ubuntu-latest, 3.12)`, preserving the required context while\n avoiding matrix evaluation for routine develop PRs.\n - Added `compat-test` for non-required OS/Python combinations on\n main/push surfaces.\n - Added develop-PR gates to non-required `test.yml` jobs:\n `compat-test`, `check-test-patterns`, `security`,\n `integration-smoke`, `integration-table-formats`, `test-package`,\n `parity`, and `pyspark-tests`.\n - Left `integration` label-gated as-is so `run-integration` can still\n opt a PR into the heavier integration lane intentionally.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the develop PR trigger policy for non-required test.yml jobs", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w2"], "notes": "Add or update `.github/workflows/nightly.yml` so the jobs removed from\nroutine develop PRs still run on:\n - `workflow_dispatch`\n - a low-contention nightly schedule\n\nThe nightly/manual path should cover the multi-OS/Python matrix,\nintegration smoke and table-format checks, package install validation,\nparity, security audit, and PySpark lane unless w1 finds a specific lane\nbelongs somewhere else. Use existing commands and dependency setup from\n`test.yml`; do not invent a parallel test stack.\n\nIf failure issue automation is added, keep it narrow and idempotent:\none open issue per failing nightly class, labeled `incident:nightly-red`,\nwith the run URL and failing job names. If that automation is too large\nfor this PR, defer it explicitly and ensure nightly failures are still\nvisible through GitHub notifications.\n\nImplementation record (2026-04-30):\n - Added `.github/workflows/nightly.yml` with `workflow_dispatch` and a\n daily 06:00 UTC schedule.\n - Nightly coverage includes the full OS/Python fast-test matrix,\n patch-safety check, security audit, integration smoke,\n table-format integration, full local integration, package install,\n parity, and non-blocking PySpark lanes.\n - Reused the existing `uv` setup and test commands from `test.yml`.\n - Deferred automatic issue creation to keep this PR scoped; failures\n remain visible through GitHub Actions notifications and manual\n workflow dispatch is available for reruns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move broad validation to nightly/workflow_dispatch without losing coverage", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w2", "w3"], "notes": "Audit `.github/workflows/test.yml`, `.github/workflows/nightly.yml`, and\n`.github/workflows/release.yml` after the split/gating work. Confirm\n`pull_request` to `main` and `push`/tag release paths still run\nrelease-grade coverage automatically, not only through nightly:\n - Full OS/Python compatibility coverage remains on a main/release path.\n - Package build/install validation remains on a main/release path.\n - Release-flow checks are unchanged.\n\nA comment in the workflow is acceptable when it prevents future drift,\nbut avoid comment-only churn unless it clarifies a real branch-specific\ninvariant.\n\nImplementation record (2026-04-30):\n - Audited `.github/workflows/test.yml`: `pull_request` to `main` and\n `push` to `main` still run the required Ubuntu 3.12 fast-test job,\n the additional compatibility matrix, package install, security,\n smoke/table-format, and parity lanes. The new gates exclude only\n `pull_request` events whose base branch is `develop`.\n - Audited `.github/workflows/validate-main-pr.yml`: main PRs remain\n restricted to release branches through `validate-base`.\n - Audited `.github/workflows/release.yml`: `push` tags matching `v*`\n still run dependency bounds, reproducible build, package checks,\n publish, GitHub release creation, and post-publish installation\n across Ubuntu/macOS/Windows and Python 3.10/3.12/3.13 coverage.\n - No release workflow edits were needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Preserve main PR and tag-release validation semantics", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w2", "w3", "w4"], "notes": "Update `CLAUDE.md` and `AGENTS.md` only where the workflow contract is\nwrong or incomplete. The guidance should say:\n - Develop required checks are already minimal: `lint` and\n `test (ubuntu-latest, 3.12)`.\n - Strict-base is already off for `develop`; do not describe Step 3 as\n a strict-base deployment.\n - Heavy non-required lanes moved out of routine develop PR execution\n remain available through nightly/workflow_dispatch or explicit labels.\n - `main` PRs and tag releases retain release-grade validation.\n - `make pr-refresh` remains an escape hatch for genuine branch drift;\n do not deprecate it solely because strict-base was already off.\n\nImplementation record (2026-04-30):\n - Updated `CLAUDE.md` and `AGENTS.md` to state that develop required\n checks are intentionally minimal and strict-base is already off.\n - Documented the split between routine develop PR gates and broad\n nightly/workflow_dispatch validation.\n - Preserved `make pr-refresh` as an escape hatch rather than a\n deprecated path.\n - Documented that main PRs and tag releases keep release-grade\n validation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update agent docs for the replanned CI contract", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Run focused workflow/TODO validation first, then the repo preflight\nappropriate to the code touched. For workflow-only changes, at minimum:\n - YAML validation on edited workflow and TODO files\n - `uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph`\n - `make pr-preflight`\n - `/code review W6` or equivalent adversarial review of the diff\n - `make pr-open`\n\nThe PR description must state that no branch-protection or ruleset PATCH\nis part of Step 3 because Step 0 already proved the desired required\ncontexts and strict-base setting are live on `develop`.\n\nImplementation record (2026-04-30):\n - Rebased the implementation branch onto `origin/develop` after the\n Step 3 replan PR merged.\n - Validated edited YAML files with `pre-commit` `check-yaml`.\n - Ran `todo_cli.py check-graph`; graph validation passed.\n - Reconfirmed the live `develop-squash-only` ruleset still requires\n only `lint` and `test (ubuntu-latest, 3.12)` with strict-base off.\n - Reconfirmed classic branch protection remains absent.\n - Ran `make pr-preflight`; CI lint checks passed and fast tests\n reported `21268 passed, 6 skipped`.\n - Final branch review found no blocking issues.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify locally, review the change, and open the PR", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "merge workflow changes first, then patch branch protection/rulesets", "dont": "apply branch-protection changes before the workflow PR lands", "why": "the required check name will not exist on develop yet"}, {"INSTEAD": "add develop content validation or classify docs/** as code", "dont": "assume docs.yml protects develop docs PRs", "why": "it currently triggers PR checks for main only"}, {"INSTEAD": "default unknown paths to full PR-tier", "dont": "classify unknown paths as non-code", "why": "new top-level areas can affect builds/tests and a false green weakens branch protection"}, {"INSTEAD": "share via `.github/path-filters.yml` or a single shell helper", "dont": "duplicate the path ruleset between CI and local pr-preflight", "why": "they will drift and the local skip will lie to agents"}, {"INSTEAD": "move the develop PR lint gate under the umbrella first", "dont": "make ci-required-result the only required check while required lint remains isolated in lint.yml", "why": "the result job cannot aggregate another workflow"}, {"INSTEAD": "route them through full PR-tier or a blocking gitignore-lint aggregation", "dont": "mark .gitignore or .gitattributes as safe content-only", "why": "they affect repository behavior and tracked-file conflict guardrails"}, {"INSTEAD": "keep Makefile in the code path list", "dont": "mark Makefile changes as non-code", "why": "Makefile edits change developer-facing surface and have caused real regressions"}, {"INSTEAD": "pin by SHA", "dont": "pin `dorny/paths-filter@v3` by tag", "why": "tags are mutable and a compromised release could exfil secrets"}, {"INSTEAD": "run a cheap content guard while skipping Python fast tests", "dont": "skip all validation on content-only PRs", "why": "markdown/YAML/TODO/docs errors are relevant and PR #66/#67 surfaced exactly that class of issue"}, {"INSTEAD": "fork-test before enabling on develop", "dont": "skip the smoke-test (w4)", "why": "a buggy umbrella that always greens would make develop unprotected and you would not notice until something breaks"}, {"INSTEAD": "use job-level `if:` with an umbrella instead", "dont": "use top-level `paths:` / `paths-ignore:` filters on the workflow `on:` block", "why": "GitHub treats workflows-not-run as 'pending' for required-checks, leaving the PR unmergeable"}], "approach": "Minimum implementation surface: `.github/path-filters.yml`\n(ruleset), the PR-tier workflow (`test.yml` or `pr.yml`) that adds\n`ci-paths`, `content-guard`, `code-lint`, `code-test`, and\n`ci-required-result`, plus a small local helper for `make pr-preflight`\nand the pre-push hook.\n\nFinal implementation uses `scripts/path_filter_decision.py`, a stdlib-only\nhelper, instead of a third-party path-filter action. That keeps CI and local\n`make pr-preflight` on the same code path, avoids a mutable action tag, and\nlets content-only PRs classify paths before dependency installation.\n\nRequired-check aggregation has one hard GitHub Actions constraint:\n`needs:` only works inside one workflow. If the required `lint` gate\nstays in `.github/workflows/lint.yml`, `ci-required-result` cannot\nobserve it. Move/duplicate the required develop PR lint/type work into\nthe umbrella workflow before making `ci-required-result` the only\nrequired check.\n\nFor the umbrella aggregation step, the explicit pass/fail script\n(rather than relying on `needs:` propagation alone) is intentional:\nit makes the \"content-only can pass without Python tests\" semantic visible in the workflow\nlog, and it correctly handles the case where `needs.lint.result` is\nthe literal string `\"skipped\"` (which evaluates truthy in some\ncomparison contexts).\n\nSequence the rollout: workflow changes first (w6 merged to develop),\nTHEN branch-protection/ruleset change (w7). Reverse order leaves develop\nunmergeable for whatever interval because the required umbrella check name\nwill not exist on the base branch yet.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 144, "reason": "Post-merge runs are the safety net; skipping them based on paths could mask a code regression that only manifests when a doc-only commit lands on top of an existing code change. Keep post-merge always-runs.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Deliberately rejected: post-merge CI is kept always-on as the regression safety net, explicitly not path-skipped.", "summary": "Path-based skip on Step 4 develop-post-merge.yml"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 145, "reason": "Nightly is scheduled, not PR-triggered. The 'doc-only' question doesn't apply \u2014 nightly runs the full matrix regardless of recent commits.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly states path-based skip does not apply to nightly.yml since it's schedule-triggered and always runs full matrix.", "summary": "Path-based skip on nightly.yml"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 146, "reason": "Premature optimization. dorny/paths-filter completes in seconds; caching adds complexity for negligible gain.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly called premature optimization (dorny/paths-filter already completes in seconds); no need signal.", "summary": "Cache the path-filter ruleset across steps for performance"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 147, "reason": "Bigger redesign than the umbrella. Step 3a only separates safe content-only PRs from everything else. Revisit only if Step 5 metrics show it's worth the complexity.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Per-path CI matrix beyond content-vs-code gated on Step 5 metrics; dev-loop-step-5-measurement-window is still open/active, condition unevaluated.", "summary": "Per-path matrix dispatch beyond content-vs-code (e.g., only run explorer tests for results-explorer changes)"}], "deps": ["dev-loop-step-1-local-loop-cleanup", "dev-loop-step-3-ci-rebalance-and-strict-base-off"], "description": "Companion to Step 3. After Step 3 removes broad non-required lanes\nfrom routine develop PR CI while preserving the existing lint +\nUbuntu 3.12 fast-test required contexts, this step closes the remaining waste:\nPRs that touch zero code (TODOs, blind-spots, decisions, blog drafts,\nagent-doc edits) still run the surviving 2-3 jobs unnecessarily.\n\nEvidence: PR #66 and PR #67 \u2014 both pure `_project/TODO/**` changes \u2014\ntriggered ~16 jobs each (full matrix + integration + parity + PySpark)\npre-Step-3, and would still trigger ~3 jobs post-Step-3. Both rounds\nof waste are unnecessary; nothing in those PRs could possibly affect\nthe test suite.\n\nApproach: introduce a single always-reporting required check\n(`ci-required-result`) that classifies the PR's diff, runs a cheap\ncontent guard for content-only PRs, and dispatches the Step-3 code\nPR-tier (`lint`/type + Ubuntu 3.12 fast tests) only when code paths\nchanged. For TODO/blog/agent-doc-only PRs, the required check should\nreport green in <60 seconds without running Python fast tests.\n\nBranch protection contract: `ci-required-result` becomes the required\nstatus check on develop. Any jobs whose failures should block merge\nmust be in the same workflow dependency graph as the result job, because\nGitHub Actions `needs:` cannot aggregate jobs from separate workflows.\nThe current `lint.yml` workflow therefore moves its develop PR gate\nunder the umbrella. Leave `lint.yml` for main/push/manual use, or for\nnon-required reporting only.\n\nSingle PR via `make pr-open`. Independently shippable AFTER Step 3\nlands so the PR-tier workflow shape is stable. The starting required-checks\nlist is already established by the live `develop` ruleset; Step 3a is the\nlater step that changes it to the umbrella result check.", "id": "dev-loop-step-3a-path-based-ci-skip", "preserves": ["Branch-protection contract \u2014 required check always reports a status; the umbrella is the new required check, not a removal of all required checks", "Code-touching PRs continue to run full PR-tier CI as today (post-Step-3 narrowed) \u2014 the skip applies only when every changed path is in the safe content-only allowlist", "Content validation \u2014 content-only PRs skip Python fast tests but still run cheap TODO/markdown/docs hygiene relevant to the changed paths", "Existing PR-tier code checks from Step 3 \u2014 Step 3a gates them, it does not remove lint/type or Ubuntu 3.12 fast tests for code/infra/unknown-path PRs", "Fail-closed path behavior \u2014 unknown paths, .gitignore, .gitattributes, workflow/config/tooling changes run full PR-tier unless explicitly and safely reclassified", "Single source of truth for the path ruleset \u2014 `path-filters.yml` is referenced by both CI and local pr-preflight; do not duplicate the rules", "Step 4's develop-post-merge.yml safety net is unchanged \u2014 post-merge auto-revert still runs on every landed SHA, doc-only or not"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this; post-merge runs on every SHA, doc-only or not)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/nightly.yml (Step 3 owns this; nightly always runs the full matrix)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml (release CI is unchanged)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets \u2014 Step 3a w7 IS the rule edit, but the edit is to the required-checks list only; do not change strict-base, dismiss-stale-reviews, enforce-admins, etc."}, {"kind": "do_not_modify", "path_glob": "Makefile pr-fanout target (Step 1 owns this)"}, {"kind": "do_not_modify", "path_glob": "Makefile worktree-add, worktree-claim, worktree-pool-*, worktree-release targets (Step 2 owns these)"}, {"kind": "do_not_modify", "path_glob": "tests/ (no test changes; only CI orchestration and a tiny local helper)"}, {"kind": "only_modify", "path_glob": ".claude/commands/pr.md (one-shot PR command summary)"}, {"kind": "only_modify", "path_glob": ".github/path-filters.yml (new path-filter ruleset)"}, {"kind": "only_modify", "path_glob": ".github/workflows/lint.yml (remove/disable duplicated develop PR trigger after its required gate moves under the umbrella, or leave as non-required reporting only)"}, {"kind": "only_modify", "path_glob": ".github/workflows/test.yml or .github/workflows/pr.yml (whichever Step 3 ended up with as PR-tier; add umbrella/result/content/code jobs)"}, {"kind": "only_modify", "path_glob": ".pre-commit-config.yaml (pr-preflight-fast-tests hook \u2014 add path-filter shortcut)"}, {"kind": "only_modify", "path_glob": "AGENTS.md (umbrella + local skip paragraph)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (umbrella + local skip paragraph)"}, {"kind": "only_modify", "path_glob": "CONTRIBUTING.md (required-check and preflight prose)"}, {"kind": "only_modify", "path_glob": "Makefile (pr-preflight target \u2014 add path-filter shortcut)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-path-filter-smoke-test-*.md (smoke-test evidence)"}, {"kind": "only_modify", "path_glob": "scripts/path_filter_decision.sh or scripts/path_filter_decision.py (new local helper)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 3a: path-based CI skip for content-only PRs", "verifications": [{"command": "test -f .github/path-filters.yml && grep -cE 'safe[-_]?content|content[-_]?only|needs[-_]?code|code[-_]?ci|unknown' .github/path-filters.yml", "description": "Path filter ruleset exists and has safe-content plus fail-closed code categories", "expected": ">= 1", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE 'ci-paths:|ci-required-result:|content-guard:' .github/workflows/test.yml .github/workflows/pr.yml 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "ci-required umbrella/result jobs and content guard are present in PR-tier workflow", "expected": ">= 3", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -cE \"if:.*needs\\.ci-paths\\.outputs\\.needs-code-ci.*true\" .github/workflows/test.yml .github/workflows/pr.yml 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "Code jobs gate on the fail-closed classifier output", "expected": ">= 2", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -cE 'needs: \\[.*code-lint|code-lint:|ci-required-result:' .github/workflows/test.yml .github/workflows/pr.yml 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "Develop PR lint/type failures are aggregated by the required result check", "expected": ">= 2", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 4}, {"command": "repo=$(gh repo view --json nameWithOwner --jq .nameWithOwner)\nfound=0\nfor ruleset_id in $(gh api \"repos/${repo}/rulesets\" --jq '.[] | select(.target == \"branch\" and ((.conditions.ref_name.include // []) | any(. == \"refs/heads/develop\"))) | .id'); do\n found=1\n gh api \"repos/${repo}/rulesets/${ruleset_id}\" --jq '[.rules[] | select(.type == \"required_status_checks\") | .parameters.required_status_checks[]?.context]'\ndone\ntest \"$found\" = \"1\"\n", "description": "Branch-protection required check is the umbrella, not the subordinate jobs", "expected": "list contains 'ci-required-result' and does NOT contain bare 'lint' or 'test (ubuntu-latest, 3.12)'", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -cE 'TODO.only|docs/|under 60 seconds|unknown.*full|gitignore|code-lint|code-test|content guard' _project/decisions/dev-loop-path-filter-smoke-test-*.md", "description": "Smoke-test decision record proves content-only skip, code fallback, unknown fallback, and gitignore handling", "expected": ">= 6", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 6}, {"command": "make pr-preflight 2>&1 | grep -cE 'No code changes detected|skipping fast tests' || echo 'requires a doc-only diff to demonstrate'", "description": "Local pr-preflight skips only fast tests for content-only diffs", "expected": ">= 1 when run on a doc-only branch; absent on a code-changed branch", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 7}, {"command": "grep -cE 'ci-required-result|path-filters\\.yml|content-only|path.*filter|skipping fast tests' CLAUDE.md AGENTS.md CONTRIBUTING.md .claude/commands/pr.md", "description": "Agent and contributor docs document the new required check and path behavior", "expected": ">= 4", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 8}, {"command": "! grep -R \"dorny/paths-filter@v3\" .github/workflows", "description": "No mutable dorny/paths-filter v3 tag is present", "expected": "no matches", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 9}], "work": [{"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": [], "notes": "Decide what is explicitly safe to treat as content-only. The rule is\nfail-closed: a PR can skip code CI only when EVERY changed path is in\nthe safe content-only allowlist. Any path outside that allowlist is a\ncode/infra path and runs the full post-Step-3 PR-tier.\n\nSafe content-only paths (skip Python fast tests; run content guard):\n - `_project/TODO/**`, `_project/DONE/**`\n - `_project/blind-spots/**`\n - `_project/decisions/**`\n - `_project/handoffs/**`\n - `_project/notes/**`, `_project/research/**`, `_project/planning/**`\n - `_blog/**`\n - `docs/**` ONLY if the content guard includes docs validation for\n develop PRs; `docs.yml` currently runs PR checks for `main`, not\n `develop`, so do not rely on it as-is.\n - `*.md` at repo root (README.md, CONTRIBUTING.md, etc.)\n - `CLAUDE.md`, `AGENTS.md`\n\nExamples that MUST run full PR-tier (non-exhaustive; unknown paths\nalso run full PR-tier):\n - `benchbox/**`, `tests/**`, `scripts/**`, `examples/**`\n - `landing/**`, `results-data/**`, `results-explorer/**`, `docker/**`\n - `_binaries/**`, `_sources/**`\n - `_project/scripts/**`, `_project/config/**`, `_project/compat/**`\n - `pyproject.toml`, `uv.lock`, `Makefile`\n - `.github/workflows/**`, `.pre-commit-config.yaml`\n - `.claude/commands/**` (slash-command behavior affects agents)\n - `.gitignore`, `.gitattributes` (repo behavior and tracked-file\n conflict guardrails; `.gitignore` also has gitignore-lint)\n\nEdge case: a single PR that touches BOTH categories runs full\nPR-tier (failsafe). The filter is \"all paths safe \u2192 skip code CI\";\nanything else runs code CI.\n\nDocument the ruleset in `.github/path-filters.yml` (a reusable\nfilter file referenced by the umbrella and local helper) so it has\na single source of truth.\n\nImplementation record (2026-04-30):\n - Added `.github/path-filters.yml` with a `safe-content` allowlist\n covering TODO/DONE records, blind spots, decisions, handoffs, notes,\n research/planning records, blog drafts, docs, root markdown, and\n agent docs.\n - Added `content-guard` patterns matching the safe content surface so\n content-only PRs still run targeted validation before the umbrella\n can report green.\n - Added explicit `code-ci` examples for code, tests, tooling, workflow,\n Makefile, config, lockfile, git behavior, and agent-command paths.\n - Kept unknown paths implicit and fail-closed so a new top-level path\n runs full PR-tier CI instead of silently skipping code checks.\n - W1 review issue fixed: removed a broad `**` code-ci pattern because\n it would make path-filter tooling report code CI for every content\n change and defeat the skip.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define a fail-closed path-filter ruleset", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w1"], "notes": "Edit the workflow that runs on `pull_request` to develop (Step 3\nmay have split test.yml; if so, edit the new pr-tier workflow;\notherwise edit test.yml). Add:\n\n - `ci-paths`: always runs, checks out the PR, uses\n `dorny/paths-filter@` (not `@v3`) or the shared helper\n to emit:\n - `safe-content-only: true|false`\n - `needs-code-ci: true|false`\n - `changed-paths` / optional debug summary\n - optional metric line/artifact: content-only?, skipped code\n CI?, estimated runner minutes saved (for Step 5)\n - `content-guard`: runs when a PR contains safe content paths. This\n is intentionally cheap and targeted: TODO YAML validation,\n markdown/YAML hygiene, docs validation for `docs/**` develop PRs,\n and gitignore-lint if `.gitignore` is ever routed through this\n workflow. It must catch the kind of markdown/YAML errors that\n mattered on PR #66/#67.\n - `code-lint`: the post-Step-3 lint/type gate, gated on\n `needs-code-ci == 'true'`.\n - `code-test`: Ubuntu 3.12 fast tests, gated on\n `needs-code-ci == 'true'`.\n - `ci-required-result`: always runs, aggregates all of the above,\n and is the required check name.\n\nDo not try to aggregate the existing `.github/workflows/lint.yml`\n`lint` job from `ci-required-result`: `needs:` cannot cross workflow\nboundaries. Move/duplicate the required develop PR lint/type gate\ninto the umbrella workflow before making `ci-required-result` the\nsole required check. Leave `lint.yml` for main/push/manual surfaces\nor non-required reporting only.\n\nRequired behavior:\n 1. `ci-paths` and `ci-required-result` always run (no `if:`\n condition on those jobs).\n 2. Content-only PRs run `content-guard`, skip `code-lint` and\n `code-test`, and pass only if `content-guard` succeeds.\n 3. Code/infra/unknown-path PRs run `content-guard` as applicable\n plus `code-lint` and `code-test`.\n 4. If `needs-code-ci == true`, `ci-required-result` fails unless\n both `code-lint` and `code-test` succeeded.\n 5. If `needs-code-ci == false`, `ci-required-result` fails unless\n `content-guard` succeeded or no content guard was needed.\n\nConcrete shape (sketch):\n\n jobs:\n ci-paths:\n runs-on: ubuntu-latest\n outputs:\n needs-code-ci: ${{ steps.classify.outputs.needs-code-ci }}\n safe-content-only: ${{ steps.classify.outputs.safe-content-only }}\n steps:\n - uses: actions/checkout@v4\n - uses: dorny/paths-filter@\n id: filter\n with:\n filters: .github/path-filters.yml\n - id: classify\n run: scripts/path_filter_decision.sh\n\n content-guard:\n needs: ci-paths\n if: needs.ci-paths.outputs.safe-content-only == 'true' ||\n needs.ci-paths.outputs.needs-code-ci == 'true'\n ...\n\n code-lint:\n needs: ci-paths\n if: needs.ci-paths.outputs.needs-code-ci == 'true'\n ...\n\n code-test:\n needs: ci-paths\n if: needs.ci-paths.outputs.needs-code-ci == 'true'\n ...\n\n ci-required-result:\n needs: [ci-paths, content-guard, code-lint, code-test]\n if: always()\n runs-on: ubuntu-latest\n steps:\n - run: |\n set -euo pipefail\n if [ \"${{ needs.ci-paths.outputs.needs-code-ci }}\" = \"false\" ]; then\n if ! { test \"${{ needs.content-guard.result }}\" = \"success\" ||\n test \"${{ needs.content-guard.result }}\" = \"skipped\"; }; then\n echo \"content-guard=${{ needs.content-guard.result }}\"\n exit 1\n fi\n echo \"Content-only PR \u2014 skipped Python code CI.\"\n exit 0\n fi\n if [ \"${{ needs.code-lint.result }}\" = \"success\" ] && \\\n [ \"${{ needs.code-test.result }}\" = \"success\" ]; then\n exit 0\n fi\n echo \"code-lint=${{ needs.code-lint.result }} code-test=${{ needs.code-test.result }}\"\n exit 1\n\nThe required check on develop branch protection becomes\n`ci-required-result`, NOT bare `lint` or `test` directly, only after\nthe workflow has landed on develop.\n\nImplementation record (2026-04-30):\n - Added `.github/workflows/pr.yml` for `pull_request` events targeting\n `develop`; `.github/workflows/test.yml` and\n `.github/workflows/lint.yml` now keep main/push surfaces while this\n workflow owns the develop PR gate.\n - Added always-reporting `ci-paths` and `ci-required-result` jobs.\n `ci-paths` writes a JSON decision artifact and exposes\n `safe-content-only`, `needs-code-ci`, `content-guard-needed`,\n changed paths, code paths, and unknown paths as job outputs.\n - Added `content-guard` for content paths. It validates changed YAML\n with `check-yaml`, markdown with the existing markdownlint hook,\n TODO/DONE YAML with `validate_todo.py` plus `todo_cli.py check-graph`,\n and `docs/**` changes with `make docs-validate`.\n - Added gated `code-lint` and `code-test` jobs. Their displayed check\n names remain `lint` and `test (ubuntu-latest, 3.12)` so the current\n pre-W7 ruleset still receives the required contexts until the\n post-merge ruleset edit switches to `ci-required-result`.\n - Added `scripts/path_filter_decision.py`, a stdlib-only classifier\n that reads `.github/path-filters.yml`, fails closed on unknown\n paths, writes GitHub outputs, emits helper path lists, and supports\n local/CI callers without duplicating rules.\n - W2 review issue fixed: added explicit `git fetch` steps for the base\n branch before diff classification so Actions runners do not depend\n on checkout-specific remote-ref behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ci-required umbrella, content guard, and gated code jobs", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w1"], "notes": "Step 1 made pre-push hooks opt-in via BENCHBOX_PREPUSH=1. When\nhumans/agents DO opt in (e.g. running `make pr-preflight`), the\nfast-test lane still runs even on content-only changes \u2014 same waste,\njust locally instead of in CI.\n\nAdd a check at the top of `pr-preflight` and the\npr-preflight-fast-tests pre-commit hook that runs the same path\ndecision against `git diff --name-only origin/develop...HEAD`. If\nno code paths matched, run the cheap content guard, print\n\"No code changes detected; skipping fast tests.\", and skip only the\nPython fast-test lane.\n\nReuse the path-filter ruleset from `.github/path-filters.yml`\nvia a small helper (`scripts/path_filter_decision.sh` or Python if\nYAML parsing is needed) so the rule lives in exactly one place.\n\nImplementation record (2026-04-30):\n - Updated `make pr-preflight` so lint/type checks still run first,\n then the Python fast-test lane delegates to `make\n pr-preflight-fast-tests`.\n - Added `pr-preflight-fast-tests`, which classifies\n `origin/develop...HEAD` with `scripts/path_filter_decision.py`.\n Code/infra/unknown-path diffs still run `uv run -- python -m pytest\n -m fast -q`; content-only diffs run the cheap content guard and\n print `No code changes detected; skipping fast tests.`\n - Added `pr-content-guard` so local preflight mirrors CI content\n validation for YAML, markdown, TODO/DONE graph, and docs reference\n checks.\n - Updated the opt-in `pr-preflight-fast-tests` pre-push hook to call\n the same Make target when `BENCHBOX_PREPUSH=1`, avoiding a separate\n hook-only path decision.\n - Extended `scripts/path_filter_decision.py` with `--json-in` and\n `--check` so Makefile logic can consume the same classifier output\n without duplicating JSON parsing in shell.\n - W3 review found no actionable issues after checking quoting,\n content-only guard behavior, and code-path fallback behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add the equivalent local skip in pr-preflight", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w2"], "notes": "In a fork or via `act` locally, simulate two PR diffs:\n 1. Touch only `_project/TODO/foo.yaml` \u2014 confirm\n `ci-required-result` reports green in under 60 seconds, the\n content guard runs, and code-lint/code-test do not run.\n 2. Touch `docs/development/foo.md` \u2014 confirm docs/content\n validation runs for develop PRs without Python fast tests.\n 3. Touch `benchbox/cli/run.py` \u2014 confirm full PR-tier runs as\n today and umbrella aggregates correctly.\n 4. Touch an unknown top-level path (e.g., `quality/example.txt`)\n \u2014 confirm the classifier fails closed and runs full PR-tier.\n 5. Touch `.gitignore` \u2014 confirm full PR-tier or gitignore-lint is\n included in the blocking aggregation.\nDocument the test in `_project/decisions/dev-loop-path-filter-\nsmoke-test-.md` with workflow run URLs and timings.\n\nImplementation record (2026-04-30):\n - Checked for `act`; it is not installed in this environment, so a\n local classifier/content-guard smoke run was used before the workflow\n PR exists on GitHub.\n - Added `_project/decisions/dev-loop-path-filter-smoke-test-2026-04-30.md`\n with the five required cases: TODO-only, `docs/**`, code,\n unknown top-level path, and `.gitignore`.\n - Confirmed TODO-only and docs paths classify as safe content-only,\n run content validation, and skip Python fast tests.\n - Confirmed code, unknown, and `.gitignore` paths classify as\n `needs_code_ci=true`, routing through `code-lint` and `code-test`.\n - W4 review caveat recorded: GitHub workflow run URLs are not available\n until the workflow PR is opened; W6 must use the branch PR run as the\n first hosted workflow evidence.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a tiny test that proves the skip path works", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w2", "w3"], "notes": "Update every place that currently hard-codes the old\n`lint` + `test (ubuntu-latest, 3.12)` contract:\n - `CLAUDE.md`\n - `AGENTS.md`\n - `CONTRIBUTING.md`\n - `.claude/commands/pr.md`\n\nRequired prose:\n - Develop PRs report through `ci-required-result`.\n - Content-only PRs run content validation and skip Python fast\n tests.\n - Code/infra/unknown-path PRs run the post-Step-3 lint/type +\n Ubuntu 3.12 fast-test gate.\n - The shared path ruleset is `.github/path-filters.yml`.\n - Local `make pr-preflight` uses the same classifier.\n\nImplementation record (2026-04-30):\n - Updated `CLAUDE.md`, `AGENTS.md`, `CONTRIBUTING.md`, and\n `.claude/commands/pr.md` to describe `ci-required-result` as the\n develop PR umbrella.\n - Documented `.github/path-filters.yml` as the shared path ruleset.\n - Documented that content-only PRs run content validation and skip\n Python fast tests, while code/infra/workflow/tooling/unknown paths run\n the post-Step-3 lint/type + Ubuntu 3.12 fast-test gate.\n - Documented that local `make pr-preflight` uses the same classifier\n and skips only the Python fast-test lane on content-only branches.\n - W5 review found no stale docs that still describe the develop\n required gate as bare `lint` + `test (ubuntu-latest, 3.12)`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the umbrella in agent and contributor docs", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "`make lint format typecheck`; `BENCHBOX_PREPUSH=1 make pr-preflight`;\n`make pr-open`. The PR description must state that the branch\nprotection/ruleset edit happens AFTER this PR merges, because the\nnew required check name does not exist on develop until then.\n\nInclude smoke-test fork run URLs and confirmation that PR\n#66/#67-style TODO-only PRs would now skip Python code CI while\nstill running content validation.\n\nImplementation record (2026-04-30):\n - Rebased the branch onto `origin/develop` after PR #78 landed.\n Resolved the Makefile conflict by preserving upstream worktree-pool\n targets and Step 3a's `pr-preflight-fast-tests` / `pr-content-guard`\n targets in the same phony lists.\n - Ran `make lint format typecheck`; ruff passed, format left files\n unchanged, and `ty check` exited 0 while reporting the repo's existing\n 426 diagnostics.\n - Ran `BENCHBOX_PREPUSH=1 make pr-preflight`; CI lint checks passed,\n timing policy reported 0 violations, dependency audit reported 0\n violations, and the path-aware fast-test lane ran because this branch\n touches code/workflow paths.\n - Fast tests reported `21268 passed, 6 skipped, 41 warnings, 4 subtests\n passed in 63.30s`.\n - Final W6 review confirmed the branch classifies as\n `needs_code_ci=true`, develop PR checks moved out of `lint.yml` and\n `test.yml`, and `.github/workflows/pr.yml` supplies `lint`,\n `test (ubuntu-latest, 3.12)`, and `ci-required-result`.\n - Hosted PR run #25186848736 exposed one W6 issue:\n `content-guard` ran `pre-commit` without first syncing dev\n dependencies. Fixed by adding `uv sync --group dev` to the\n `content-guard` job before YAML/markdown validation.\n - Hosted push runs exposed a second W6 issue: a reusable rules file\n under `.github/workflows/` is treated by GitHub as a workflow file.\n Moved the shared ruleset to `.github/path-filters.yml` and updated\n the classifier/docs to avoid an invalid workflow named after the\n rules file.\n - Hosted PR run #25187095898 exposed a third W6 issue: the project\n environment did not include the `pre-commit` executable, so\n `uv run -- pre-commit` resolved only on machines with a global\n install. Added `pre-commit` to the dev dependency group and refreshed\n `uv.lock` so CI and local content guard runs use the locked CLI;\n also documented/allowlisted it as a CLI-only dev dependency.\n - W6 follow-up for PR creation: after this commit, run `make pr-open`\n and set the PR body to state that the required-check ruleset edit is\n W7 and happens only after this workflow PR lands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Open and merge the workflow PR before touching branch protection", "wid": "w6"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w6"], "notes": "Apply via `gh api -X PATCH repos/joeharris76/BenchBox/branches/\ndevelop/protection` (or the Step-0-found ruleset). Required\nchecks before this step (post-Step-3): `lint`, `test (ubuntu-latest, 3.12)`.\nRequired checks after this step: `ci-required-result`.\n\nThe workflow changes from w6 MUST already be on develop. Do not make\nthis branch-protection edit a prerequisite for opening or merging\nthe workflow PR.\n\nRecord the before/after JSON in `_project/decisions/\ndev-loop-path-filter-smoke-test-.md` or a follow-up decision\nrecord for audit. Do not change strict-base, dismiss-stale-reviews,\nenforce-admins, or review settings in this step.\n\nImplementation record (2026-04-30):\n - Confirmed PR #79 merged to `develop` at `2026-04-30T20:29:55Z`\n as squash commit `9c82b61f5be83cf018af216a7cf99866b7a4da04`.\n - Captured ruleset `develop-squash-only` (`15611785`) before the edit:\n required checks were `lint` and `test (ubuntu-latest, 3.12)`;\n `strict_required_status_checks_policy` was already `false`.\n - Updated the repository ruleset with a full `PUT` body derived from\n the live ruleset JSON, changing only the required status check list\n to `ci-required-result`.\n - Re-read the live ruleset after the update and confirmed required\n checks now contain only `ci-required-result`; squash-only review\n settings, strict-base policy, linear history, non-fast-forward, and\n deletion rules were preserved.\n - Recorded the before/after JSON in\n `_project/decisions/dev-loop-path-filter-smoke-test-2026-04-30.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Post-merge: update branch-protection required checks", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "open it but require manual approval", "dont": "auto-merge the revert PR", "why": "a human should decide whether to revert vs fix-forward"}, {"INSTEAD": "every landed SHA must be verified independently", "dont": "cancel-in-progress on develop-post-merge", "why": "fast PR landings would skip safety checks on intermediate SHAs"}, {"INSTEAD": "keep the surface small", "dont": "chain to Step 5 work in this PR", "why": "metrics emission is the deliverable here; aggregation/threshold-checking is Step 5"}, {"INSTEAD": "over-paging trains agents to ignore the channel", "dont": "page or alert on every red", "why": "the auto-revert PR IS the alert"}, {"INSTEAD": "30 minutes in a fork is cheap insurance", "dont": "skip the fork smoke test (w6)", "why": "a buggy auto-revert on production develop would be loud and embarrassing"}], "approach": "Build the workflow changes incrementally:\n 1. cancel-in-progress (w1) is a small, low-risk YAML edit\n 2. Auto-revert (w2) is the substantive change \u2014 use peter-evans/\n create-pull-request@v6 (mature, well-tested) rather than rolling\n a custom revert script\n 3. Metrics (w3) is independent of auto-revert; both can run in\n parallel jobs in the same workflow\n 4. Make target (w5) is the human-facing surface for w3\n 5. Docs (w4) once the behavior is confirmed\n 6. Smoke test in fork (w6) BEFORE merging \u2014 auto-revert that\n opens spurious PRs would be very noisy to roll back\n\nFor revert conflicts: the fallback to \"open an issue\" is intentional.\nAn auto-revert that conflicts probably means later commits depend on\nthe broken one, so a human needs to decide whether to revert the\nwhole stack or fix forward.\n\nFor metrics: keep it simple. JSON files in workflow artifacts, no\ndatabase, no service. Step 5 reads the last 30 days and aggregates.\nIf we end up wanting longer retention, push to a `metrics` orphan\nbranch instead \u2014 but that's a Step 5 decision, not Step 4.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-4-post-merge-safety-net", "id": 148, "reason": "Out of scope; the auto-revert PR is the notification. Add only if Step 5 shows reverts are missed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Slack/Discord notifications gated on Step 5 showing reverts are being missed; no such evidence indicated, condition unmet.", "summary": "Slack/Discord notifications on develop-red"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-4-post-merge-safety-net", "id": 149, "reason": "Intentional human gate. A human decides revert vs fix-forward. Revisit if revert PRs accumulate unmerged.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Intentional permanent human-gate design decision, not deferred work; revisit only if revert PRs pile up, no evidence of that.", "summary": "Auto-merge of revert PRs after green CI"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-4-post-merge-safety-net", "id": 150, "reason": "JSON artifacts + make dev-loop-metrics is sufficient for Step 5. Build a dashboard only if the metrics become a routine inspection target.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Dashboard/Grafana only warranted 'if metrics become a routine inspection target'; no evidence that threshold was reached.", "summary": "Metrics dashboard / Grafana"}], "deps": ["dev-loop-step-3-ci-rebalance-and-strict-base-off"], "description": "Strengthens `develop-post-merge.yml` so that relaxing strict-base in\nStep 3 has a credible safety net, not just a notification. When\ndevelop goes red post-merge, the workflow automatically opens a revert\nPR for the offending SHA (or an issue if revert conflicts), and emits\nbasic dev-loop metrics that feed Step 5's measurement window.\n\nSequence: this MUST land after Step 3 (relaxed strict-base), because\nthe auto-revert is the mitigation for the increased post-merge risk\nintroduced by Step 3.\n\nSingle PR via `make pr-open`.", "id": "dev-loop-step-4-post-merge-safety-net", "preserves": ["Existing develop-post-merge.yml job definitions and their pass/fail signal \u2014 Step 4 ADDS auto-revert, does not replace existing checks", "GITHUB_TOKEN scope: auto-revert needs `contents: write` and `pull-requests: write` \u2014 confirm permissions block in workflow declares these", "Issue and PR labels are namespaced (incident:*) so they don't conflict with existing labels", "Metrics artifacts are JSON, not human-readable text \u2014 Step 5 aggregator depends on machine-parseable shape"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/nightly.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml (release CI is unchanged)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 3 owns these)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-open, pr-fanout, pr-refresh targets (Steps 1 and 3 own these)"}, {"kind": "do_not_modify", "path_glob": "Makefile worktree-add, worktree-claim, worktree-pool-*, worktree-release targets (Step 2 owns these)"}, {"kind": "only_modify", "path_glob": ".github/workflows/develop-post-merge.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md (failure flow and dev-loop metrics sections)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (failure flow, dev-loop metrics, and Pre-approved Commands sections)"}, {"kind": "only_modify", "path_glob": "Makefile (dev-loop-metrics target only)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-auto-revert-fork-test-*.md (new fork validation evidence)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 4: develop-post-merge auto-revert + dev-loop metrics", "verifications": [{"command": "grep -A 3 'concurrency:' .github/workflows/develop-post-merge.yml | grep -c 'cancel-in-progress: false'", "description": "develop-post-merge.yml uses cancel-in-progress: false", "expected": ">= 1", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE '(auto-revert|revert.*failure)' .github/workflows/develop-post-merge.yml", "description": "Auto-revert job exists in develop-post-merge.yml", "expected": ">= 1", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -cE 'name: metrics' .github/workflows/develop-post-merge.yml", "description": "Metrics job emits an artifact named 'metrics'", "expected": ">= 1", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 3}, {"command": "make dev-loop-metrics 2>&1 | grep -cE 'P50|P95|red rate|runner'", "description": "make dev-loop-metrics produces a non-empty summary", "expected": ">= 4", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 4}, {"command": "grep -cE 'dev-loop-metrics|P50|P95|post-merge red|runner minutes' CLAUDE.md AGENTS.md", "description": "dev-loop-metrics is documented for agents and pre-approved in CLAUDE.md", "expected": ">= 3", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -cE 'incident:develop-red|auto-revert' CLAUDE.md AGENTS.md", "description": "Failure flow is documented in agent-facing docs", "expected": ">= 2", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 6}, {"command": "grep -cE 'Fork validation|failing SHA|revert PR|conflict issue|workflow run' _project/decisions/dev-loop-auto-revert-fork-test-*.md", "description": "PR description includes fork-test evidence", "expected": ">= 4", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": [], "notes": "Currently the workflow likely has a `concurrency:` block that\nmay cancel in-progress runs when a new push lands. Change:\n concurrency:\n group: develop-post-merge-${{ github.sha }}\n cancel-in-progress: false\nGroup key on `github.sha` (not `github.ref`) so each landed SHA\ngets its own concurrency lane and runs to completion. This is\nnecessary so fast-landing PRs don't cause the workflow to skip\nchecking intermediate SHAs.\n\nImplementation record (2026-04-30):\n - Updated `.github/workflows/develop-post-merge.yml` so the\n concurrency group is `develop-post-merge-${{ github.sha }}`.\n - Set `cancel-in-progress: false` so a later push to `develop`\n cannot cancel the post-merge safety check for an earlier landed SHA.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set cancel-in-progress: false on develop-post-merge.yml", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w1"], "notes": "New job `auto-revert-on-failure` triggered by\n`workflow_run.conclusion == 'failure'` (or as a continuation\njob in the same workflow with `if: failure()`). Steps:\n 1. Identify the failing SHA (`github.sha` of the post-merge run)\n 2. Identify the PR that introduced the SHA via\n `gh pr list --state merged --search \"\"`\n 3. Create branch `auto-revert/` from develop\n 4. `git revert --no-edit ` (single SHA, no merge commits)\n 5. If revert succeeds: push branch, open PR with title\n `Revert (#) \u2014 develop went red post-merge`,\n body includes failing run URL + originating PR link, label\n `incident:develop-red`, request review from PR author\n 6. If revert fails (conflict): open issue with same metadata,\n label `incident:develop-red-revert-conflict`, no PR\nUse `peter-evans/create-pull-request@v6` action or a small\ngh-cli script.\n\nImplementation record (2026-04-30):\n - Added `auto-revert-on-failure` to\n `.github/workflows/develop-post-merge.yml` with\n `needs: [lint, fast-test]` and an `always()` guard that runs only\n when one of those post-merge checks fails.\n - Added job-level permissions `contents: write`,\n `pull-requests: write`, and `issues: write`, which are the minimum\n scopes needed to push a revert branch and open either a PR or issue.\n - Used an inline `gh` script so the same job can identify the\n associated squash-merge PR, create idempotent incident labels, open\n an `auto-revert/` PR when `git revert --no-edit` succeeds,\n and request review from the originating PR author when possible.\n - Added conflict fallback: if the revert does not apply cleanly, the\n job aborts the revert and opens an\n `incident:develop-red-revert-conflict` issue with the failing SHA,\n run URL, originating PR URL, and attempted branch name.\n - W2 review issue addressed during implementation: the job uses\n `needs.*.result` rather than job-level `failure()` alone, so skipped\n or successful post-merge jobs do not accidentally trigger a revert.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add auto-revert PR generation on red", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": [], "notes": "Add a `metrics` job that runs with `if: always()` for every push to\ndevelop, so both green and red post-merge runs emit a machine-readable\nrecord. Computes and writes JSON artifact:\n {\n \"merged_at\": \"\",\n \"pr_number\": N,\n \"pr_open_to_merged_seconds\": N,\n \"post_merge_red\": false,\n \"develop_red_detected_at\": null,\n \"auto_revert_pr_opened_at\": null,\n \"time_to_revert_pr_seconds\": null,\n \"next_green_develop_run_at\": null,\n \"time_to_recover_red_seconds\": null,\n \"conflict_on_merge\": ,\n \"orphaned_failed_pr\": false,\n \"ci_runner_minutes\": N\n }\nOn failure, the auto-revert job should update or emit the same shape\nwith `post_merge_red: true`, `develop_red_detected_at`, and\n`auto_revert_pr_opened_at`. `next_green_develop_run_at` and\n`time_to_recover_red_seconds` may be null at artifact creation time;\n`make dev-loop-metrics` must fill them by scanning later successful\ndevelop-post-merge runs. Artifact retention: 90 days (covers the\nStep 5 measurement window). One file per merge. Step 5 aggregates\nthese records plus the explicit PR audit sources named there.\n\nImplementation record (2026-04-30):\n - Added a `metrics` job to `.github/workflows/develop-post-merge.yml`\n with `if: always()` and `needs: [lint, fast-test,\n auto-revert-on-failure]`, so every develop push emits metrics after\n the selected post-merge checks and any auto-revert attempt finish.\n - The metrics job uses read-only `actions`/`contents` permissions,\n queries the associated squash-merge PR through GraphQL, and writes\n `dev-loop-metrics/.json`.\n - The JSON includes the Step 4 shape: merge timestamp, PR number,\n PR-open-to-merge seconds, post-merge red status, red detection time,\n auto-revert PR timestamp when one was opened, recovery placeholders,\n conflict/orphan flags, and summed completed-job runner minutes.\n - Added a 90-day artifact upload named `metrics` for Step 5's\n measurement window.\n - W3 review issue addressed during implementation: the metrics job\n has only read permissions, and the GitHub jobs query requests one\n bounded page (`per_page=100`) so the captured payload is valid JSON.\n - W3 review timing issue fixed: `develop_red_detected_at` is derived\n from the failed `lint`/`fast-test` job completion timestamp instead\n of the later metrics-job start time, so `time_to_revert_pr_seconds`\n cannot go negative when auto-revert finishes before metrics runs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit per-PR dev-loop metrics as workflow artifact", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w2"], "notes": "Add a section \"If develop goes red\":\n - The post-merge workflow detects the failure within ~5 min\n - It auto-opens a revert PR (label: incident:develop-red) OR\n an issue if revert conflicts\n - Original PR author gets review-requested on the revert\n - Agents do NOT need to monitor CI proactively\n - To repair: `make worktree-claim BRANCH=fix/`,\n fix, resubmit\nReplace any older \"monitor your PR until green\" guidance.\n\nImplementation record (2026-04-30):\n - Updated `CLAUDE.md` with an \"If develop goes red\" flow: the\n post-merge workflow opens an `incident:develop-red` revert PR, or an\n `incident:develop-red-revert-conflict` issue when the revert\n conflicts.\n - Updated `AGENTS.md` with the same operating guidance and removed\n the placeholder language that Step 4 would expand the safety net\n later.\n - Documented that agents do not need to monitor CI proactively after\n auto-merge; the generated revert PR or conflict issue is the alert.\n - Documented the repair path:\n `make worktree-claim BRANCH=fix/`, fix, preflight,\n and resubmit with `make pr-open`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the new failure flow in CLAUDE.md and AGENTS.md", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w3"], "notes": "`make dev-loop-metrics` downloads the last N (default 30) days\nof metrics artifacts and prints a summary:\n - P50/P95 PR-to-merged time\n - Post-merge red rate\n - Conflict rate\n - Runner minutes total\nUsed by Step 5 reviewer (manual or scripted) to check thresholds.\nImplementation: `gh run list --workflow develop-post-merge.yml\n--json conclusion,databaseId | gh run download -n metrics`,\nthen aggregate the JSON files.\nDocument this target in CLAUDE.md and AGENTS.md near the post-merge\nsafety-net / dev-loop workflow text. In CLAUDE.md, add it to the\npre-approved read-only PR/Worktree or metrics bucket; the command\nreads workflow artifacts through `gh` and does not mutate repo state.\n\nImplementation record (2026-04-30):\n - Added `make dev-loop-metrics` with `DEV_LOOP_METRICS_DAYS` (default\n 30) and `DEV_LOOP_METRICS_LIMIT` (default 100). The target lists\n recent `develop-post-merge.yml` push runs on `develop`, downloads\n available `metrics` artifacts, and aggregates their JSON files.\n - The summary prints metrics artifact count, P50/P95 PR-to-merged\n seconds, post-merge red rate, conflict rate, and total runner\n minutes. If no artifacts exist yet, it still prints a non-empty\n `n/a` summary instead of failing.\n - Documented the target in `CLAUDE.md` and `AGENTS.md` near the\n post-merge safety-net flow.\n - Added `make dev-loop-metrics` to the CLAUDE.md pre-approved\n read-only PR/Worktree command bucket because it only reads workflow\n artifacts through `gh`.\n - W5 review issues addressed during implementation: the target handles\n an empty metrics-artifact window without failing, and P50/P95 use\n nearest-rank percentiles instead of lower-rank indexing.\n - Final Step 4 verification issue addressed: the summary prints\n `runner minutes total` with lowercase `runner` so the TODO\n verification command passes as written.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make dev-loop-metrics target for ad-hoc inspection", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Before merging this PR, validate the auto-revert path end-to-end\nin a fork: cherry-pick a commit that fails the post-merge tests,\npush to a fork's develop, confirm the workflow opens a revert PR\nwith correct title/body/labels. If revert conflicts, confirm the\nfallback issue is opened correctly.\nDocument the test in PR description AND write\n`_project/decisions/dev-loop-auto-revert-fork-test-.md` with\nfork repo, failing SHA, workflow run URL, revert PR URL or conflict\nissue URL, and screenshots/CLI excerpts sufficient for review.\n\nImplementation record (2026-04-30):\n - A true same-account fork was impossible: GitHub rejected\n `joeharris76/BenchBox` -> `joeharris76/BenchBox-dev-loop-smoke-*`\n because one user account cannot own both a parent and a fork. No\n organization target was available, so W6 used public disposable\n mirror `joeharris76/benchbox-dev-loop-smoke-20260430`.\n - Configured the mirror and canonical repository workflow setting with\n `default_workflow_permissions=read` and\n `can_approve_pull_request_reviews=true`, which is required for\n `GITHUB_TOKEN` to create pull requests when the job requests\n `pull-requests: write`.\n - First smoke SHA `35dd478645e5ed34b5b21d33e625bab8e88ddef5`\n exposed a workflow-file edge case: GitHub rejected the revert branch\n push because the reverted commit changed `.github/workflows/*` and\n `GITHUB_TOKEN` lacks `workflows` permission.\n - W6 review issue addressed in `.github/workflows/develop-post-merge.yml`:\n push failures and `gh pr create` failures now open a manual-action\n issue labeled `incident:develop-red-revert-conflict` instead of\n failing without an incident artifact.\n - Final smoke SHA `cff5c591941c989cd0c3003d552142d93489e238`\n validated the normal path: run\n `https://github.com/joeharris76/benchbox-dev-loop-smoke-20260430/actions/runs/25189439117`\n opened revert PR\n `https://github.com/joeharris76/benchbox-dev-loop-smoke-20260430/pull/1`\n from `auto-revert/cff5c591941c` to `develop` with label\n `incident:develop-red`.\n - Evidence written to\n `_project/decisions/dev-loop-auto-revert-fork-test-2026-04-30.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke test against a known-bad SHA in a fork", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "wait for the data", "dont": "begin Step 6 planning during the measurement window", "why": "that primes the conclusion"}, {"INSTEAD": "if thresholds were wrong, that's a separate REASSESS conclusion, not a silent edit", "dont": "redefine thresholds during evaluation", "why": "that's confirmation bias dressed as analysis"}, {"INSTEAD": "wait the full window", "dont": "shorten the measurement window because it feels long", "why": "30 days captures within-month variability (releases, vacations, etc.) that shorter windows miss"}, {"INSTEAD": "a verbal 'we measured and it was fine' loses information", "dont": "skip writing the decision record", "why": "future re-evaluation needs to know what was decided and why"}], "approach": "This is a deliberately low-effort TODO. The work is \"wait, then look\nat numbers and write down the answer.\" The goal is a forcing\nfunction: without this gate, Step 6 might get built reflexively\n(\"we said we would\") rather than because it's needed.\n\nBe honest about what the metrics show. If the numbers are clearly\nbelow threshold (e.g., P95 PR-to-merged is 8 minutes), DO NOT BUILD\nis the correct call. If they're clearly above, BUILD. If they're\nambiguous, REASSESS \u2014 don't force a binary decision on noisy data.\n\nFor \"sustained\" interpretation: use 70% of days above threshold as\nthe rule. One bad week shouldn't trigger a queue; sustained pain\nshould. Adjust if 70% turns out wrong empirically.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-5-measurement-window", "id": 18, "reason": "That's Step 6, gated on this step's recommendation. Keeping the decision/build separation prevents 'might as well build it while I'm here.'", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Explicitly named as Step 6, which is already an open tracked item gated on Step 5's recommendation. (tracked by dev-loop-step-6-queue-decision-gate)", "summary": "Building infrastructure based on measurement findings"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-5-measurement-window", "id": 19, "reason": "If measurement reveals we want continuous tracking, that's a follow-up. For Step 5, the 90-day artifact retention from Step 4 is sufficient.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Speculative follow-up gated on 30-day measurement findings; parent Step 5 item is still open/active, so the condition hasn't been evaluated. (tracked by dev-loop-step-5-measurement-window)", "summary": "Long-term metrics retention beyond 90-day artifact window"}], "deps": ["dev-loop-step-1-local-loop-cleanup", "dev-loop-step-2-worktree-pool", "dev-loop-step-3-ci-rebalance-and-strict-base-off", "dev-loop-step-3a-path-based-ci-skip", "dev-loop-step-4-post-merge-safety-net"], "description": "Decision-gate item that decides whether Step 6 (queue) is needed.\nRuns after Steps 1-4 plus Step 3a have been merged for ~30 days.\nAggregates the metrics emitted by Step 4 against pre-defined thresholds\nand produces a written go/no-go recommendation.\n\nNo code changes during the window itself; the deliverable is a\ndecision record. If all thresholds are below trigger, Step 6 is\ncancelled and the simplification effort is closed. If one or more\ntrigger, Step 6 is unblocked.\n\nThis TODO is intentionally LOW-effort but HIGH-leverage \u2014 it prevents\nbuilding queue infrastructure for a problem that doesn't exist at\nscale.\n", "id": "dev-loop-step-5-measurement-window", "preserves": ["Existing dev-loop metrics emission \u2014 if Step 4 metrics break mid-window, fix forward; do not roll back", "Steps 1-4 implementation untouched during measurement \u2014 no code changes during the window", "Threshold definitions from this TODO \u2014 if measurement is ambiguous, use REASSESS rather than redefining thresholds post-hoc"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml, pr.yml, develop-post-merge.yml, nightly.yml, release.yml (no code changes during the measurement window)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (no further changes during the window)"}, {"kind": "do_not_modify", "path_glob": "Makefile (no target changes during the window)"}, {"kind": "do_not_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-{0,1,2,3,3a,4}-*.yaml (Steps 1-4 plus Step 3a implementations are frozen during measurement)"}, {"kind": "do_not_modify", "path_glob": "tests/conftest.py (Step 1 helper stays as-is)"}, {"kind": "only_modify", "path_glob": "AGENTS.md (one paragraph)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (one paragraph)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml (status/deps only, or move to DONE if DO NOT BUILD)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-measurement-*.md (new)"}], "state": "active", "title": "Dev-loop simplification \u2014 Step 5: 30-day measurement window with queue decision thresholds", "verifications": [{"command": "ls _project/decisions/dev-loop-measurement-*.md && grep -cE '^### (P95|Post-merge|Time-to-recover|Cross-PR|Orphaned|Runner)' _project/decisions/dev-loop-measurement-*.md", "description": "Measurement record exists with all six metrics evaluated", "expected": ">= 6", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE '^Recommendation: (BUILD QUEUE|DO NOT BUILD QUEUE|REASSESS)' _project/decisions/dev-loop-measurement-*.md", "description": "Recommendation section is present and unambiguous", "expected": "1", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -E '^status:' _project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml", "description": "Step 6 TODO status updated to reflect decision", "expected": "Either 'Not Started' (if BUILD), 'Blocked' (if REASSESS), or moved to DONE (if DO NOT BUILD)", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -cE 'Dev-loop status \\(as of|P95 PR-to-merged|post-merge red rate|BUILD QUEUE|DO NOT BUILD QUEUE|REASSESS' CLAUDE.md AGENTS.md", "description": "Agent docs include current dev-loop status summary", "expected": ">= 3", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": [], "notes": "The measurement window is 30 days of normal usage AFTER both\nStep 3a and Step 4 have landed. Prior data is contaminated by the\nold workflow. Mark this TODO Blocked until that date.\n\nTo start the timer: when both Step 3a and Step 4 PRs have merged,\nset the window start timestamp to the later of `step3a_merged_at`\nand `step4_merged_at`, comment on this TODO with the merge dates,\nand update `metadata.due_date` to start_date + 30 calendar days.\n\nImplementation record (2026-04-30):\n - Step 3a PR #80 merged to `develop` at `2026-04-30T20:36:27Z`.\n - Step 4 PR #81 merged to `develop` at `2026-04-30T21:33:45Z`.\n - Step 4 is the later prerequisite, so the measurement window\n starts at `2026-04-30T21:33:45Z`.\n - Set `metadata.due_date` to `2026-05-30`; the existing date\n remains correct because Step 4 merged after Step 3a.\n - Set this TODO status to `Blocked` and left W1 `in_progress`\n until the due date. No code or workflow changes are allowed\n during the measurement window.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "in_progress", "summary": "Wait for 30 calendar days after Step 3a and Step 4 have both merged before starting w2", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w1"], "notes": "Run `make dev-loop-metrics DEV_LOOP_METRICS_DAYS=30\nDEV_LOOP_METRICS_LIMIT=500`. The target downloads `metrics`\nartifacts from `develop-post-merge.yml` runs in the window and\nprints PR-to-merge P50/P95, post-merge red rate, conflict rate,\nand runner-minutes total. Capture stdout to the measurement\nrecord below.\n\nOutput is plain text only; the target does not emit JSON. To\ncompute additional aggregates beyond the printed lines, download\nthe raw artifacts and aggregate with `jq`:\n\n gh run list --workflow develop-post-merge.yml --branch develop \\\n --event push --created \">=\" --limit 500 \\\n --json databaseId --jq '.[].databaseId' \\\n | xargs -I{} gh run download {} -n metrics -D /tmp/metrics/{}\n jq -s '...' /tmp/metrics/*/*.json\n\nIf artifact coverage is below 80% of merges (i.e., emission\nfailures), extend the window 7 days and retry.\n\nCross-run derivation needed for Step 4 placeholder fields:\n`next_green_develop_run_at` and `time_to_recover_red_seconds` are\nemitted as `null` in each per-SHA metrics file because recovery\ncan only be observed on a *later* SHA's run. W2 must pair each\n`post_merge_red=true` artifact with the next chronologically\nnewer artifact whose `lint` and `fast-test` jobs succeeded, and\nderive `time_to_recover_red_seconds` from the gap between\n`develop_red_detected_at` and the next-green run's `merged_at`.\nStore the derived series in the measurement record alongside the\nraw printout.\n\nMetrics must combine:\n - Step 4 metrics artifacts (PR open->merged, post-merge red,\n time-to-revert-PR, runner minutes; recovery time derived as\n above)\n - PR audit for the same window:\n `gh pr list --state all --base develop --search \"updated:>=\"`\n and labels/comments containing `queue:conflict`,\n `conflict`, `pr-refresh`, `auto-revert`, or\n `incident:develop-red`\n - Post-merge workflow runs:\n `gh run list --workflow develop-post-merge.yml --created\n .. --json conclusion,createdAt,updatedAt,url`\nSave raw aggregation to `_project/decisions/\ndev-loop-measurement-.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Aggregate dev-loop metrics for the 30-day window", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w2"], "notes": "Apply the trigger matrix from the plan:\n | Metric | Trigger threshold |\n | P95 PR open \u2192 merged | sustained > 1h |\n | Post-merge red rate on develop | sustained > 5% |\n | Time-to-recover red develop (P95) | > 30 min |\n | Cross-PR conflict repairs / week | > 2/week |\n | Orphaned failed PRs (no resub <24h) | any pattern |\n | Runner minutes total | budget signal |\nFor each metric: PASS (below trigger) / FAIL (at or above).\n\"Sustained\" means >= 70% of days in the window above threshold;\nnot just one outlier week.\n\nData-source rules:\n - Use Step 4 artifacts for PR open->merged, post-merge red,\n time-to-revert-PR, and runner minutes.\n - Use the cross-run derivation from w2 for time-to-recover red\n develop. The per-SHA `time_to_recover_red_seconds` field is\n emitted as `null` by Step 4 by design; the value is computed\n in w2 by pairing red runs with the next green run.\n - Use the PR audit from w2 for cross-PR conflict repairs/week.\n - Count orphaned failed PRs from PRs labeled/commented as failed,\n conflicted, auto-reverted, or develop-red with no new commit,\n review response, or replacement PR within 24h.\n - If a metric cannot be computed because Step 4 did not emit the\n required fields, mark REASSESS and create a follow-up to fix\n metrics emission; do not silently drop the metric.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Evaluate against queue-trigger thresholds", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w3"], "notes": "Single markdown file appended to the measurement record from w2.\nSections:\n - Window: to \n - Per-metric PASS/FAIL with raw numbers\n - Recommendation: BUILD QUEUE | DO NOT BUILD QUEUE | REASSESS\n - If BUILD: which queue path (native vs steward), rationale\n tied to Step 0 w6 finding\n - If DO NOT BUILD: cancel Step 6 TODO, close the simplification\n effort\n - If REASSESS: extend window 30 days, identify which\n measurements were ambiguous, and what would resolve them\nCommit the record. Update Step 6 TODO status (unblock or close).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Write decision record with go/no-go for Step 6", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w4"], "notes": "Add a one-paragraph \"Dev-loop status (as of )\" note in\nboth files that documents:\n - Whether the queue is being built or not\n - Current P95 PR-to-merged time\n - Current post-merge red rate\nFuture agents read this to understand the state of the merge\nmachinery without re-running the analysis.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update CLAUDE.md / AGENTS.md with measurement findings", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "create exactly one selected-path TODO", "dont": "add both Path A and Path B implementation units to this TODO", "why": "the flat work DAG cannot mark the unchosen path skipped"}, {"INSTEAD": "keep queue light, heavy stays nightly", "dont": "add full matrix to queue-tier CI", "why": "that's exactly the over-engineering Steps 3-5 prevented"}, {"INSTEAD": "premature start wastes effort", "dont": "begin queue work without Step 5 saying BUILD", "why": "Step 5 might say DO NOT BUILD"}, {"INSTEAD": "both protect different failure modes", "dont": "remove develop-post-merge.yml just because the queue runs queue-tier CI", "why": "queue-tier and post-merge check different SHAs"}, {"INSTEAD": "`pull_request_target` is forbidden anywhere in steward workflows", "dont": "skip trust lanes", "why": "public repos with write-token automation are an exfiltration risk"}, {"INSTEAD": "use App with scoped permissions", "dont": "use a long-lived PAT for the steward", "why": "rotation burden and audit trail are worse than a GitHub App"}], "approach": "Lock path selection first, then generate the selected implementation TODO.\nDo not put both implementations into this item's `work[]`; the local TODO\nmodel cannot represent branch choice without a skipped status. If the\ngenerated TODO needs to differ from the locked spec, cite the Step 5\nmeasurement or Step 0 availability evidence that forced the change.\n\nPath A is significantly simpler if available. Path B is a real software\nproject (workflow + bot credentials + state machine) and needs fork smoke\ntesting before enabling on develop.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-6-queue-decision-gate", "id": 20, "reason": "Only one selected-path TODO should be created. The rejected path is documented in the decision record to avoid a deadlocked flat DAG.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: This is the explicitly rejected/unselected queue path; by design only the selected Step 6 path (already open) is tracked.", "summary": "Unselected queue implementation path"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-6-queue-decision-gate", "id": 21, "reason": "Org transfer is a separate decision with broader implications (URLs, integrations, billing). If Step 5 says BUILD and Path A requires transfer, escalate that decision separately.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Native MQ/org-transfer escalation is a conditional sub-note of the still-open, active parent queue-decision-gate item. (tracked by dev-loop-step-6-queue-decision-gate)", "summary": "Native MQ migration via org transfer"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-6-queue-decision-gate", "id": 22, "reason": "Labels + comments are sufficient for one-person ops. Add dashboard if queue volume justifies it.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: This deferral's own from_item IS the open tracked item dev-loop-step-6-queue-decision-gate; not a separate untracked task.", "summary": "Queue dashboard / observability beyond labels"}], "deps": ["dev-loop-step-5-measurement-window"], "description": "Deferred queue gate and spec handoff. This TODO does NOT implement a\nqueue directly. If Step 5 measurement returns BUILD QUEUE, this item\nchooses the viable path and creates exactly one follow-up implementation\nTODO from the locked spec below:\n\n - Path A: native GitHub Merge Queue\n - Path B: BenchBox Merge Steward\n\nThis shape is intentional. The TODO work-unit graph only supports\n`pending`, `in_progress`, and `done`, so mutually exclusive Path A/Path B\nimplementation units cannot live in the same flat DAG without deadlocking\ndownstream work. The generated follow-up TODO becomes the executable\nimplementation item; this gate records the path and prevents re-debating\nsettled design decisions.\n\nStatus starts as Blocked (on Step 5). Three resolution paths:\n - Step 5 -> DO NOT BUILD: this TODO moves to DONE with completion note\n \"queue not needed; closed by Step 5 measurement \"\n - Step 5 -> BUILD QUEUE: this TODO unblocks, status -> Not Started,\n work units below create the selected implementation TODO\n - Step 5 -> REASSESS: stays Blocked, deps point to next measurement\n", "id": "dev-loop-step-6-queue-decision-gate", "preserves": ["Existing pr-open semantics during deprecation period \u2014 do not break in-flight workflows mid-rollout", "Heavy validation stays in nightly \u2014 do not move it back to the queue path", "Only one path implementation TODO exists \u2014 no flat-DAG deadlock from mutually exclusive Path A/Path B work", "Step 3's lightweight PR-tier CI \u2014 the queue does not promote develop to release-grade gating", "develop-post-merge.yml safety net from Step 4 remains active even with a queue"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml, develop-post-merge.yml, nightly.yml, release.yml (implementation TODO territory)"}, {"kind": "do_not_modify", "path_glob": "CLAUDE.md, AGENTS.md, CONTRIBUTING.md (implementation TODO will own queue-flow docs and trust lanes)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (implementation TODO toggles merge-queue rule)"}, {"kind": "do_not_modify", "path_glob": "Makefile (implementation TODO will own pr-submit and queue handoff)"}, {"kind": "do_not_modify", "path_glob": "tests/, .pre-commit-config.yaml (orthogonal to queue path selection)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml (status/completion updates)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6a-native-merge-queue.yaml (new, Path A only)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6b-merge-steward.yaml (new, Path B only)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-queue-path-*.md (new decision record)"}], "state": "active", "title": "Dev-loop simplification \u2014 Step 6: queue implementation gate \u2014 DEFERRED until Step 5 justifies", "verifications": [{"command": "grep -cE '^Selected path: (A|B) \\(' _project/decisions/dev-loop-queue-path-*.md", "description": "Path A or Path B chosen with rationale recorded", "expected": "1", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 1}, {"command": "test \"$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml 2>/dev/null | wc -l | tr -d ' ')\" = \"1\" && echo ok", "description": "Exactly one selected-path implementation TODO exists", "expected": "ok", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 2}, {"command": "f=$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml); uv run --project _project/scripts -- python _project/scripts/validate_todo.py \"$f\" --strict", "description": "Generated implementation TODO validates strictly", "expected": "valid", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 3}, {"command": "f=$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml); ! grep -qE 'ONLY if|mark this .* skipped|needs: \\[w2, w3\\]' \"$f\" && echo ok", "description": "Generated implementation TODO has no mutually exclusive DAG branches", "expected": "ok", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 4}, {"command": "f=$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml); grep -cE 'queue-tier|pr-submit|queue:approved|queue:abandoned|pull_request_target|develop-post-merge|CLAUDE.md|AGENTS.md|CONTRIBUTING.md|Pre-approved' \"$f\"", "description": "Common queue policy is present in generated TODO", "expected": ">= 10", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": [], "notes": "Re-read Step 0 w6 finding (native MQ availability) and the Step 5\ndecision record. Path A (native MQ) requires repo/ruleset support for\nGitHub Merge Queue. Path B (steward) is the fallback.\n\nIf Step 0 confirmed user-owned repo and GitHub APIs still do not\nexpose merge-queue rules for this repo, Path B is mandatory unless\nthe user has since transferred the repo to an org. If org transfer\nis on the table, evaluate it as a separate decision (one-time\nfriction vs ongoing steward maintenance), not inside implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Confirm BUILD decision and choose native MQ vs steward", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w1"], "notes": "Create `_project/decisions/dev-loop-queue-path-.md` with:\n - Selected path: A or B\n - Step 0 evidence used\n - Step 5 BUILD QUEUE metrics that justified starting\n - Why the unselected path was rejected\n - Whether org transfer was considered and why it is/is not part\n of the queue rollout\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Write queue path decision record", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w2"], "notes": "Create ONE of these files, not both:\n - Path A: `_project/TODO/main/planning/dev-loop-step-6a-native-merge-queue.yaml`\n - Path B: `_project/TODO/main/planning/dev-loop-step-6b-merge-steward.yaml`\n\nThe generated TODO must be a normal implementation item with flat\n`work[]` units and no mutually exclusive branches.\n\nPath A locked spec:\n 1. Enable \"Require merge queue\" on develop branch protection or\n the ruleset that targets develop.\n 2. Add `merge_group:` trigger to the workflows that report\n queue-tier checks.\n 3. Keep Step 3's lightweight PR-tier checks; queue-tier is also\n light (lint + Ubuntu 3.12 fast tests), not release-grade.\n 4. Confirm `gh pr merge --auto --squash` remains the submission\n mechanism for develop PRs.\n 5. Update CLAUDE.md/AGENTS.md to describe the queue-managed flow.\n 6. Update CLAUDE.md pre-approved command buckets for any new\n queue-facing commands or labels.\n\nPath B locked spec:\n 1. Implement BenchBox Merge Steward as a single-concurrency\n workflow triggered by `queue:ready`.\n 2. Concurrency group: `benchbox-develop-integrator`,\n `cancel-in-progress: false`.\n 3. Loop oldest-first by `queue:ready` label timestamp.\n 4. For each PR: reset to current origin/develop, apply PR branch\n onto that tip, run queue-tier checks, then merge or eject.\n 5. Conflict: label `queue:conflict`, comment with files, continue.\n 6. Test fail: label `queue:failed`, comment with job, continue.\n 7. Use a GitHub App with `contents: write` and\n `pull_requests: write`; no long-lived PAT.\n 8. Smoke test in a fork with clean, conflicting, failing, and\n untrusted PRs before enabling on develop.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Create exactly one selected-path implementation TODO", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w3"], "notes": "The selected implementation TODO must include these common work units\nor guardrails:\n - Queue-tier CI is light: lint + Ubuntu 3.12 fast tests only.\n Heavy validation stays in nightly.\n - Add `make pr-submit` as pr-open + queue handoff + hygiene:\n refuse dirty worktree, capture immutable head SHA, skip drafts,\n and bypass auto-queue for workflow/steward changes.\n - Keep `make pr-open` as a deprecated alias for one release.\n - Update CLAUDE.md and AGENTS.md with the submitted/queued flow,\n including when agents use `make pr-submit` vs deprecated\n `make pr-open`, whether they should wait, and how queue\n ejections are repaired.\n - Update CLAUDE.md Pre-approved Commands so `make pr-submit`\n and any safe queue-status commands are explicitly classified;\n queue admin/reset commands must not be hidden in broad wildcards.\n - Define trusted vs untrusted lanes. Untrusted PRs require manual\n `queue:approved`; `pull_request_target` is forbidden in steward\n workflows.\n - Update CONTRIBUTING.md with the external contributor path:\n untrusted PRs require manual `queue:approved`; PR-controlled\n workflow/steward changes bypass auto-queue.\n - Define bounce ownership and retry budget: original author owns\n `queue:conflict` / `queue:failed`; 3 ejections max, then\n `queue:abandoned`.\n - Keep develop-post-merge.yml active as Step 4's safety net.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add common queue policy requirements to the selected TODO", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w2", "w3", "w4"], "notes": "Run strict validation on the generated TODO and `todo_cli.py\ncheck-graph`. The generated TODO is the implementation item to work\nnext. This gate is complete once:\n - Exactly one implementation TODO exists\n - The decision record exists\n - The generated TODO is valid, graph-safe, and has runnable\n verification commands\n - The unselected path is documented as rejected/deferred\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Validate the generated TODO and close this gate", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "revert alias normalization or make the run platform key the suffixed `-df` name to work around the mismatch (breaks result labeling and `platforms status/enable/install`).", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Usability / Config", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-07T00:00:00Z", "deferrals": [], "deps": [], "description": "RESOLVED by merged PR #1054 \"fix(cli): key DataFrame platform options by base\nname so run can reach them\" (branch fix/dataframe-platform-option-keying,\nmerged 2026-07-08) plus review follow-up PR #1062 \"fix: forward DataFrame\nplatform options to adapter construction (#1054 review)\". This TODO fell out\nof the PR #1042 (fix/datafusion-df-mode-erasure) audit; the in-flight session\ntask_ce5bd997 that was implementing it landed as #1054. No re-implementation\nwas needed. See the \"resolution\" block below for the empirical confirmation.\n\n---\n\nAll DataFrame-platform `--platform-option` specs are registered under\nsuffixed keys (`datafusion-df`, `polars-df`, `pandas-df`, `modin-df`,\n`cudf-df`, `dask-df`) in the `_OPTION_SPEC_ROWS` table in\nbenchbox/platforms/__init__.py (~lines 768-786). But `benchbox run` parses\nplatform options with the POST-alias platform key:\nbenchbox/cli/commands/run.py `_parse_plat_bench_options` (:584) calls\n`PlatformHookRegistry.parse_options(s.platform_key, ...)` where\n`s.platform_key` has already been normalized by PLATFORM_ALIASES in\nbenchbox/cli/platform.py (datafusion-df -> datafusion, polars-df -> polars,\netc.). The specs are registered under the suffixed key, so the base-name\nlookup finds nothing.\n\nResult: `benchbox run --platform datafusion-df --platform-option\ntarget_partitions=4` fails with\n\"Unknown platform option 'target_partitions' for platform 'datafusion'\".\nVerified empirically on develop 2026-07-07. Every DataFrame platform option\n(streaming, rechunk, n_rows, dtype_backend, engine, device_id,\ntarget_partitions, repartition_joins, parquet_pushdown, batch_size,\nmemory_limit, temp_dir, n_workers, ...) is unreachable from the run CLI.\n\nSame key mismatch affects the sibling lookups in run.py:\n`PlatformHookRegistry.get_default_options(s.platform_key)` (:645) and\n`list_option_specs(platform_key)` (:658). Note an asymmetry to preserve or\nreconcile: `_describe_platform_options` (:453-455) lowercases the raw name\nWITHOUT alias normalization, so `--describe-platform-options datafusion-df`\nDOES list the options \u2014 while `run --platform datafusion-df` then rejects\nthem. Describe and parse must agree after the fix.\n\nDistinct from qpc-16 (that TODO is about plan_max_depth /\nplan_capture_timeout_seconds in benchbox/core/results/platform_options.py, a\ndifferent allowlist). This one is the cli.platform_hooks PlatformHookRegistry\nused by `run`.\n\nIN FLIGHT: a background session (spawned task task_ce5bd997) is actively\nimplementing this fix as of 2026-07-07. If that lands a PR, complete this\nTODO against it rather than starting fresh work.\n\nContext: PR #1042 (fix/datafusion-df-mode-erasure) added\n`_apply_dataframe_suffix_mode` in run.py so `-df` names resolve DataFrame\nmode while the platform key stays the base name by design. DO NOT undo that;\nthe option-spec fix must keep the base-name platform key and instead make\nthe specs reachable under it.", "id": "df-suffix-platform-options-unreachable", "preserves": ["PR #1042 behavior: `-df` names resolve DataFrame mode and the platform key stays the base name.", "Registered default values for DataFrame options unchanged when the option is not set.", "SQL-platform --platform-option behavior (e.g. databricks, postgresql) unchanged."], "prior_art": [{"concept": "benchbox/cli/commands/run.py _parse_plat_bench_options:584 \u2014 parse_options(s.platform_key) with normalized base name (the consumer that fails)", "decision": "reuse", "path": "benchbox/cli/commands/run.py"}, {"concept": "benchbox/cli/platform.py PLATFORM_ALIASES:55-63 + _apply_dataframe_suffix_mode (PR #1042) \u2014 where `-df` is normalized to base name; the option key must match this", "decision": "reuse", "path": "benchbox/cli/platform.py"}, {"concept": "benchbox/core/hooks/platform_hooks.py PlatformHookRegistry.register_option_specs / parse_options / get_default_options / describe_options \u2014 the registry keying", "decision": "reuse", "path": "benchbox/core/hooks/platform_hooks.py"}, {"concept": "benchbox/platforms/__init__.py _OPTION_SPEC_ROWS:768-786 \u2014 DataFrame option specs keyed by `-df`; EXTEND: re-key to base name or register under both", "decision": "extend", "path": "benchbox/platforms/__init__.py"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/hooks/platform_hooks.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "DataFrame -df platform options are unreachable from `benchbox run`", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/test_run_command_branches.py tests/unit/cli/test_platform.py -q -k 'option or dataframe or df'", "description": "DataFrame -df platform options are accepted by run and reach adapter config", "expected": "all pass, including new datafusion-df / polars-df --platform-option acceptance tests", "item_id": "df-suffix-platform-options-unreachable", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reproduce and pin: assert `run --platform datafusion-df --platform-option target_partitions=4 --dry-run` currently rejects the option; capture the failing message as the regression baseline.", "wid": "w1"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide keying: re-key DataFrame specs in _OPTION_SPEC_ROWS to base names, or register under both keys. Confirm adapter_factory._get_dataframe_adapter config passthrough unaffected.", "wid": "w2"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement chosen keying so parse_options/get_default_options/list_option_specs resolve DataFrame options under the base key used by run.py.", "wid": "w3"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile describe/parse asymmetry: --describe-platform-options and run --platform-option must accept the same names for `-df` and base names.", "wid": "w4"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regression tests: datafusion-df target_partitions parses and reaches adapter config; base and suffixed paths agree; cover a second DataFrame platform (polars-df streaming).", "wid": "w5"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Adversarial `code` review; fix Critical/Required findings; re-verify.", "wid": "w6"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Open PR to develop.", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use the parameter override pattern from tpch/dataframe_queries.py", "dont": "hardcode date ranges", "why": "FlightData uses parameterized {start_date}/{end_date} based on available data"}, {"INSTEAD": "implement using native DataFrame API (col/lit/join/groupby)", "dont": "implement queries as SQL strings passed to DataFrame engines", "why": "that defeats the purpose of DataFrame benchmarking"}], "approach": "Follow the read_primitives/dataframe_queries.py pattern: single file with\nexpression_impl and pandas_impl functions for each query, registered in a\nQueryRegistry. Use the get_dataframe_queries() fallback method on the benchmark\nclass (no hard-coded dispatch needed).\n\nReference benchbox/core/flightdata/queries.py for the SQL source - each query's\nSQL maps directly to DataFrame operations. Start with the 5 on-time performance\nqueries (simplest aggregation patterns), then delay analysis, route analytics,\ntemporal patterns, and carrier comparisons.\n\nUse DataFrameQuery categories: JOIN, AGGREGATE, GROUP_BY, FILTER, SORT for\nappropriate classification. Parameterize date ranges via a module-level defaults\ndict with set_parameter_overrides() pattern from tpch/dataframe_queries.py.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-flightdata", "id": 381, "reason": "The get_dataframe_queries() fallback method is sufficient. Hard-coded dispatch only needed for stream_id permutation support (TPC-H/TPC-DS compliance).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason states the existing fallback method is sufficient and hard-coded dispatch is only needed for out-of-scope stream_id permutation support.", "summary": "Add FlightData dataframe queries to hard-coded dispatch in dataframe_runner.py"}], "deps": [], "description": "Add dual-family DataFrame query implementations (expression + pandas) for all 20\nFlightData OLAP queries, enabling execution on Polars, PySpark, DataFusion, Pandas,\nModin, cuDF, and Dask platforms.\n\nFlightData is a 3-table benchmark (flights fact table + airlines/airports dimensions)\nwith 20 queries across 5 categories: on-time performance (5), delay analysis (4),\nroute analytics (4), temporal patterns (4), and carrier comparisons (3). The queries\nuse standard OLAP patterns - GROUP BY with aggregations, LEFT JOINs to dimension\ntables, CASE expressions, HAVING clauses, date filtering, and ROUND/AVG/SUM/COUNT.\n\nThis is one of the simpler benchmarks to port because the schema is small (3 tables),\njoins are straightforward (single-key LEFT JOINs to dimension tables), and query\npatterns are standard aggregation/filter operations that map cleanly to both\nexpression-family and pandas-family APIs.\n\nMotivation: FlightData uses real-world aviation data and is popular for OLAP\ndemos. DataFrame support enables benchmarking DataFrame platforms on realistic\nanalytics workloads beyond TPC-H/TPC-DS.", "id": "df-support-flightdata", "preserves": ["FlightDataBenchmark SQL execution path remains unchanged - all existing SQL queries and parameterized date ranges work identically", "FlightDataDownloader data generation produces files loadable by both SQL and DataFrame paths", "benchbox run --platform duckdb --benchmark flightdata still works (SQL mode unaffected)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/flightdata/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/flightdata/"}], "state": "done", "title": "Add DataFrame execution support to FlightData benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.flightdata.dataframe_queries import get_dataframe_queries; r = get_dataframe_queries(); print(f'{len(r)} queries registered'); assert len(r) == 20\"", "description": "All 20 DataFrame queries are registered in the QueryRegistry", "expected": "20 queries registered", "item_id": "df-support-flightdata", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['flightdata']['supports_dataframe'])\"", "description": "FlightData benchmark reports supports_dataframe=True", "expected": "True", "item_id": "df-support-flightdata", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/flightdata/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-flightdata", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-flightdata", "needs": [], "notes": "Create benchbox/core/flightdata/dataframe_queries.py with expression_impl\nfunctions for all 20 queries. Follow the pattern in read_primitives/dataframe_queries.py.\n\nKey translation patterns:\n- LEFT JOIN airlines/airports -> ctx.get_table(\"flights\").join(ctx.get_table(\"airlines\"), ...)\n- CASE WHEN -> col(\"x\").when(...).then(...).otherwise(...) (Polars) or F.when() (PySpark)\n- ROUND(AVG(...), 2) -> col(\"x\").mean().round(2)\n- date filtering -> col(\"flight_date\") >= lit(start_date)\n- HAVING COUNT(*) >= N -> filter after group_by\n- Integer division for dep_hour -> (col(\"crs_dep_time\") / lit(100)).cast(int)\n\nQueries use parameterized date ranges ({start_date}, {end_date}) - implement\na parameters dict similar to TPCH_DEFAULT_PARAMS in tpch/dataframe_queries.py.\n\nAll 20 queries should be implementable; no SQL-only constructs present.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for all 20 FlightData queries", "wid": "w1"}, {"evidence": null, "item_id": "df-support-flightdata", "needs": ["w1"], "notes": "Add pandas_impl functions to each DataFrameQuery in dataframe_queries.py.\n\nKey translation patterns:\n- LEFT JOIN -> pd.merge(flights, airlines, left_on=\"reporting_airline\", right_on=\"code\", how=\"left\")\n- CASE WHEN -> np.where() or df[\"col\"].apply()\n- GROUP BY + agg -> df.groupby([...], as_index=False).agg(...)\n- HAVING -> filter after groupby\n- ROUND -> .round(2)\n- ORDER BY ... LIMIT N -> .sort_values(...).head(N)\n- Integer division -> df[\"crs_dep_time\"] // 100\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create pandas-family implementations for all 20 FlightData queries", "wid": "w2"}, {"evidence": null, "item_id": "df-support-flightdata", "needs": ["w1"], "notes": "Integration steps:\n1. Add get_dataframe_queries() method to FlightDataBenchmark in benchmark.py\n that imports and returns the QueryRegistry from dataframe_queries.py\n2. Set supports_dataframe: True in benchmark_registry.py for flightdata entry\n3. Ensure FlightData data loader works with DataFrame platforms (CSV/Parquet loading)\n4. Add any skip lists if needed (unlikely - all queries are standard OLAP)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up FlightData benchmark class and registry for DataFrame dispatch", "wid": "w3"}, {"evidence": null, "item_id": "df-support-flightdata", "needs": ["w2", "w3"], "notes": "Create tests/unit/core/flightdata/test_dataframe_queries.py:\n- Test query registration (all 20 queries registered, correct IDs)\n- Test expression_impl functions execute without error on small synthetic data\n- Test pandas_impl functions execute without error on small synthetic data\n- Test result shape/columns match expectations\n- Test parameterized date ranges\n- Mark tests with @pytest.mark.fast where possible\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for FlightData DataFrame queries", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use descriptive variable names (e.g., info_type_genre, info_type_rating for the double it join in 12a)", "dont": "assume table aliases from SQL map to DataFrame variables", "why": "IMDB schema has self-referential patterns"}, {"INSTEAD": "focus on correct multi-join execution and performance measurement", "dont": "attempt to replicate SQL optimizer stress-testing in DataFrame mode", "why": "DataFrame APIs define explicit join order, there is no optimizer to stress"}, {"INSTEAD": "use chained .join() calls that the DataFrame engine can optimize", "dont": "use nested loops or iterative joins", "why": "that destroys performance"}], "approach": "Follow the read_primitives/dataframe_queries.py single-file pattern with QueryRegistry.\nUse get_dataframe_queries() fallback method on the benchmark class.\n\nStart with 4-5 table queries (w1) to establish the multi-join translation pattern,\nthen scale to 7-9 table queries. The central translation challenge is converting\nimplicit cross-joins (comma-separated FROM with WHERE conditions) into explicit\n.join() chains. Use the title table as the hub for most queries since most\njunction tables reference title.id.\n\nFor the expression family, the join pattern is:\n title.join(movie_companies, left_on=\"id\", right_on=\"movie_id\")\n .join(company_type, left_on=\"company_type_id\", right_on=\"id\")\n .filter(col(\"kind\") == lit(\"production companies\"))\n .select(col(\"title\").min().alias(\"movie_title\"))\n\nUse QueryCategory.JOIN for all queries, plus AGGREGATE and FILTER as appropriate.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-joinorder", "id": 382, "reason": "Research question, not implementation work. Could inform future variant generation for DataFrame join ordering.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly framed as a research question, not implementation work; best captured under the future-ideas backlog if ever pursued. (tracked by future-ideas-backlog)", "summary": "Benchmark join order sensitivity - measure if different .join() sequences change performance"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-joinorder", "id": 383, "reason": "The full JOB has 113 queries; current benchmark embeds 13 representative ones. Extending is separate scope.", "resolution": "promoted", "resolved_item": "add-remaining-job-queries-beyond-the-core-13", "resolved_reason": null, "summary": "Add remaining JOB queries beyond the core 13"}], "deps": [], "description": "Add dual-family DataFrame query implementations (expression + pandas) for all 13\nJoinOrder benchmark queries, enabling execution on Polars, PySpark, DataFusion,\nPandas, Modin, cuDF, and Dask platforms.\n\nJoinOrder is based on \"How Good Are Query Optimizers, Really?\" (VLDB 2015) and uses\na 21-table IMDB-like schema to stress-test query optimizer join ordering. The 13\nqueries (1a through 12a, plus 1b) feature complex multi-table joins (5-9 tables),\nLIKE predicates, IN clauses, cross-table filtering, and MIN aggregations.\n\nThis is the most challenging benchmark to port to DataFrames because:\n1. Multi-table joins (5-9 tables per query) require careful chaining of .join() calls\n2. Implicit cross-joins in FROM clauses (comma-separated tables with WHERE join\n conditions) must be translated to explicit .join() sequences\n3. LIKE patterns with wildcards need string method equivalents\n4. Join order matters for performance, and DataFrame APIs may not preserve the\n optimizer stress-testing intent - document this caveat\n\nThe purpose of DataFrame JoinOrder is NOT to test optimizer join ordering (DataFrames\ndefine explicit join order), but to measure multi-join execution performance across\nplatforms with identical join sequences.", "id": "df-support-joinorder", "preserves": ["JoinOrderBenchmark SQL execution path unchanged - embedded queries and file-loaded queries work identically", "JoinOrderGenerator synthetic data generation produces files loadable by both SQL and DataFrame paths", "get_queries_by_complexity() classification still works for SQL mode"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/joinorder/"}], "state": "done", "title": "Add DataFrame execution support to JoinOrder benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.joinorder.dataframe_queries import get_dataframe_queries; r = get_dataframe_queries(); print(f'{len(r)} queries registered'); assert len(r) == 13\"", "description": "All 13 DataFrame queries are registered", "expected": "13 queries registered", "item_id": "df-support-joinorder", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['joinorder']['supports_dataframe'])\"", "description": "JoinOrder benchmark reports supports_dataframe=True", "expected": "True", "item_id": "df-support-joinorder", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/joinorder/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-joinorder", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-joinorder", "needs": [], "notes": "Create benchbox/core/joinorder/dataframe_queries.py with a QueryRegistry.\nStart with the simpler queries that join 4-5 tables:\n\n- 1a: 5 tables (company_type, info_type, movie_companies, movie_info_idx, title)\n MIN aggregations, LIKE/NOT LIKE on mc.note, equality joins\n- 1b: Same structure as 1a with different predicates\n- 2a: 5 tables (company_name, keyword, movie_companies, movie_keyword, title)\n Equality joins, country_code filter\n- 3a: 4 tables (keyword, movie_info, movie_keyword, title)\n LIKE on keyword, IN clause on mi.info, production_year filter\n- 4a: 5 tables (info_type, keyword, movie_info_idx, movie_keyword, title)\n String comparison (mi_idx.info > '2.0'), production_year filter\n\nTranslation pattern for implicit cross-joins:\n FROM t1, t2, t3 WHERE t1.id = t2.fk AND t2.id = t3.fk\n -> t1.join(t2, left_on=\"id\", right_on=\"fk\").join(t3, left_on=\"id\", right_on=\"fk\")\n\nFor LIKE patterns: col(\"note\").str.contains(\"(co-production)\")\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for queries 1a-4a (4-5 table joins)", "wid": "w1"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w1"], "notes": "More complex queries with 5-8 table joins:\n\n- 5a: 5 tables - European movies with theatrical release notes\n- 6a: 5 tables - Superhero movies with specific actor (LIKE + IN)\n- 7a: 8 tables (aka_name, cast_info, info_type, link_type, movie_link, name,\n person_info, title) - most complex join chain, BETWEEN on production_year,\n gender filter, name_pcode_cf LIKE\n- 8a: 7 tables - Japanese dubbed movies, role_type filter, NOT LIKE\n- 9a: 8 tables - American voice actresses, BETWEEN on year, IN on ci.note,\n person_role_id join to char_name\n\nKey challenge: 7a and 9a have 8-table joins requiring careful join sequencing.\nChoose join order that matches natural FK relationships in the IMDB schema.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for queries 5a-9a (5-8 table joins)", "wid": "w2"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w1"], "notes": "The most complex queries:\n\n- 10a: 7 tables - American producers, company_type + role_type filters\n- 11a: 9 tables (company_name, company_type, keyword, link_type, movie_companies,\n movie_info, movie_keyword, movie_link, title) - most tables in any query,\n country_code != '[pl]', LIKE on cn.name, IS NULL check on mc.note\n- 12a: 8 tables - Drama/horror movies with high ratings, double info_type join\n (it1 for genres, it2 for rating), compound conditions\n\n11a is the most complex query in the benchmark - plan the join sequence carefully\naround the title table as the central fact table.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for queries 10a-12a (7-9 table joins)", "wid": "w3"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w1", "w2"], "notes": "Add pandas_impl functions for queries 1a, 1b, 2a, 3a, 4a, 5a, 6a (7 queries).\nKey differences from expression family:\n\n- Joins: pd.merge(left, right, left_on=..., right_on=..., how=\"inner\")\n- LIKE: df[\"col\"].str.contains(\"pattern\", na=False)\n- NOT LIKE: ~df[\"col\"].str.contains(\"pattern\", na=False)\n- IN: df[\"col\"].isin([...])\n- MIN agg: df.agg({\"col\": \"min\"}) or df[\"col\"].min()\n- String comparison: df[df[\"info\"] > \"2.0\"]\n- IS NULL: df[\"col\"].isna()\n\nMulti-table joins chain: reduce(lambda acc, rhs: pd.merge(acc, rhs, ...), tables)\nor sequential merge calls.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create pandas-family implementations for queries 1a-6a (simpler joins)", "wid": "w4"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w2", "w3"], "notes": "Add pandas_impl functions for queries 7a, 8a, 9a, 10a, 11a, 12a (6 queries).\nSame translation patterns as w4.\n\n7a-9a have 7-8 table joins - the sequential pd.merge() chain will be long.\nUse descriptive intermediate variable names for readability:\n merged_title_mc = pd.merge(title, movie_companies, ...)\n merged_with_cn = pd.merge(merged_title_mc, company_name, ...)\n\n11a (9 tables) is the most complex - test this one carefully.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create pandas-family implementations for queries 7a-12a (complex joins)", "wid": "w5"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w4", "w5"], "notes": "Integration:\n1. Add get_dataframe_queries() method to JoinOrderBenchmark in benchmark.py\n2. Set supports_dataframe: True in benchmark_registry.py for joinorder entry\n3. Verify JoinOrderGenerator synthetic data works with DataFrame data loading\n (CSV/Parquet format compatibility)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up JoinOrder benchmark class and registry for DataFrame dispatch", "wid": "w6"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w6"], "notes": "Create tests/unit/core/joinorder/test_dataframe_queries.py:\n- Test all 13 queries registered with correct IDs\n- Test expression_impl on small synthetic IMDB-like data (build minimal\n fixtures with a few rows per table that satisfy join conditions)\n- Test pandas_impl on same fixtures\n- Test result shapes are non-empty (queries with valid data should return rows)\n- Test LIKE/IN/NOT LIKE predicates filter correctly\n- Mark tests @pytest.mark.fast where possible\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for JoinOrder DataFrame queries", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use platform-native introspection", "dont": "attempt to implement INFORMATION_SCHEMA queries in DataFrame mode", "why": "DataFrames access metadata through native API (df.schema, spark.catalog), not SQL catalog tables"}, {"INSTEAD": "document the expected coverage gap clearly", "dont": "force 100% query parity with SQL mode", "why": "ACL, view hierarchy, and constraint queries are inherently SQL-only"}, {"INSTEAD": "audit and complete what exists", "dont": "rewrite the existing 1821-line module from scratch", "why": "significant work is already done"}], "approach": "This benchmark has 1821 lines of existing DataFrame operations code. The work is\naudit + integration + gap-filling, not greenfield.\n\nFollow the write_primitives operations manager pattern for PySpark catalog/lakehouse\noperations (stateful, sequential). For schema introspection operations that are\nindependent and stateless (df.schema, df.describe()), consider using the QueryRegistry\npattern instead - this enables the standard DataFrame runner to execute them.\n\nA hybrid approach may be best: QueryRegistry for Tier 1 schema introspection\n(works on all platforms), operations manager for Tier 2/3 catalog+lakehouse\n(PySpark only). The benchmark class exposes both via get_dataframe_queries()\n(returns Tier 1 registry) and get_dataframe_operations() (returns Tier 2/3 manager).\n\nExpect ~25-30% coverage in DataFrame mode vs SQL mode. This is acceptable because\nmany SQL metadata queries (INFORMATION_SCHEMA, ACL, views) are inherently SQL\nconstructs with no DataFrame equivalent.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-metadata-primitives", "id": 384, "reason": "Requires PySpark + Delta Lake runtime. Add to live_integration suite.", "resolution": "promoted", "resolved_item": "integration-tests-with-real-pyspark-delta-lake-catalog", "resolved_reason": null, "summary": "Integration tests with real PySpark + Delta Lake catalog"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-metadata-primitives", "id": 385, "reason": "DDL generation (CREATE TABLE with 1000 columns) is SQL-specific. DataFrame equivalent (constructing wide DataFrames) is a separate design problem.", "resolution": "promoted", "resolved_item": "complexity-stress-testing-wide-tables-large-catalogs-in-data", "resolved_reason": null, "summary": "Complexity stress testing (wide tables, large catalogs) in DataFrame mode"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-metadata-primitives", "id": 386, "reason": "DataFrame APIs don't have GRANT/REVOKE equivalents. These 11 queries are permanently SQL-only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason states ACL/permission testing is 'permanently SQL-only' for DataFrame APIs; not applicable.", "summary": "ACL/permission testing in DataFrame mode"}], "deps": [], "description": "Complete and integrate the existing DataFrame operations module for the Metadata\nPrimitives benchmark, enabling schema introspection benchmarking on Polars, PySpark,\nPandas, and other DataFrame platforms.\n\nMetadata Primitives tests database metadata introspection - INFORMATION_SCHEMA queries,\nSHOW/DESCRIBE/PRAGMA commands, and catalog operations - using 62 SQL queries across\n10 categories: schema (9), column (8), stats (6), query (4), wide table (5), view\nhierarchy (4), complex type (5), large catalog (6), constraint (4), and ACL (11).\n\nA substantial dataframe_operations.py already exists (1821 lines) with:\n- MetadataOperationType enum (13+ operation types across 4 categories)\n- MetadataOperationCategory enum (SCHEMA, CATALOG, LAKEHOUSE, COMPLEXITY)\n- OPERATION_CATEGORIES mapping dict\n- Platform-specific support levels (all platforms for schema, PySpark for catalog,\n PySpark+Delta/Iceberg for lakehouse metadata)\n\nUnlike SQL metadata introspection (which queries INFORMATION_SCHEMA), DataFrame\nmetadata operations use native DataFrame API: df.schema, df.dtypes, df.describe(),\ndf.columns, len(df), and platform-specific catalog APIs (spark.catalog.listTables()).\n\nThe benchmark has 3 tiers of DataFrame support:\n1. Schema introspection: ALL platforms (Polars, Pandas, PySpark, DataFusion, etc.)\n2. Catalog operations: PySpark with configured catalog only\n3. Lakehouse metadata: PySpark + Delta Lake/Iceberg only", "id": "df-support-metadata-primitives", "preserves": ["Complexity testing DDL generation (wide tables, view hierarchies, large catalogs) still works for SQL mode", "MetadataPrimitivesBenchmark SQL execution path unchanged - all 62 SQL queries still work", "Platform-specific SQL variants (ClickHouse, Snowflake, BigQuery, DataFusion) unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/metadata_primitives/"}], "state": "done", "title": "Add DataFrame execution support to Metadata Primitives benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['metadata_primitives']['supports_dataframe'])\"", "description": "Metadata Primitives reports supports_dataframe=True", "expected": "True", "item_id": "df-support-metadata-primitives", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.metadata_primitives.dataframe_operations import MetadataOperationType, MetadataOperationCategory; print(f'{len(MetadataOperationType)} ops, {len(MetadataOperationCategory)} categories')\"", "description": "Operations module loads without error", "expected": "13 ops, 4 categories", "item_id": "df-support-metadata-primitives", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/metadata_primitives/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-metadata-primitives", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-metadata-primitives", "needs": [], "notes": "Read benchbox/core/metadata_primitives/dataframe_operations.py (1821 lines) thoroughly.\nCompare against the 62 SQL queries in catalog/queries.yaml.\n\nCreate a mapping table across 3 tiers:\n\nTier 1 - Schema introspection (ALL platforms):\n- list_columns -> df.columns\n- get_dtypes -> df.dtypes / df.schema\n- describe_stats -> df.describe()\n- row_count -> len(df) / df.count()\n- column_count -> len(df.columns)\nMap which of the 62 SQL queries correspond to these operations.\n\nTier 2 - Catalog operations (PySpark only):\n- list_databases -> spark.catalog.listDatabases()\n- list_tables -> spark.catalog.listTables()\n- table_exists -> spark.catalog.tableExists()\nMap SQL INFORMATION_SCHEMA queries to catalog API calls.\n\nTier 3 - Lakehouse metadata (PySpark + Delta/Iceberg):\n- table_history -> DeltaTable.history()\n- table_detail -> DeltaTable.detail()\nMap SQL queries to format-specific metadata APIs.\n\nIdentify:\n- Which operations are fully implemented?\n- Which are stubs?\n- Which SQL queries have no DataFrame equivalent (e.g., ACL queries)?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit existing dataframe_operations.py - map SQL queries to DataFrame operations, identify gaps", "wid": "w1"}, {"evidence": null, "item_id": "df-support-metadata-primitives", "needs": ["w1"], "notes": "Ensure Tier 1 operations work across all platforms:\n\nExpression family (Polars, DataFusion, PySpark):\n- Schema access via ctx.get_table(name).schema\n- Column listing via ctx.get_table(name).columns\n- Statistics via describe() or platform-specific stats methods\n\nPandas family (Pandas, Modin, cuDF, Dask):\n- df.dtypes, df.describe(), df.info(), df.columns, len(df)\n- df.memory_usage() for size statistics\n- df.nunique() for cardinality\n\nThese are the most broadly supported operations and should work on all 8\nDataFrame platforms. Implement as DataFrameQuery objects with expression_impl\nand pandas_impl, even though these are metadata operations - the QueryRegistry\npattern enables integration with the standard DataFrame runner.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Complete schema introspection operations for all DataFrame platforms", "wid": "w2"}, {"evidence": null, "item_id": "df-support-metadata-primitives", "needs": ["w1"], "notes": "Tier 2 (PySpark catalog):\n- Implement using spark.catalog.* API\n- listDatabases(), listTables(), listColumns(), tableExists()\n- These require a SparkSession with a configured catalog\n\nTier 3 (PySpark + Delta/Iceberg):\n- DeltaTable.forPath(spark, path).history() / .detail()\n- Iceberg snapshot listing via table properties\n\nDefine comprehensive skip lists:\n- SKIP_FOR_DATAFRAME: ACL queries (GRANT/REVOKE timing), view hierarchy queries\n (DataFrames don't have views), constraint queries (FK/PK are schema-level)\n- SKIP_FOR_POLARS/PANDAS: Catalog and lakehouse operations\n- SKIP_FOR_EXPRESSION_FAMILY: Catalog operations (except PySpark)\n\nMany of the 62 SQL queries are SQL-specific (INFORMATION_SCHEMA, SHOW commands)\nand will be skipped. Document the expected coverage: ~15-20 operations in\nDataFrame mode vs 62 SQL queries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Complete catalog and lakehouse operations for PySpark and define skip lists", "wid": "w3"}, {"evidence": null, "item_id": "df-support-metadata-primitives", "needs": ["w2", "w3"], "notes": "Integration:\n1. Add get_dataframe_queries() or get_dataframe_operations() to\n MetadataPrimitivesBenchmark - decide which pattern based on w1 audit.\n If operations are independent (stateless introspection), use QueryRegistry.\n If operations are sequential/stateful, use operations manager.\n2. Set supports_dataframe: True in benchmark_registry.py\n3. Ensure TPC-H schema tables are loaded before metadata introspection runs\n (metadata queries inspect the loaded tables)\n4. Handle complexity testing (wide tables, large catalogs) - may need to\n generate test structures in DataFrame mode too\n\nTests (tests/unit/core/metadata_primitives/test_dataframe_operations.py):\n- Test MetadataOperationType enum values\n- Test per-platform capability detection\n- Test schema introspection operations on small synthetic DataFrames\n- Test skip list correctness (Polars skips catalog ops, etc.)\n- Test operation categories mapping\n- Mark tests @pytest.mark.fast\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up MetadataPrimitivesBenchmark class, registry, and add unit tests", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "each variant must use a genuinely different execution strategy", "dont": "create trivial variants (e.g., just renaming aliases)", "why": "the point is structural diversity"}, {"INSTEAD": "every variant must produce identical output for the same input data and parameters", "dont": "create variants that produce different results", "why": "semantic equivalence is the core property"}, {"INSTEAD": "import base implementations and build variants on top", "dont": "modify tpch/dataframe_queries.py", "why": "TPC-Havoc extends TPC-H, not modifies it"}], "approach": "Create a dataframe_queries/ subdirectory (like clickbench) with per-query modules\n(q01.py through q22.py). Each module exports 10 DataFrameQuery objects.\n\nImport the base TPC-H DataFrame implementation as v1 (baseline) for each query from\ntpch/dataframe_queries.py, then define v2-v10 as structural alternatives.\n\nA central registry.py aggregates all queries into a single QueryRegistry with\nIDs like \"Q1v1\", \"Q1v2\", etc. The benchmark's get_dataframe_queries() returns\nthis registry.\n\nDesign the variant categories based on actual TPC-H query structure - not all\nvariant types apply to all queries (e.g., join-reorder only applies to multi-join\nqueries). Document which variant strategies apply to which queries in\nDATAFRAME_VARIANTS.md.\n\nUse the existing TPC-H parameter override system (set_parameter_overrides) for\nseed-based reproducibility.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-tpchavoc", "id": 387, "reason": "Full equivalence testing is expensive (220 queries x 2 families). Start with Q1-Q3 equivalence in unit tests, defer comprehensive testing to integration suite.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: A 220-variant equivalence gate already exists in CI (tests/integration/test_tpchavoc_variant_equivalence.py: 'full gate sweeps all 220 variants'); already done.", "summary": "Automated variant equivalence testing across all 22 queries at SF=0.01"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-tpchavoc", "id": 388, "reason": "Analysis tooling is follow-up work after the implementations exist and have been benchmarked.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: 'Variant performance analysis tooling - fastest DataFrame patterns per platform' is a vague future-analysis idea with no concrete design, scope, or committed demand.", "summary": "Variant performance analysis tooling - identify which DataFrame patterns are fastest per platform"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-tpchavoc", "id": 389, "reason": "The first complete registry covers Q16-Q22 with canonical execution plus final projection/order replay to preserve equivalence. Deeper variants for Q19 condition unions, Q20 semi-join ordering, Q21 anti-join alternatives, and Q22 materialized average balance should be developed with broader equivalence coverage.", "resolution": "promoted", "resolved_item": "replace-q16-q22-result-replay-variants-with-deeper-hand-writ", "resolved_reason": null, "summary": "Replace Q16-Q22 result-replay variants with deeper hand-written structural rewrites"}], "deps": [], "description": "Add DataFrame variant implementations for TPC-Havoc, enabling execution of\nstructurally diverse TPC-H query variants on Polars, PySpark, DataFusion, Pandas,\nModin, cuDF, and Dask platforms.\n\nTPC-Havoc generates 10 structural SQL variants per TPC-H query (22 queries x 10\nvariants = 220 total). Each variant is semantically equivalent but uses different\nSQL constructs (CTEs, scalar subqueries, window functions, UNION, etc.) to stress\ndifferent optimizer components.\n\nFor DataFrame execution, the concept translates differently: instead of varying SQL\nsyntax, DataFrame variants vary the API call sequence - different join orders,\ndifferent aggregation strategies (pre-filter vs post-filter), lazy vs eager\nevaluation patterns, and different window function formulations. The goal shifts\nfrom \"stress the SQL optimizer\" to \"measure how different DataFrame API patterns\naffect execution performance on the same logical query.\"\n\nTPC-H already has DataFrame queries (tpch/dataframe_queries.py) with 22 queries\nin both expression and pandas families. TPC-Havoc DataFrame extends these with\nalternative implementations that use structurally different DataFrame API patterns.\n\nThis is architecturally unique - no other benchmark has the variant concept in\nDataFrame mode. Requires designing a new variant registration pattern for\nDataFrameQuery objects.", "id": "df-support-tpchavoc", "preserves": ["ResultValidator variant equivalence checking still works for SQL mode", "TPCHBenchmark base class DataFrame queries (tpch/dataframe_queries.py) are NOT modified - TPC-Havoc imports them as baselines", "TPCHavocBenchmark SQL variant generation unchanged - all 220 SQL variants still work via VariantGenerator/StaticSQLVariant", "benchbox run --platform duckdb --benchmark tpchavoc still runs SQL variants"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/dataframe_queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/"}], "state": "done", "title": "Add DataFrame execution support to TPC-Havoc benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.tpchavoc.dataframe_queries import get_dataframe_queries; r = get_dataframe_queries(); print(f'{len(r)} variants registered'); assert len(r) == 220\"", "description": "All 220 DataFrame variants are registered", "expected": "220 variants registered", "item_id": "df-support-tpchavoc", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_METADATA; print(BENCHMARK_METADATA['tpchavoc']['supports_dataframe'])\"", "description": "TPC-Havoc benchmark reports supports_dataframe=True", "expected": "True", "item_id": "df-support-tpchavoc", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/tpchavoc/test_tpchavoc_dataframe_queries.py -k 'q1_equivalence' -v", "description": "Q1 variants produce equivalent results", "expected": "all tests pass, 0 failures", "item_id": "df-support-tpchavoc", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-tpchavoc", "needs": [], "notes": "Design how DataFrame variants integrate with the existing DataFrameQuery/QueryRegistry\nsystem. Key decisions:\n\n1. Query ID scheme: Use \"Q1v1\", \"Q1v2\", ... \"Q1v10\" or \"Q1-scalar-subquery\",\n \"Q1-cte-rewrite\"? The former is consistent with SQL variant IDs; the latter\n is more descriptive for DataFrame where structural names matter.\n\n2. Registry structure: One flat QueryRegistry with all 220 entries, or a nested\n structure (VariantRegistry wrapping QueryRegistry)? Flat is simpler and works\n with the existing dataframe_runner dispatch.\n\n3. Variant categories for DataFrame (map from SQL variant types):\n - join-reorder: Different .join() sequences for same result\n - pre-filter: Filter before join vs after join\n - window-alternative: Window function vs groupby+join for same logic\n - cte-inline: Intermediate DataFrame vs chained expression\n - aggregation-strategy: Multiple small groupbys vs single large groupby\n - broadcast-hint: Explicit broadcast for small tables vs default\n - lazy-eager: .collect() placement variations (expression family only)\n\n4. How variants relate to base TPC-H queries: Import from tpch/dataframe_queries.py\n as the \"v0\" baseline, then define v1-v10 as alternatives.\n\nWrite a short design doc in benchbox/core/tpchavoc/DATAFRAME_VARIANTS.md\ndocumenting the chosen approach before implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design DataFrame variant architecture and registration pattern", "wid": "w1"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w8", "w9"], "notes": "Integration:\n1. Add get_dataframe_queries() method to TPCHavocBenchmark\n - Must return all variants (not just base queries)\n - Consider whether to return all 220 at once or support filtering by query/variant\n2. Set supports_dataframe: True in benchmark_registry.py for tpchavoc\n3. Add variant equivalence validation for DataFrame results\n (extend ResultValidator to handle DataFrame output comparison)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up TPC-Havoc benchmark class and registry for DataFrame dispatch", "wid": "w10"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w10"], "notes": "Create tests/unit/core/tpchavoc/test_dataframe_queries.py:\n- Test variant registration (220 total, 10 per query)\n- Test all variants for Q1 produce identical results on small TPC-H data\n- Test expression_impl and pandas_impl for at least Q1-Q3 variants\n- Test variant metadata (description, categories)\n- Mark equivalence tests @pytest.mark.slow (they need TPC-H SF=0.01 data)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for TPC-Havoc DataFrame variants", "wid": "w11"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "Create benchbox/core/tpchavoc/dataframe_queries/ directory with per-query modules\n(q01.py, q02.py, q03.py). For each query, create 10 expression_impl variants.\n\nQ1 (Pricing Summary - filter + groupby + aggregation):\n- v1: Baseline (import from tpch dataframe Q1)\n- v2: Pre-compute aggregation columns before groupby\n- v3: Window functions instead of groupby (partition + distinct)\n- v4: Two-pass: compute sums first, then averages from sums\n- v5: Filter using .when().then() instead of boolean mask\n- v6: Explicit column selection before groupby (projection pushdown hint)\n- v7: Sort before groupby (tests sort-based aggregation)\n- v8: Intermediate DataFrame for filtered data (CTE equivalent)\n- v9: Compute each aggregate separately then join results\n- v10: Reverse sort order then reverse at end\n\nQ2 (Minimum Cost Supplier) and Q3 (Shipping Priority) follow similar variant\naxes but emphasize join-reorder variants due to multi-table joins.\n\nEach variant must produce identical results to the base query.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q1-Q3 (30 variants)", "wid": "w2"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "Continue with q04.py, q05.py, q06.py.\n\nQ4 (Order Priority Checking - EXISTS subquery):\n- Variant axes: semi-join vs filter-join, pre-filter lineitem dates vs post-filter\nQ5 (Local Supplier Volume - 6-table join + groupby):\n- Variant axes: join order (6 permutations), pre-filter nation/region before join\nQ6 (Forecasting Revenue Change - single-table filter + sum):\n- Variant axes: filter order, compound vs chained filters, pre-select columns\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q4-Q6 (30 variants)", "wid": "w3"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q07.py through q10.py. These are join-heavy queries where the primary variant\naxis is join order:\n- v1: Baseline join order from tpch/dataframe_queries.py\n- v2-v4: Alternative join orders (small-to-large, large-to-small, star-join)\n- v5-v7: Pre-filter variants (filter dimension tables before joining)\n- v8-v10: Aggregation strategy variants\n\nQ7 (Volume Shipping), Q8 (National Market Share), Q9 (Product Type Profit),\nQ10 (Returned Item Reporting).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q7-Q10 (40 variants)", "wid": "w4"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q11.py through q14.py. Mix of subquery-heavy and aggregation-heavy queries.\n\nQ11 (Important Stock Identification - HAVING with subquery):\n- Variant axes: subquery as intermediate DF vs inline, threshold pre-compute\nQ12 (Shipping Modes and Order Priority - CASE + groupby):\n- Variant axes: CASE as .when() chain vs boolean masks, filter-first vs agg-first\nQ13 (Customer Distribution - LEFT JOIN + groupby + groupby):\n- Variant axes: two-stage vs single-stage aggregation, NOT LIKE as anti-join\nQ14 (Promotion Effect - CASE inside SUM):\n- Variant axes: conditional sum vs filter+sum+join, pre-filter parts\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q11-Q14 (40 variants)", "wid": "w5"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q15.py through q18.py. Complex subquery patterns.\n\nQ15 (Top Supplier - subquery for max revenue):\n- Variant axes: intermediate DF vs chained, top-1 vs filter-to-max\nQ16 (Parts/Supplier Count - DISTINCT, NOT IN):\n- Variant axes: anti-join vs ~isin(), distinct before vs after join\nQ17 (Small Quantity Order Revenue - correlated subquery):\n- Variant axes: join-with-avg vs window function, filter timing\nQ18 (Large Volume Customer - HAVING):\n- Variant axes: filter-after-groupby vs pre-aggregate, join order\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q15-Q18 (40 variants)", "wid": "w6"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q19.py through q22.py. Most complex query patterns.\n\nQ19 (Discounted Revenue - compound OR conditions):\n- Variant axes: union of filtered sets vs single complex filter, per-brand subframes\nQ20 (Potential Part Promotion - multi-level subquery):\n- Variant axes: join chain vs nested intermediates, semi-join vs filter\nQ21 (Suppliers Who Kept Orders Waiting - EXISTS/NOT EXISTS):\n- Variant axes: semi-join + anti-join vs boolean masks, join order for 4 tables\nQ22 (Global Sales Opportunity - nested subqueries):\n- Variant axes: intermediate DFs vs chained, avg pre-compute vs inline\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q19-Q22 (40 variants)", "wid": "w7"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w2", "w3", "w4", "w5"], "notes": "Add pandas_impl variants to q01.py through q11.py, paralleling expression_impl.\nMany variant strategies translate directly (join reorder, pre-filter, etc.).\n\nReplace expression-specific variants (lazy/eager, collect placement) with\npandas-specific alternatives:\n- .apply() vs vectorized operations\n- pd.merge() vs df.join() (index-based)\n- .groupby().agg() vs .pivot_table()\n- iterative merge chain vs reduce(merge, tables)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement pandas-family variants for Q1-Q11 (110 variants)", "wid": "w8"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w6", "w7"], "notes": "Add pandas_impl variants to q12.py through q22.py.\n\nFor the complex subquery patterns (Q15-Q22), pandas variants emphasize:\n- Intermediate DataFrames vs chained operations\n- .merge() join order permutations\n- .pipe() chains vs sequential assignment\n- Index-based operations vs column-based\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement pandas-family variants for Q12-Q22 (110 variants)", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "skip cleanly with informative messages", "dont": "attempt to implement transactions on Polars/Pandas", "why": "they have no ACID semantics"}, {"INSTEAD": "use the Delta Lake/Iceberg native patterns documented in dataframe_operations.py", "dont": "map SQL BEGIN/COMMIT/ROLLBACK directly", "why": "Delta Lake uses atomic writes, not explicit transaction blocks"}, {"INSTEAD": "audit and complete what exists", "dont": "rewrite the existing 1290-line module from scratch", "why": "significant work is already done"}], "approach": "This benchmark already has substantial DataFrame operations code (1290 lines).\nThe work is primarily audit + integration, not greenfield implementation.\n\nFollow the write_primitives pattern: the benchmark class exposes\nget_dataframe_operations() returning an operations manager, not\nget_dataframe_queries() returning a QueryRegistry. The dataframe_runner\nalready handles this via the operations manager path.\n\nStart with the audit (w1) to understand what's implemented vs stub. The\nexisting code uses DataFrameMaintenanceCapabilities and TransactionIsolation\nfrom benchbox/core/dataframe/maintenance_interface.py - verify these are\ncorrectly integrated.\n\nOnly PySpark+Delta/Iceberg supports DataFrame transactions. Polars, Pandas,\nand other platforms should be explicitly skipped with clear messaging.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-transaction-primitives", "id": 390, "reason": "Requires PySpark + Delta Lake runtime. Add to live_integration test suite with @pytest.mark.live_integration.", "resolution": "promoted", "resolved_item": "integration-tests-with-real-pyspark-delta-lake", "resolved_reason": null, "summary": "Integration tests with real PySpark + Delta Lake"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-transaction-primitives", "id": 391, "reason": "Iceberg support depends on PySpark + Iceberg catalog setup. Defer until Delta Lake path is validated.", "resolution": "promoted", "resolved_item": "iceberg-table-format-support-testing", "resolved_reason": null, "summary": "Iceberg table format support testing"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-transaction-primitives", "id": 392, "reason": "Some SQL operations (savepoints, explicit isolation levels) have no DataFrame equivalent. Document the parity gap rather than attempting impossible translations.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly rejects the deferred action: 'document the parity gap rather than attempting impossible translations.'", "summary": "Map all 23 SQL operations to DataFrame equivalents with full parity"}], "deps": [], "description": "Complete and integrate the existing DataFrame operations module for the Transaction\nPrimitives benchmark, enabling execution on PySpark + Delta Lake/Iceberg platforms.\n\nTransaction Primitives tests ACID transaction semantics: commit, rollback, savepoints,\nisolation levels, and multi-statement transactions using 23 SQL operations on TPC-H\nstaging tables (txn_orders, txn_lineitem, txn_customer).\n\nA substantial dataframe_operations.py already exists (1290 lines) with:\n- TransactionOperationType enum (10 operation types: ATOMIC_INSERT, ROLLBACK_TO_VERSION,\n TIME_TRAVEL_QUERY, CONCURRENT_WRITE, etc.)\n- DataFrameTransactionCapabilities dataclass (per-platform capability tracking)\n- Integration with DataFrameMaintenanceCapabilities and TransactionIsolation\n- Platform-specific support declarations (PySpark+Delta, PySpark+Iceberg)\n\nKey constraint: DataFrame transaction semantics are fundamentally different from SQL.\nSQL uses BEGIN/COMMIT/ROLLBACK; Delta Lake uses atomic writes with RESTORE for\nrollback; Iceberg uses snapshot-based isolation. Polars/Pandas have NO transaction\nsupport. This benchmark is PySpark-only in DataFrame mode.\n\nThe work is primarily integration - wiring the existing operations module into the\nbenchmark class, runner, and registry - plus filling in any incomplete operation\nimplementations and adding tests.", "id": "df-support-transaction-primitives", "preserves": ["Platform overrides for SQL mode (e.g., DuckDB savepoint skips) still work", "TPC-H staging table schema (txn_orders, txn_lineitem, txn_customer) unchanged", "TransactionPrimitivesBenchmark SQL execution path unchanged - all 23 SQL operations still work via OperationExecutor interface"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/transaction_primitives/"}], "state": "done", "title": "Add DataFrame execution support to Transaction Primitives benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['transaction_primitives']['supports_dataframe'])\"", "description": "Transaction Primitives reports supports_dataframe=True", "expected": "True", "item_id": "df-support-transaction-primitives", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.transaction_primitives.dataframe_operations import TransactionOperationType; print(f'{len(TransactionOperationType)} operation types')\"", "description": "Operations manager loads without error", "expected": "10 operation types", "item_id": "df-support-transaction-primitives", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/transaction_primitives/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-transaction-primitives", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-transaction-primitives", "needs": [], "notes": "Read benchbox/core/transaction_primitives/dataframe_operations.py (1290 lines) thoroughly.\nCompare against the 23 SQL operations defined in catalog/operations.yaml.\n\nCreate a mapping table:\n- Which SQL operations have DataFrame equivalents already implemented?\n- Which operations are impossible in DataFrame mode (e.g., SQL-level savepoints)?\n- Which operations need new implementations?\n- Which DataFrameTransactionCapabilities are fully implemented vs stubs?\n\nCheck integration points:\n- Does the module use get_maintenance_operations_for_platform() correctly?\n- Does it handle DataFrameMaintenanceCapabilities properly?\n- Are the TransactionIsolation levels correctly mapped?\n\nDocument findings in a comment block at the top of the file or in a separate\naudit notes file to guide w2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit existing dataframe_operations.py - map operations to benchmark operations, identify gaps", "wid": "w1"}, {"evidence": null, "item_id": "df-support-transaction-primitives", "needs": ["w1"], "notes": "Based on the w1 audit:\n\n1. Complete any stub implementations in dataframe_operations.py\n2. Define SKIP_FOR_DATAFRAME list for operations that have no DataFrame equivalent:\n - SQL-level savepoints (SAVEPOINT/ROLLBACK TO) - no Delta Lake/Iceberg equivalent\n - Explicit isolation level setting (SET TRANSACTION ISOLATION LEVEL)\n - Operations requiring explicit BEGIN/COMMIT blocks\n3. Define SKIP_FOR_POLARS and SKIP_FOR_PANDAS as complete skip lists (all operations)\n since transaction primitives only work on PySpark+Delta/Iceberg\n4. Ensure error handling for unsupported platforms returns clear skip messages\n\nThe operations manager pattern (not QueryRegistry pattern) is correct for this\nbenchmark - transaction operations are stateful and sequential, not independent\nqueries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Complete missing operation implementations and add skip lists for unsupported operations", "wid": "w2"}, {"evidence": null, "item_id": "df-support-transaction-primitives", "needs": ["w2"], "notes": "Integration steps:\n1. Add get_dataframe_operations() method to TransactionPrimitivesBenchmark in\n benchmark.py - similar to write_primitives pattern:\n ```python\n def get_dataframe_operations(self, platform_name):\n from .dataframe_operations import get_transaction_manager\n return get_transaction_manager(platform_name)\n ```\n2. Set supports_dataframe: True in benchmark_registry.py for transaction_primitives\n3. Verify the dataframe_runner.py handles operations-based benchmarks correctly\n (it should fall through to get_dataframe_operations if no get_dataframe_queries)\n4. Add platform capability checks - only PySpark with Delta Lake or Iceberg\n should attempt this benchmark in DataFrame mode\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up TransactionPrimitivesBenchmark class and registry for DataFrame dispatch", "wid": "w3"}, {"evidence": null, "item_id": "df-support-transaction-primitives", "needs": ["w3"], "notes": "Create tests/unit/core/transaction_primitives/test_dataframe_operations.py:\n- Test TransactionOperationType enum values match expected operations\n- Test DataFrameTransactionCapabilities for PySpark reports correct support\n- Test Polars/Pandas capabilities report all operations as unsupported\n- Test skip list generation for non-PySpark platforms\n- Test operation manager instantiation with mock platform\n- Mock-based tests (real PySpark+Delta tests belong in integration suite)\n- Mark all tests @pytest.mark.fast\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for Transaction Primitives DataFrame operations", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "collapse object storage keys, submission IDs, and public result URLs into one identifier", "instead": "separate them explicitly", "why": "retries, rebuilds, and public permalinks have different lifecycles"}, {"dont": "make the explorer query arbitrary raw uploads as its primary data source", "instead": "serve derived read models", "why": "that couples UX latency and safety to uncurated artifacts"}], "approach": "Use a split write/read architecture. Store raw bundles once, normalize into\ndurable metadata, and publish rebuildable read models for the explorer.\nPrefer static-friendly public reads and keep authenticated mutations behind\nthe hosted service boundary.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-27T00:00:00Z", "completed_pr": null, "created_at": "2026-03-29T00:00:00Z", "deferrals": [], "deps": ["define-hosted-results-contract-and-governance-model", "define-results-platform-product-and-launch-strategy"], "description": "Once BenchBox decides what a hosted result means, it still needs an\narchitecture that separates the mutable write path from the public read path.\nThe write path must ingest immutable raw bundles reliably; the read path must\nserve fast public exploration without forcing the browser to query arbitrary\nuploads directly.\n\nThis item defines the service boundary, storage tiers, async ingest model,\nrebuild strategy, and the derived public read models consumed by the results\nexplorer.\n\n**Phase assignment:** This TODO covers the full hosted ingest architecture\n(Phase 3). The Phase 1 static build pipeline (bundles \u2192 manifests \u2192 DuckDB\nsnapshot) is simpler and is tracked by dedicated Phase 1 implementation TODOs.\nThis item remains relevant for Phase 3 API design but is NOT a blocker for\nPhase 1 or Phase 2.", "id": "design-results-ingest-storage-and-derived-read-model", "preserves": ["Raw schema-v2 bundles and companion files remain immutable source artifacts", "The public explorer can be rebuilt from stored source artifacts plus durable metadata"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/analysis/"}], "state": "done", "title": "Design results ingest, storage, and derived read-model architecture", "verifications": [{"command": "test -f _project/analysis/ingest-architecture-design.md && echo PRESENT", "description": "Architecture design doc committed to _project/analysis/", "expected": "PRESENT", "item_id": "design-results-ingest-storage-and-derived-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -c 'submission API\\|storage model\\|metadata schema\\|read model\\|deployment\\|backfill\\|failure' _project/analysis/ingest-architecture-design.md", "description": "Design doc addresses all required topic areas", "expected": "7", "item_id": "design-results-ingest-storage-and-derived-read-model", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": [], "notes": "Output: 1-page reference matrix documenting each platform's bundle format,\nstorage tier, idempotency model, and public read surface. Use as a\ngrounding input for w2 and w3 design choices.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Catalog 3 reference ingest architectures - survey Geekbench, OpenBenchmarking, and CloudSpecs submission \u2192 storage \u2192 projection pipelines", "wid": "w1"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w1"], "notes": "Define the hosted submission interface: what the CLI POSTs, how the service\nacknowledges receipt, and what guarantees are given on idempotent re-submit.\nOutput: submission-api.md with endpoint sketch and idempotency rules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write submission API surface proposal: endpoints, idempotency rules, and bundle upload contract", "wid": "w2"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w1"], "notes": "Define where and how raw schema-v2 bundles are stored after receipt. Key\ndecisions: key naming (must survive rename/rebrand), retention policy,\nintegrity verification (hash check on ingest), and access control\nboundaries. Output: storage-model.md with object key scheme and retention\nrules.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write raw bundle storage model: object key scheme, retention tiers, and integrity properties", "wid": "w3"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w2", "w3"], "notes": "Capture the normalized data layer that separates the write path from the\npublic read path. Include submission-level fields (actor, timestamp, trust\ntier, visibility) and how they map to cohort membership and compare views.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define durable metadata schema for runs, submissions, actors, trust, and cohorts", "wid": "w4"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w4"], "notes": "Include static manifests, leaderboard projections, and whether a Parquet or\nDuckDB snapshot is worth shipping for browser-side analysis. Document the\nrebuild trigger and incremental update strategy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define derived public read models: manifests, projections, and DuckDB-vs-Parquet tradeoffs", "wid": "w5"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w2", "w3", "w5"], "notes": "Document what runs as a long-lived service vs. a triggered worker vs. a\nstatic asset. Include deployment topology (e.g., Fly.io + R2 or equivalent)\nand cost model at low submission volume.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Choose deployment boundaries: API service, async workers, object storage, and static explorer assets", "wid": "w6"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w4", "w5", "w6"], "notes": "How does the platform recover from a corrupted read model, a failed\nsubmission, or a schema migration? Define rebuild triggers, retention\nwindows, and failure escalation paths.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define backfill, rebuild, retention, and failure-recovery procedures", "wid": "w7"}, {"evidence": null, "item_id": "design-results-ingest-storage-and-derived-read-model", "needs": ["w2", "w3", "w4", "w5", "w6", "w7"], "notes": "Consolidate w1-w7 outputs into a single DESIGN.md under _project/analysis/.\nThis doc is the authoritative input for implementation TODOs that follow.\nPhase 3 cannot start without this review passing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write architecture design doc and get peer review before Phase 3 begins", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "collapse the four guardrails into a single bullet on Step 6", "instead": "keep them as four named acceptance criteria so reviewers can trace each one back to its source.", "why": "each one cites a specific blind-spot and a specific failure mode"}, {"dont": "pre-author a steward-queue implementation TODO 'just in case'", "instead": "wait for the verdict.", "why": "that primes the Step 5 decision and violates the Step 5 anti-pattern list"}, {"dont": "re-point the blind-spots' todo_id at dev-loop-step-6-queue-decision-gate before w2 has actually folded the guardrails into it", "instead": "leave todo_id on this TODO until the queue spec demonstrably owns them.", "why": "Step 6 may resolve as DO NOT BUILD and never carry them"}], "approach": "This is a watcher-and-handoff TODO, not an implementation TODO. The\nonly state changes happen at two well-defined trigger points:\n 1. Step 5 publishes its decision record (w1 branches).\n 2. If BUILD QUEUE: edit Step 6 (w2) and promote the blind-spots\n (w3) in a single PR.\nKeep the PR scoped to the four files: Step 6 YAML and the four\nblind-spot files. Do not touch Step 5, the dev-loop infrastructure,\nor any other architecture in the same change.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Process and Workflow", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-08T00:00:00Z", "deferrals": [], "deps": ["dev-loop-step-5-measurement-window"], "description": "The 2026-04-30 L2 merge-loop audit produced four blind-spot findings\nthat all gate any future steward-queue implementation. They were\npromoted to `merged-to-todo` -> this TODO on 2026-05-11, so the\nblind-spot log records that they are integrated into a planned item\nrather than sitting in the `actionable` re-triage queue:\n\n - 2026-04-30-114721-global-test-lock-cpu-protection \u2014 keep the\n global lock default until measurement shows concurrent xdist\n runs are CPU-safe.\n - 2026-04-30-114722-queue-single-point-failure \u2014 require explicit\n failure-mode analysis before adding a steward queue; prefer\n native GitHub features if they appear.\n - 2026-04-30-114723-queue-dwell-contract \u2014 treat dwell-time\n targets as part of any queue acceptance criteria; update agent\n handoff docs if dwell becomes materially longer than CI.\n - 2026-04-30-114724-post-merge-safety-tension \u2014 define the exact\n failure that develop-post-merge.yml handles; do not add a queue\n unless Step 5 metrics show a pre-merge ordering problem.\n\nAll four are blocked on the same gate:\n`dev-loop-step-5-measurement-window` \u2014 a 30-day measurement window\nthat started 2026-04-30 and closes ~2026-05-30. Until Step 5\npublishes its decision record, none of these findings are\nindividually actionable, which is exactly why they belong to one\nhandoff TODO instead of four `actionable` log entries.\n\nThis TODO does NOT pre-build the queue (that anti-pattern is\nexplicit in Step 5's `do_not_modify` list). It exists to ensure\nthat, IF Step 5 returns BUILD QUEUE, Step 6 inherits these four\nfindings as named acceptance criteria \u2014 not as four ad-hoc reviewer\ncomments that get lost in the queue spec.\n\nIf Step 5 returns DO NOT BUILD or REASSESS, this TODO closes as\nsuperseded with a one-line note pointing to the Step 5 decision\nrecord; the four blind-spots stay `merged-to-todo` pointing here\n(this TODO's DONE record carries the outcome) \u2014 no further\nstatus flip is needed.\n", "id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "preserves": ["All four blind-spots' historical record \u2014 the only mutations are the `status`/`todo_id` frontmatter line and append-only triage-log entries; the `## Finding` / `## Why this matters` body text is immutable.", "The 'do not pre-build the queue during measurement' anti-pattern in Step 5.", "dev-loop-step-5-measurement-window's calendar-gated Blocked status \u2014 this TODO must not push Step 5 to publish early."], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "Makefile"}, {"kind": "do_not_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-5-measurement-window.yaml"}, {"kind": "do_not_modify", "path_glob": "tests/conftest.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114721-global-test-lock-cpu-protection.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114722-queue-single-point-failure.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114723-queue-dwell-contract.md"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-114724-post-merge-safety-tension.md"}], "state": "active", "title": "Pre-register four 2026-04-30 blind-spots as Step 6 queue acceptance criteria", "verifications": [{"command": "ls _project/decisions/dev-loop-measurement-*.md", "description": "Step 5 decision record exists before this TODO begins w2", "expected": "at least one file", "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -E '2026-04-30-114(721|722|723|724)-' _project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml", "description": "If BUILD QUEUE: Step 6 YAML lists the four guardrails by blind-spot ID", "expected": "four matching lines, one per blind-spot id", "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "last_result": null, "last_run": null, "seq": 2}, {"command": "for id in 2026-04-30-114721-global-test-lock-cpu-protection 2026-04-30-114722-queue-single-point-failure 2026-04-30-114723-queue-dwell-contract 2026-04-30-114724-post-merge-safety-tension; do uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py show $id | grep -E 'status:|todo_id:'; done", "description": "Four blind-spots show status: merged-to-todo with todo_id populated", "expected": "all four show status: merged-to-todo and a non-null todo_id", "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "needs": [], "notes": "Trigger: `dev-loop-step-5-measurement-window` publishes\n`_project/decisions/dev-loop-measurement-.md` with a\nBUILD QUEUE / DO NOT BUILD / REASSESS verdict.\n - BUILD QUEUE -> proceed to w2.\n - DO NOT BUILD -> close this TODO as completed-via-supersede\n with a one-line note citing the Step 5 decision SHA. The\n four blind-spots stay `merged-to-todo` -> this TODO; the\n DONE record is their disposition. (Optionally append a\n one-line triage entry to each pointing at the decision SHA.)\n - REASSESS -> stay Blocked; let Step 5's REASSESS path drive\n the next iteration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Watch Step 5 decision record; branch on outcome", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "needs": ["w1"], "notes": "Edit `_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml`\n(only after Step 5 unblocks it) so its `success_metrics` /\n`must_preserve` explicitly include:\n - \"CPU-protection default preserved: queue must not silently\n flip the global test-lock default before concurrent-xdist\n measurement clears it as safe.\"\n Cites: 2026-04-30-114721-global-test-lock-cpu-protection.\n - \"Failure-mode analysis is a prerequisite: a written analysis\n of steward-queue failure modes (single-point-of-failure,\n recovery, fallback commands) lands before the steward\n implementation TODO is authored.\"\n Cites: 2026-04-30-114722-queue-single-point-failure.\n - \"Dwell-time SLO is a queue acceptance criterion: queue spec\n names a target P50/P95 dwell-time and a triggering\n measurement source; agent handoff docs update if dwell\n becomes materially longer than current CI.\"\n Cites: 2026-04-30-114723-queue-dwell-contract.\n - \"Post-merge safety tension is resolved before queue lands:\n queue spec states explicitly which failures\n develop-post-merge.yml handles vs which the queue handles,\n and does not duplicate safety machinery without measurable\n recovery improvement.\"\n Cites: 2026-04-30-114724-post-merge-safety-tension.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Fold the four guardrails into Step 6's acceptance criteria", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-queue-acceptance-criteria-from-blind-spots", "needs": ["w2"], "notes": "The four blind-spots are already `merged-to-todo` ->\n`dev-loop-queue-acceptance-criteria-from-blind-spots` (promoted\n2026-05-11). On the BUILD QUEUE path, once w2 has folded the\nguardrails into Step 6, re-run the promote action so the link\npoints at the queue spec that now owns them:\n `uv run --project _project/scripts -- python _project/scripts/sweep_blind_spots.py triage --action promote --todo-id dev-loop-step-6-queue-decision-gate`\nThe triage-log entry should cite Step 5's decision SHA and name\nthis TODO as the prior merge path. On any non-BUILD path this\nwork unit is a no-op \u2014 leave todo_id pointing here.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-point the four blind-spots' todo_id to Step 6 (BUILD QUEUE path only)", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "begin Step 1-4 work in parallel with this TODO", "instead": "wait for the GO signal in the decision record", "why": "the verification may invalidate downstream plans"}, {"dont": "paraphrase API responses in the decision record", "instead": "paste verbatim JSON", "why": "future re-plans need raw values"}, {"dont": "skip w2 (rulesets)", "instead": "checking only one surface gives a false answer", "why": "GitHub repos commonly encode protection in rulesets rather than classic branch protection"}], "approach": "Pure read-only investigation. Run `gh api` calls in sequence,\npaste raw JSON into the decision record, then write conclusions.\nDo not modify branch protection, rulesets, or any workflow during\nthis TODO. The single artifact is a markdown file documenting\nwhat is true today.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-0-verify-assumptions", "id": 375, "reason": "Each downstream step is a separate TODO with its own scope and verification", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Refers to dev-loop Steps 1-4 implementation, already superseded \u2014 project has progressed to Steps 5/6, both already open tracked items.", "summary": "Acting on the findings (Steps 1-4 implementation)"}], "deps": [], "description": "Decision-gate item that precedes Steps 1-4 of the dev-loop simplification\nplan. The plan rests on three load-bearing assumptions: (a) \"Require\nbranches to be up to date before merging\" is enabled on `develop`,\n(b) PR-tier CI on `develop` is heavier than necessary, (c) native\nGitHub Merge Queue is unavailable on this user-owned repo.\n\nThis TODO produces a written decision record. If any assumption is\nwrong, downstream TODOs (Step 1-4) must be re-planned BEFORE shipping.\n\nNo commits are produced by this TODO except the decision record itself.\n\nContext (for future readers): see `_project/blind-spots/` for the L2\naudit findings that motivated the dev-loop simplification, captured\nin Step 1.", "id": "dev-loop-step-0-verify-assumptions", "preserves": ["All API outputs recorded verbatim, not paraphrased \u2014 future agents may need the raw values", "No code or workflow changes during this step \u2014 verification only"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml (audited but not edited)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": "CLAUDE.md, AGENTS.md (no edits \u2014 Steps 1-4 update these)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 0 reads via gh api; never PATCH)"}, {"kind": "do_not_modify", "path_glob": "Makefile (no edits \u2014 read-only investigation)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-verification-2026-04-30.md (new file, the decision record)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 0: verify assumptions before changing anything", "verifications": [{"command": "test -f _project/decisions/dev-loop-verification-2026-04-30.md && grep -cE '^## (W[1-6]|Conclusion)' _project/decisions/dev-loop-verification-2026-04-30.md", "description": "Decision record exists and contains all six findings", "expected": ">= 7", "item_id": "dev-loop-step-0-verify-assumptions", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE 'Step [1-6]:' _project/decisions/dev-loop-verification-2026-04-30.md", "description": "Decision record explicitly addresses each downstream step", "expected": ">= 4", "item_id": "dev-loop-step-0-verify-assumptions", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Run: `gh api repos/joeharris76/BenchBox/branches/develop/protection\n--jq '.required_status_checks'` and record the value of `strict`.\nIf strict is already false, the staleness diagnosis is wrong and\nStep 3.4 (disable strict-base) is a no-op. Re-plan before Step 3.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Check classic branch protection on develop for `strict: true`", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Required checks may be encoded in a ruleset, not in classic\nbranch protection. Run: `gh api repos/joeharris76/BenchBox/rulesets`\nand inspect each ruleset that targets `develop`. Record:\n - Ruleset names targeting develop\n - Required status checks per ruleset\n - Whether `require_branches_up_to_date` is set in any of them\nThe Step 3.4 toggle must be applied wherever it's encoded.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Check GitHub rulesets targeting develop for required-checks and strict-base", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Read `.github/workflows/test.yml` `on:` block; identify which jobs\nrun on `pull_request` to develop. Cross-reference with recent run\ndurations: `gh run list --workflow test.yml --branch develop --limit 20\n--json conclusion,startedAt,updatedAt,name`. Record P50/P95 CI\nduration on develop PRs today. This is the baseline that Step 3.1\n(cut to lint + Ubuntu 3.12 fast tests) reduces.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit current PR-tier CI cost on develop", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": ["w3"], "notes": "Run: `gh run list --workflow develop-post-merge.yml --branch develop\n--limit 30 --json conclusion`. Record current red/green ratio. If\nalready noisy, Step 4 needs to handle false positives carefully.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit develop-post-merge.yml health", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Use a concrete proxy rather than free-text grep. For the last 50\nmerged PRs targeting develop, compute the gap between the last\nsuccessful CI run start and the PR merge time. A gap > 30 minutes\nwith intervening pushes is a strong signal of a strict-base re-CI\ncycle (the PR was green, develop advanced, CI re-ran):\n - `gh pr list --state merged --base develop --limit 50 \\\n --json number,mergedAt,headRefOid,labels,statusCheckRollup`\n - For each PR, compute `merged_at - last_successful_check_started_at`\n across `statusCheckRollup` entries\n - Count PRs where that gap > 30 min AND statusCheckRollup has\n >= 2 distinct startedAt timestamps for the same check name\n (proxy for re-CI after a base push)\nSecondary signal (lower confidence): run\n`gh pr list --state merged --base develop --limit 50 --json number,comments`\nand grep PR comments for explicit `pr-refresh`, `make pr-refresh`,\nor `merged develop into branch`. Use only as corroboration.\n\nRecord:\n - Strict-base re-CI rate (primary proxy, %)\n - Explicit pr-refresh mentions (secondary, count)\nIf the primary proxy is < 5%, the queue is overkill even if Step 4\nmetrics later say otherwise \u2014 Step 6 should reflect that.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Estimate actual pr-refresh / staleness pain rate", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": [], "notes": "Run: `gh api repos/joeharris76/BenchBox --jq '{visibility,\nowner_type: .owner.type, owner_login: .owner.login,\nfull_name, default_branch, private}'`. Also inspect the branch\nprotection / ruleset surfaces from w1/w2 for any native merge-queue\nrule. Do NOT infer merge-queue availability from unrelated repo\nfields such as `.has_issues`; that field only says whether Issues\nare enabled.\n\nNative MQ generally requires org-owned repos (with plan-dependent\navailability for private repos). User-owned repos should be treated\nas \"native MQ unavailable\" unless GitHub's protection/ruleset API\nshows merge-queue rules can be applied. Record the evidence so Step 6\npicks the right path (native vs steward) without re-investigation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm native GitHub Merge Queue availability", "wid": "w6"}, {"evidence": null, "item_id": "dev-loop-step-0-verify-assumptions", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": "Single markdown file with: timestamp, raw outputs of w1-w6,\nconclusions, and explicit GO/NO-GO/RE-PLAN signal for each of\nSteps 1-4. Future agents read this before starting any downstream\nstep.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write decision record at _project/decisions/dev-loop-verification-2026-04-30.md", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "localize the test lock to per-worktree by default", "instead": "keep global default and offer override only", "why": "10 concurrent xdist runs (auto-workers) would exhaust CPU and produce timing-flake"}, {"dont": "remove pr-refresh target in this PR", "instead": "that target stays useful until then", "why": "branch protection still requires up-to-date until Step 3 lands"}, {"dont": "skip the blind-spot capture (w5)", "instead": "they are 6 small files", "why": "the protocol explicitly requires file-first capture and this is the natural home"}], "approach": "Order the work units to minimize risk: hooks first (w1) is the\nload-bearing change; fanout (w2) depends on hooks being opt-in;\nlock (w3) is independent but read tests/README.md:257 first to\nunderstand the existing rationale; docs (w4) reflect the new\nreality; blind-spots (w5) are the protocol-debt cleanup; PR (w6)\nships everything as one cohesive unit.\n\nFor w1, prefer wrapping hook commands in `bash -c` with the env-var\nguard \u2014 it's smaller than rewriting hook definitions and easier to\nrevert if something breaks.\n\nFor w2, the `xargs -P` approach is simpler than GNU parallel\n(parallel may not be installed). Keep the per-worktree output\nprefixed with the worktree name so logs remain readable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [], "deps": ["dev-loop-step-0-verify-assumptions"], "description": "First implementation step of the dev-loop simplification plan. Removes\nlocal-loop friction so agents can push and run `pr-fanout` without\nhook serialization, while preserving the global test lock that protects\nthe workstation from concurrent xdist storms.\n\nThis step also captures the consolidated L2 blind-spot findings from\nthe four-round merge-loop audit to `_project/blind-spots/`, satisfying\nthe protocol-debt accumulated during the planning rounds.\n\nSingle PR via `make pr-open`. Independently shippable; no dependency\non Steps 2-4 landing first.", "id": "dev-loop-step-1-local-loop-cleanup", "preserves": ["Existing pr-fanout output structure (one section per worktree) \u2014 only parallelism changes, not the report shape", "Existing test runtime when BENCHBOX_PREPUSH is set \u2014 no functional change to the hook itself", "Global test lock as the DEFAULT \u2014 agents that don't set the env var continue to be CPU-protected", "make pr-preflight as a working manual command (it is the human-facing escape hatch)"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 3 manual deployment)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-refresh target (Step 3 deprecates this; here it stays)"}, {"kind": "do_not_modify", "path_glob": "Makefile worktree-add, worktree-prune, worktree-pool-* targets (Step 2 owns these)"}, {"kind": "only_modify", "path_glob": ".pre-commit-config.yaml"}, {"kind": "only_modify", "path_glob": "AGENTS.md (manual-coordination and pre-push hook paragraphs)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (manual-coordination and pre-push hook paragraphs)"}, {"kind": "only_modify", "path_glob": "Makefile (pr-fanout target only)"}, {"kind": "only_modify", "path_glob": "_project/blind-spots/2026-04-30-*.md (6 new files)"}, {"kind": "only_modify", "path_glob": "tests/README.md (around line 257)"}, {"kind": "only_modify", "path_glob": "tests/conftest.py (lock-path logic only)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 1: opt-in pre-push, bounded fanout, conservative test lock", "verifications": [{"command": "BENCHBOX_PREPUSH= uv run -- pre-commit run --hook-stage pre-push --all-files 2>&1 | grep -cE '(Skipped|Passed)'", "description": "All pre-push hooks no-op when BENCHBOX_PREPUSH is unset", "expected": "matches the count of pre-push hooks in .pre-commit-config.yaml", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": "BENCHBOX_PREPUSH=1 uv run -- pre-commit run pr-preflight-fast-tests --hook-stage pre-push --all-files 2>&1 | tail -3", "description": "Pre-push hooks execute when BENCHBOX_PREPUSH=1", "expected": "hook runs (success or expected failure on uncommitted state); does not skip", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -E '^PR_FANOUT_JOBS \\?= ' Makefile", "description": "pr-fanout default parallelism is 4", "expected": "PR_FANOUT_JOBS ?= 4", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 3}, {"command": "BENCHBOX_TEST_LOCK_DIR=/tmp/bb-lock-test uv run -- python -m pytest -m fast -q tests/unit/ -x 2>&1 | tail -5 && ls /tmp/bb-lock-test/test.lock", "description": "Test lock honors BENCHBOX_TEST_LOCK_DIR override", "expected": "lock file present at /tmp/bb-lock-test/test.lock; tests run normally", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -c \"from pathlib import Path; from tests.conftest import _get_test_lock_path; print(_get_test_lock_path() == Path.home() / '.benchbox' / 'test.lock')\"", "description": "Default test lock still uses ~/.benchbox/", "expected": "True", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -cE '(Drain one at a time|One PR per file area|Sequential by design|pre-push hook.*runs the fast lane|re-runs the fast lane automatically)' CLAUDE.md AGENTS.md", "description": "Stale guidance removed from agent-facing docs", "expected": "0", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 6}, {"command": "grep -cE 'BENCHBOX_PREPUSH=1|pre-push hooks.*opt-in|make pr-preflight.*explicit|explicit.*make pr-preflight' CLAUDE.md AGENTS.md", "description": "Agent docs say pre-push fast lane is opt-in and pr-preflight is explicit", "expected": ">= 2", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 7}, {"command": "ls _project/blind-spots/2026-04-30-*.md | wc -l", "description": "Six blind-spot findings recorded", "expected": "6", "item_id": "dev-loop-step-1-local-loop-cleanup", "last_result": null, "last_run": null, "seq": 8}], "work": [{"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Audit `.pre-commit-config.yaml` for every hook with `stages:\n[pre-push]`. Currently includes at minimum:\n - `pr-preflight-fast-tests`\n - `timing-policy-fast-lane`\nWrap each entry's command in a guard:\n `if [ \"$BENCHBOX_PREPUSH\" = \"1\" ]; then ; else exit 0; fi`\nOr use the `args` / `entry` pattern that pre-commit supports.\nDefault behavior: hooks no-op unless env var is set. `make\npr-preflight` remains the explicit human-facing invocation\n(does not depend on the env var).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate all expensive pre-push hooks behind BENCHBOX_PREPUSH=1", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": ["w1"], "notes": "Change `pr-fanout` target in `Makefile` from sequential to parallel.\nAdd `PR_FANOUT_JOBS ?= 4` near the top. Use `xargs -P \"$(PR_FANOUT_JOBS)\"`\nor GNU `parallel -j \"$(PR_FANOUT_JOBS)\"` to fan out\n`make pr-open` calls across worktrees. Drop the\n\"Sequential by design\" comment block. Document the env var in\nthe help target.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make pr-fanout bounded-parallel with PR_FANOUT_JOBS=4 default", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Read `tests/conftest.py` and locate the lock acquisition logic\n(currently `~/.benchbox/test.lock`). Do NOT change the default \u2014\na global lock is what prevents 10 concurrent xdist runs from\nsaturating the workstation. Add an env-var-aware helper, e.g.\n`_get_test_lock_path()`, that returns:\n `Path(os.environ.get(\"BENCHBOX_TEST_LOCK_DIR\",\n Path.home() / \".benchbox\")) / \"test.lock\"`\nUpdate `Makefile` `test-unlock` target to honor the same env var.\nUpdate `tests/README.md:257` area to document the override and\nexplain WHY the default stays global.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keep global test lock; add BENCHBOX_TEST_LOCK_DIR override", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Remove from both files:\n - \"Drain one at a time, oldest first\" (and surrounding pr-refresh prose)\n - \"One PR per file area at a time\"\n - \"Sequential by design\" notes around pr-fanout\n - Any claim that the pre-push hook reruns the fast lane\n automatically on every push\nReplace with a one-paragraph summary of the new model: pool\nworktrees + auto-merge + post-merge auto-revert as safety net.\nUpdate the pre-push-hook guidance to say expensive pre-push hooks\nare opt-in via `BENCHBOX_PREPUSH=1`; `make pr-preflight` is the\nexplicit local gate for humans/agents that want the fast lane before\npushing.\nNote that this paragraph will be expanded by Steps 2 and 4 once\nthose land; here, just remove the now-incorrect guidance.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Strip manual-coordination guidance from CLAUDE.md and AGENTS.md", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": [], "notes": "Six findings from the four-round merge-loop audit (2026-04-30 session):\n 1. Original plan bundled \"stop babysitting PRs\" with \"promote\n develop to release-grade gating\" \u2014 separable decisions\n 2. PR CI on develop is already heavy today; reducing it is the\n highest-leverage standalone move\n 3. Test-lock localization is not a pure win at scale; the global\n lock has a real CPU-protection job (10 worktrees \u00d7 xdist auto)\n 4. A queue creates a single point of failure with no analog in\n current setup\n 5. Dwell time silently changes the agent contract (5 min today\n vs hours under a queue)\n 6. develop-post-merge.yml is the actual safety net; queue plan\n kept it as belt-and-suspenders without resolving the tension\nOne file per finding under `_project/blind-spots/2026-04-30-HHMMSS-.md`\nusing the frontmatter schema from `_project/blind-spots/README.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture consolidated L2 blind-spot findings to _project/blind-spots/", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-1-local-loop-cleanup", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Run `make lint format typecheck`, then\n`BENCHBOX_PREPUSH=1 make pr-preflight` to confirm the opt-in path\nstill works. Then `make pr-open`. Do NOT enable the new env var\nglobally; the point is that absence-of-var is the new default.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run validation locally and open the PR", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "auto-reap stale claims", "instead": "surface as 'stale' in status, require explicit `worktree-pool-reset` to free", "why": "an agent crash mid-session may leave valuable uncommitted work"}, {"dont": "check out the local develop branch in pool worktrees", "instead": "use detached origin/develop as the Free state", "why": "the main clone owns develop and Git forbids the same branch in multiple worktrees"}, {"dont": "delete pool-NN worktrees in worktree-prune", "instead": "prune is for legacy non-pool worktrees only", "why": "the pool is the contract"}, {"dont": "hardcode pool size to 10", "instead": "read POOL_SIZE ?= 10 from the environment", "why": "the workstation may want a different cap"}, {"dont": "remove make worktree-add in this PR", "instead": "deprecate-with-notice now, remove next release", "why": "in-flight work in non-pool worktrees needs continuity"}, {"dont": "skip portable locking on claim selection", "instead": "atomic selection is required", "why": "two concurrent agents racing to claim would both pick pool-01 and one would clobber the other's branch"}], "approach": "Build the targets in dependency order: pool-init (w1) is foundational,\nthen claim (w2) and release (w3) are the core lifecycle, then status\n(w4) and reset (w5) are operational tooling, then docs (w6) and\ndeprecation (w7-w8) update the rest of the surface.\n\nUse a portable lock wrapper for atomic claim. `flock(1)` is not present\non the macOS workstation, so the Makefile cannot assume it. Prefer\n`lockf` on macOS, `flock` on Linux when available, or a small Python\nhelper that calls `fcntl.flock`. Lock file stays at\n`/.git/pool.lock`. Single shared lock across all pool\nworktrees \u2014 claim is fast (seconds) so contention is fine.\n\nFor state detection in worktree-pool-status:\n - free: `git symbolic-ref -q --short HEAD` is empty AND\n `git rev-parse HEAD` == `git rev-parse origin/develop` AND\n `git status --porcelain` is empty\n - claimed: not on develop, working tree may or may not be clean\n - stale: claimed AND `gh pr view --json state` returns MERGED\n - dirty: claimed AND `git status --porcelain` non-empty\n\nRefer to existing `make worktree-add` and `make worktree-prune`\ntargets for patterns; reuse their git-worktree command structure.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-2-worktree-pool", "id": 376, "reason": "Reasonable enhancement but out of scope; claim already updates at claim-time. Revisit if pool-status shows worktrees consistently lagging develop.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Auto-update-while-idle was conditioned on pool-status showing worktrees lagging develop; no evidence that trigger occurred; claim-time update already mitigates.", "summary": "Auto-update of pool worktrees while idle (background pull of develop)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-2-worktree-pool", "id": 377, "reason": "Pool size of 10 + global test lock is sufficient for now. Add only if metrics in Step 5 show resource-contention failures.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Gated on resource-contention data from Step 5 (open, active); measurement window hasn't concluded so trigger unmet. (tracked by dev-loop-step-5-measurement-window)", "summary": "Per-worktree resource quotas (CPU/memory limits)"}], "deps": ["dev-loop-step-0-verify-assumptions"], "description": "Replace the create-and-destroy worktree model with a retained pool of\nworktrees. Default pool size is 10 (`POOL_SIZE ?= 10`), but the targets\nmust keep the size configurable so the workstation can be tuned without\nediting Makefile logic. New agent sessions claim an available worktree,\nupdate it from current `develop`, work there until merge, then release\nit back to the pool.\n\nThis formalizes how concurrent agents share the workstation. The default\nsize of 10 bounds routine parallelism (which is also why Step 1 keeps\nthe test lock global \u2014 even with the pool bounded, 10 concurrent xdist\nruns would saturate the machine).\n\nSingle PR via `make pr-open`.\n\n## Post-merge runbook (not part of the PR diff)\n\nAfter this PR merges to develop, perform on Joe's workstation:\n\n make worktree-pool-init\n\nThen verify:\n - 10 worktrees present at `BenchBox.pool-01..10`\n - Each is detached at `origin/develop` and clean\n - Each has `.venv/` populated\n - `make worktree-pool-status` shows all 10 free\n - Each free worktree is detached and clean, NOT on the local\n `develop` branch; claim refreshes free slots to current\n `origin/develop`\n\nTracked as a follow-up checklist item in the PR description, not as a\nwork unit, because the work unit DAG is for code changes that can be\nmarked done by `todo-cli done ` against a commit.", "id": "dev-loop-step-2-worktree-pool", "preserves": ["Atomic claim semantics \u2014 two concurrent `make worktree-claim` calls must always pick different pool-NN worktrees, never collide", "Existing make worktree-add target keeps working \u2014 only adds deprecation notice; do not break legacy non-pool worktree workflows mid-session", "Pool worktrees retain their .venv/ across release/claim cycles \u2014 uv sync only runs at claim time after a fetch, and only if pyproject.toml/uv.lock changed", "make worktree-pool-init is idempotent \u2014 re-running must not destroy or reset existing pool worktrees"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": ".pre-commit-config.yaml (Step 1 owns pre-push gating)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 3 manual deployment)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-* targets (Steps 1 and 3 own these)"}, {"kind": "do_not_modify", "path_glob": "tests/conftest.py (Step 1 owns the test-lock helper)"}, {"kind": "only_modify", "path_glob": ".gitignore (add .git/pool.lock if needed)"}, {"kind": "only_modify", "path_glob": "AGENTS.md (Session start / Worktree section)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (Session start, Worktrees, and Pre-approved Commands sections)"}, {"kind": "only_modify", "path_glob": "Makefile (worktree-* targets only)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 2: fixed pool of 10 retained worktrees with claim/release", "verifications": [{"command": "tmp=$(mktemp -d); WORKTREE_POOL_PARENT=$tmp POOL_SIZE=2 make worktree-pool-init && find \"$tmp\" -maxdepth 1 -type d -name 'BenchBox.pool-*' | wc -l && WORKTREE_POOL_PARENT=$tmp POOL_SIZE=2 make worktree-pool-init; for wt in \"$tmp\"/BenchBox.pool-*; do git worktree remove --force \"$wt\" >/dev/null 2>&1 || true; done; rm -rf \"$tmp\"", "description": "make worktree-pool-init creates 10 worktrees idempotently", "expected": "2 (first run) and second run completes without error; disposable worktrees are removed", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 1}, {"command": "make worktree-claim BRANCH=bad-name 2>&1 | grep -c 'must match'", "description": "worktree-claim refuses invalid branch names", "expected": ">= 1", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 2}, {"command": "tmp=$(mktemp -d); WORKTREE_POOL_PARENT=$tmp POOL_SIZE=1 make worktree-pool-init >/dev/null && WORKTREE_POOL_PARENT=$tmp POOL_SIZE=1 make worktree-claim BRANCH=chore/test-pool-claim 2>&1 | tee /tmp/bb-pool-claim.out | grep -E 'WORKTREE_PATH=.*pool-[0-9]+'; wt=$(sed -n 's/^WORKTREE_PATH=//p' /tmp/bb-pool-claim.out); git worktree remove --force \"$wt\" >/dev/null 2>&1 || true; git branch -D chore/test-pool-claim >/dev/null 2>&1 || true; rm -rf \"$tmp\" /tmp/bb-pool-claim.out", "description": "worktree-claim selects an available pool worktree", "expected": "WORKTREE_PATH=", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 3}, {"command": "make worktree-pool-status 2>&1 | grep -cE '^pool-[0-9]+'", "description": "worktree-pool-status reports all pool worktrees", "expected": "10", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 4}, {"command": "tmp=$(mktemp -d); WORKTREE_POOL_PARENT=$tmp POOL_SIZE=3 make worktree-pool-init >/dev/null; for i in 1 2 3; do (WORKTREE_POOL_PARENT=$tmp POOL_SIZE=3 make worktree-claim BRANCH=chore/race-test-$i >\"$tmp/out-$i\" 2>&1 &); done; wait; cat \"$tmp\"/out-* | grep -E '^WORKTREE_PATH=' | sort -u | wc -l; for wt in \"$tmp\"/BenchBox.pool-*; do git worktree remove --force \"$wt\" >/dev/null 2>&1 || true; done; for i in 1 2 3; do git branch -D chore/race-test-$i >/dev/null 2>&1 || true; done; rm -rf \"$tmp\"", "description": "Concurrent claims do not race", "expected": "3 distinct claims, each on a different pool-NN worktree", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 5}, {"command": "make worktree-prune 2>&1 | grep -c 'Skipping pool worktree'", "description": "worktree-prune skips pool worktrees", "expected": ">= 0 (skips logged for any pool worktrees on develop)", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 6}, {"command": "make worktree-add BRANCH=chore/legacy-test 2>&1 | grep -c 'DEPRECATED'", "description": "worktree-add prints deprecation notice", "expected": ">= 1", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 7}, {"command": "grep -cE 'worktree-claim|worktree-release|worktree-pool-status|worktree-pool-init|worktree-pool-reset' CLAUDE.md", "description": "Claude command permissions list new pool commands explicitly", "expected": ">= 5", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 8}, {"command": "grep -cE 'worktree-claim|worktree-release|worktree-pool-status|worktree-pool-reset' CLAUDE.md AGENTS.md", "description": "Agent-facing docs describe claim/release pool lifecycle", "expected": ">= 8", "item_id": "dev-loop-step-2-worktree-pool", "last_result": null, "last_run": null, "seq": 9}], "work": [{"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": [], "notes": "One-time bootstrap. Add `POOL_SIZE ?= 10` and\n`WORKTREE_POOL_PARENT ?= ..` so verification can create disposable\npools outside the normal sibling paths. With defaults, creates\n`BenchBox.pool-01` through `BenchBox.pool-10` as siblings of the\nmain clone, each in detached HEAD at `origin/develop`, each with\n`uv sync --group dev` run, each with `pre-commit install`.\nIdempotent: if a pool-NN worktree already exists, leave it alone\n(do not touch its branch state).\nPrint summary table at end: pool-NN | path | branch.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-pool-init target \u2014 creates pool-01..POOL_SIZE from develop", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Selects the first **Free** pool worktree (detached HEAD, working\ntree clean, no local feature branch checked out). Claim fetches\nand resets the selected worktree to current `origin/develop`\nbefore creating the branch. Pool worktrees must not check out the\nlocal `develop` branch because the main clone keeps `develop`\nchecked out and Git does not allow the same branch in multiple\nworktrees.\nSteps:\n 1. Acquire a portable file lock on `/.git/pool.lock`\n (timeout 30s). Use `lockf` on macOS, `flock` on Linux if\n available, or a tiny Python helper using `fcntl.flock` so the\n target works on Joe's macOS workstation and CI.\n 2. Iterate pool-01..POOL_SIZE; first match is \"Free\"\n 3. In that worktree: `git fetch origin`, `git reset --hard\n origin/develop`, `git checkout -b $BRANCH`, `uv sync --group dev`\n 4. Release flock\n 5. Print: `WORKTREE_PATH=`\nRefuses (exit 1) if pool is fully claimed; suggests\n`make worktree-pool-status` to investigate.\nRefuses if BRANCH is empty or doesn't match `^(chore|fix|feat|docs)/.+`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-claim BRANCH= \u2014 atomic claim under portable lock", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Run from inside a pool worktree (refuses if cwd is not a pool-NN\npath). Steps:\n 1. Identify current branch\n 2. Verify PR for this branch is in MERGED state via\n `gh pr view --json state` (refuse if not, with explicit\n \"open or close PR first\" message; FORCE=1 escapes)\n 3. `git checkout --detach origin/develop`\n 4. `git fetch origin`\n 5. `git reset --hard origin/develop`\n 6. `git branch -D ` (the one just released)\n 7. `git remote prune origin`\nWorktree returns to **Free** state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-release \u2014 verify merged, return to develop", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Prints table with columns: pool-NN | path | branch | state |\nclaim_age. State is one of:\n - free: detached HEAD, working tree clean, no untracked files\n outside expected (`.benchbox/`, `.venv/`, etc.); claim\n refreshes to current origin/develop before checkout\n - claimed: HEAD on a non-develop branch\n - stale: claimed branch has been merged upstream (gh pr view\n returns MERGED) but worktree still on the branch\n - dirty: claimed with uncommitted changes \u2014 needs review\nclaim_age is `git log -1 --format=%ar` of the most recent commit\non the current branch (or \"\u2014\" for free).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-pool-status \u2014 tabular state listing", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Forcibly returns pool-NN to Free state. Refuses if working tree\nhas uncommitted changes UNLESS `FORCE=1` is set. Even with FORCE,\nprints what it's about to discard and requires interactive\nconfirmation (read -p). Useful when an agent died mid-session\nleaving the worktree in an inconsistent state.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make worktree-pool-reset POOL=NN \u2014 manual escape hatch", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w2", "w3", "w4"], "notes": "Replace the existing \"Session start\" instructions:\n - Old: `make worktree-add BRANCH=/`\n - New: `make worktree-claim BRANCH=/`\nAdd a paragraph explaining the pool model: 10 worktrees,\nretained indefinitely, claim/release lifecycle. Document the\nStale recovery flow (run `worktree-pool-status`, then\n`worktree-pool-reset POOL=NN` if needed).\nNote that `make worktree-add` remains as a deprecated alias for\none release with a stderr deprecation notice.\n\nUpdate Claude-specific pre-approved command buckets in CLAUDE.md:\n - Read-only: `make worktree-pool-status`, `make worktree-list`,\n `git worktree list*`\n - Write, feature/pool worktrees only: `make worktree-claim\n BRANCH=...`, `make worktree-release`\n - Manual/admin escape hatches, not broad auto-allow:\n `make worktree-pool-init`, `make worktree-pool-reset POOL=NN`\nAGENTS.md should describe the same lifecycle in prose even though it\ndoes not have Claude's pre-approved-command table.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update Session-start docs in CLAUDE.md and AGENTS.md", "wid": "w6"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w2"], "notes": "`worktree-add` keeps current semantics but prints to stderr:\n\"DEPRECATED: use `make worktree-claim BRANCH=...` instead. The\npool model retains worktrees rather than creating new ones.\n`worktree-add` will be removed in the next release.\" Then\nproceeds normally. Pool-NN worktrees are never created via\nworktree-add; they only come from worktree-pool-init.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Mark worktree-add as deprecated alias of worktree-claim", "wid": "w7"}, {"evidence": null, "item_id": "dev-loop-step-2-worktree-pool", "needs": ["w1"], "notes": "Pool worktrees are retained indefinitely; `worktree-prune` should\nno longer be a routine end-of-session command. Modify the target\nto:\n 1. Detect pool-NN paths and skip them\n 2. Print: \"Skipping pool worktree pool-NN (retained)\"\n 3. Continue pruning legacy non-pool worktrees as today\nUpdate help text to note: `worktree-prune` is for legacy\nworktrees only; pool worktrees are released via `worktree-release`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update worktree-prune to skip pool-NN worktrees", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "block Step 3 on Step 3a's path umbrella", "instead": "keep Step 3 scoped to non-required job cost", "why": "content-only skip is a separate follow-up"}, {"dont": "delete heavy checks outright", "instead": "keep them in nightly/workflow_dispatch or an explicit opt-in lane", "why": "moved checks still catch compatibility regressions"}, {"dont": "deprecate or remove `make pr-refresh` in Step 3", "instead": "keep it as a rare branch-drift escape hatch", "why": "the old rationale was stale strict-base churn"}, {"dont": "introduce many copy-pasted branch conditions without a design note", "instead": "prefer split workflows or a small documented condition pattern", "why": "future jobs will drift"}, {"dont": "lighten main PR or tag-release validation", "instead": "preserve release-grade coverage there", "why": "main is release-only and tags publish artifacts"}, {"dont": "patch develop branch protection or rulesets in Step 3", "instead": "record the live state and leave it unchanged", "why": "Step 0 already found strict-base false and required checks reduced"}], "approach": "Treat Step 3 as a workflow routing change. Required checks are already\nminimal, so the implementation should not touch GitHub protection/ruleset\nsettings. The highest-risk design point is check-name stability: GitHub\nrequired contexts are name-sensitive, and Step 3a may later add an umbrella\nresult check. Keep the existing required context names stable during Step 3\nunless the implementation deliberately leaves rulesets untouched and proves\nGitHub still reports the required checks.\n\nPrefer moving whole non-required lanes to a nightly/manual workflow over\nsprinkling branch conditions through every job. A split workflow makes\nownership visible: routine develop PR checks prove fast code health, while\nnightly/manual checks preserve compatibility and release confidence. If a\njob must stay in `test.yml` for status-name compatibility, gate it narrowly\nand document the reason next to the condition.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "id": 378, "reason": "Separate TODO already exists and must preserve the required contexts once Step 3's CI routing is stable.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: _project/DONE/main/active/dev-loop-step-3a-path-based-ci-skip.yaml confirms the referenced separate TODO exists and is already Completed.", "summary": "Step 3a path-aware required-check umbrella"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "id": 379, "reason": "Step 6 territory, and Step 0 found no current strict-base re-CI pain plus no exposed merge-queue rule for this user-owned repository.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Merge Queue migration is explicitly 'Step 6 territory', which is the open dev-loop-step-6-queue-decision-gate item.", "summary": "Native GitHub Merge Queue migration"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "id": 380, "reason": "Could reduce cost, but adds operational burden. Revisit only if Step 5 runner-minute metrics justify it.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Self-hosted runner adoption is explicitly gated on dev-loop Step 5 runner-minute metrics justifying it; Step 5 measurement is still in progress (dev-loop-step-5-measurement-window).", "summary": "Self-hosted runner adoption"}], "deps": ["dev-loop-step-0-verify-assumptions"], "description": "Step 0 invalidated the original strict-base diagnosis. `develop` has no\nclassic branch protection object, and the active `develop-squash-only`\nruleset already requires only `lint` and `test (ubuntu-latest, 3.12)`;\n`strict_required_status_checks_policy` is already `false`.\n\nThe remaining load-bearing Step 3 work is CI-cost reduction, not branch\nprotection. `.github/workflows/test.yml` still runs a broad non-required\nPR-tier workflow on pull requests to `develop`: a multi-version/OS test\nmatrix, security, smoke/integration lanes, table-format checks, package\ninstall validation, parity, and non-blocking PySpark. The Step 0 baseline\nmeasured recent `Tests` runs at P50 16.70 minutes and P95 19.47 minutes.\n\nReplanned Step 3 should keep the existing required contexts unchanged and\nmove or gate only non-required heavy lanes so ordinary `develop` PRs stop\nspending runner time on release-grade validation. Full validation must\nremain available through nightly and workflow_dispatch, and `main` PR/tag\nrelease coverage must remain release-grade.\n\nDesigned-together follow-up: Step 3a (`dev-loop-step-3a-path-based-ci-skip`)\nremains valid after this replan. Step 3 reduces non-required job breadth for\ncode PRs; Step 3a separately adds path-aware behavior for content-only PRs\nwhile preserving the already-minimal required contexts.", "id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "preserves": ["Full validation for main PRs and tag releases remains release-grade and must not be lightened as a side effect of reducing develop PR cost", "Heavy checks removed from routine develop PR execution remain runnable through nightly/workflow_dispatch or an explicit opt-in label", "No Step 3 branch-protection or ruleset PATCH; Step 0 already found the intended strict-base and required-check state live", "The develop required contexts remain exactly `lint` and `test (ubuntu-latest, 3.12)` unless a separate Step 3a deployment explicitly changes the ruleset required-check list", "`make pr-refresh` remains available as an escape hatch for real branch drift; Step 3 must not remove or deprecate it based on the stale strict-base premise"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/lint.yml unless w1 proves required-check naming requires it"}, {"kind": "do_not_modify", "path_glob": ".pre-commit-config.yaml (Step 1 owns pre-push gating)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (read-only verification only)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-refresh, pr-open, pr-fanout, and worktree targets"}, {"kind": "do_not_modify", "path_glob": "tests/ (no product test changes for CI routing)"}, {"kind": "only_modify", "path_glob": ".github/workflows/nightly.yml (new or updated)"}, {"kind": "only_modify", "path_glob": ".github/workflows/release.yml (audit/comment only if needed)"}, {"kind": "only_modify", "path_glob": ".github/workflows/test.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md (CI guidance only)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (CI guidance only)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-3-ci-rebalance-and-strict-base-off.yaml (replan only)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 3: reduce non-required develop PR CI cost", "verifications": [{"command": "repo=$(gh repo view --json nameWithOwner --jq .nameWithOwner)\nfound=0\nfor ruleset_id in $(gh api \"repos/${repo}/rulesets\" --jq '.[] | select(.target == \"branch\" and ((.conditions.ref_name.include // []) | any(. == \"refs/heads/develop\"))) | .id'); do\n found=1\n gh api \"repos/${repo}/rulesets/${ruleset_id}\" --jq '.rules[] | select(.type == \"required_status_checks\") | .parameters'\ndone\ntest \"$found\" = \"1\"\n", "description": "Develop ruleset still has strict-base off with only the two lightweight required contexts", "expected": "strict_required_status_checks_policy is false; required_status_checks are the lightweight develop PR contexts", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 1}, {"command": "gh api repos/joeharris76/BenchBox/branches/develop/protection --jq '.required_status_checks' 2>/dev/null || echo classic-protection-not-configured", "description": "Classic branch protection is absent or has no conflicting required-check policy", "expected": "classic-protection-not-configured OR no broader/strict required checks", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 2}, {"command": "branch=$(git branch --show-current)\npr_base=$(gh pr view \"$branch\" --json baseRefName --jq .baseRefName)\n[ \"$pr_base\" = \"develop\" ] || { echo \"PR for $branch targets $pr_base, not develop\"; exit 1; }\nrun_id=$(gh run list --workflow test.yml --branch \"$branch\" --event pull_request --limit 1 --json databaseId --jq '.[0].databaseId')\ngh run view \"$run_id\" --json jobs --jq '[.jobs[] | select(.conclusion != \"skipped\") | .name] | sort[]'\n", "description": "Routine develop PR Tests workflow no longer queues broad non-required lanes", "expected": "Only non-skipped required/lightweight develop PR job names, plus any explicitly documented lightweight umbrella/content jobs", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 3}, {"command": "gh workflow list --json name,state,path --jq '.[] | select((.name | test(\"nightly|Nightly\")) or (.path | test(\"nightly\")))'", "description": "Nightly/manual workflow exists and is active for broad validation", "expected": "nightly workflow exists and state is active", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 4}, {"command": "rg -n 'pull_request:|branches:|main|tags:|matrix:|macos-latest|windows-latest|test-package|release' .github/workflows/test.yml .github/workflows/nightly.yml .github/workflows/release.yml", "description": "Main PR and release paths still expose full validation coverage", "expected": "main/tag release validation paths and full validation lanes are present outside nightly-only execution", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 5}, {"command": "rg -n 'strict-base|strict|required checks|nightly|workflow_dispatch|release-grade|pr-refresh' CLAUDE.md AGENTS.md", "description": "Agent docs describe the current CI contract without stale strict-base/deprecation claims", "expected": "docs state strict-base is already off, required checks are minimal, nightly/manual carries heavy validation, and pr-refresh remains an escape hatch", "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": [], "notes": "Start from `_project/decisions/dev-loop-verification-2026-04-30.md`,\nthen re-read live GitHub state before editing workflows. Record:\n - Classic branch protection for `develop` remains absent or has no\n required-check drift.\n - Ruleset `develop-squash-only` still requires exactly `lint` and\n `test (ubuntu-latest, 3.12)`.\n - `strict_required_status_checks_policy` remains `false`.\n - Every job in `.github/workflows/test.yml` is classified as required,\n non-required PR-tier, push/main, label-gated, or manual/nightly.\n\nThis is a gate for the rest of Step 3. If required contexts changed\nafter Step 0, stop and re-evaluate rather than editing CI around stale\nassumptions.\n\nImplementation record (2026-04-30):\n - `gh api repos/joeharris76/BenchBox/branches/develop/protection`\n returned HTTP 404, so classic branch protection remains absent.\n - `gh api repos/joeharris76/BenchBox/rulesets/15611785` returned\n required status checks `lint` and `test (ubuntu-latest, 3.12)`;\n `strict_required_status_checks_policy` is `false`.\n - Required PR contexts: `lint` from `.github/workflows/lint.yml` and\n `test (ubuntu-latest, 3.12)` from the `Tests` matrix.\n - Non-required routine PR contexts in `.github/workflows/test.yml`:\n `test (ubuntu-latest, 3.10)`, `test (ubuntu-latest, 3.13)`,\n `test (macos-latest, 3.12)`, `test (windows-latest, 3.12)`,\n `test (windows-latest, 3.10)`, `check-test-patterns`, `security`,\n `integration-smoke` on Ubuntu and Windows, `integration-table-formats`,\n `test-package` on Ubuntu and Windows, `parity`, and `pyspark-tests`.\n - Label-gated/push-main context: `integration` runs on `push` or PRs\n carrying `run-integration`; it is skipped for ordinary develop PRs.\n - Manual/nightly broad validation does not exist yet; Step 3 must add\n it before removing non-required routine PR lanes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconfirm protection/ruleset state and classify PR-tier jobs by requirement status", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w1"], "notes": "Choose the smallest maintainable workflow shape that keeps the two\nrequired contexts stable. The default recommendation is:\n - Keep `lint` in its existing workflow.\n - Keep `test (ubuntu-latest, 3.12)` reporting under the existing Tests\n workflow/job identity for code PRs.\n - Remove, split, or job-gate non-required develop PR lanes from the\n routine `pull_request` path.\n - Preserve label-gated/manual execution for heavier checks when a PR\n intentionally needs them.\n\nPrefer a clear workflow split (`test.yml` for PR-tier required checks,\n`nightly.yml` for broad validation) over repeating fragile job-level\nconditions everywhere. If a split would break required check names,\nuse targeted conditions instead and document why.\n\nImplementation record (2026-04-30):\n - Kept `.github/workflows/lint.yml` unchanged so the required `lint`\n context stays stable.\n - Replaced the broad `test` matrix with one explicit required job named\n `test (ubuntu-latest, 3.12)`, preserving the required context while\n avoiding matrix evaluation for routine develop PRs.\n - Added `compat-test` for non-required OS/Python combinations on\n main/push surfaces.\n - Added develop-PR gates to non-required `test.yml` jobs:\n `compat-test`, `check-test-patterns`, `security`,\n `integration-smoke`, `integration-table-formats`, `test-package`,\n `parity`, and `pyspark-tests`.\n - Left `integration` label-gated as-is so `run-integration` can still\n opt a PR into the heavier integration lane intentionally.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the develop PR trigger policy for non-required test.yml jobs", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w2"], "notes": "Add or update `.github/workflows/nightly.yml` so the jobs removed from\nroutine develop PRs still run on:\n - `workflow_dispatch`\n - a low-contention nightly schedule\n\nThe nightly/manual path should cover the multi-OS/Python matrix,\nintegration smoke and table-format checks, package install validation,\nparity, security audit, and PySpark lane unless w1 finds a specific lane\nbelongs somewhere else. Use existing commands and dependency setup from\n`test.yml`; do not invent a parallel test stack.\n\nIf failure issue automation is added, keep it narrow and idempotent:\none open issue per failing nightly class, labeled `incident:nightly-red`,\nwith the run URL and failing job names. If that automation is too large\nfor this PR, defer it explicitly and ensure nightly failures are still\nvisible through GitHub notifications.\n\nImplementation record (2026-04-30):\n - Added `.github/workflows/nightly.yml` with `workflow_dispatch` and a\n daily 06:00 UTC schedule.\n - Nightly coverage includes the full OS/Python fast-test matrix,\n patch-safety check, security audit, integration smoke,\n table-format integration, full local integration, package install,\n parity, and non-blocking PySpark lanes.\n - Reused the existing `uv` setup and test commands from `test.yml`.\n - Deferred automatic issue creation to keep this PR scoped; failures\n remain visible through GitHub Actions notifications and manual\n workflow dispatch is available for reruns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Move broad validation to nightly/workflow_dispatch without losing coverage", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w2", "w3"], "notes": "Audit `.github/workflows/test.yml`, `.github/workflows/nightly.yml`, and\n`.github/workflows/release.yml` after the split/gating work. Confirm\n`pull_request` to `main` and `push`/tag release paths still run\nrelease-grade coverage automatically, not only through nightly:\n - Full OS/Python compatibility coverage remains on a main/release path.\n - Package build/install validation remains on a main/release path.\n - Release-flow checks are unchanged.\n\nA comment in the workflow is acceptable when it prevents future drift,\nbut avoid comment-only churn unless it clarifies a real branch-specific\ninvariant.\n\nImplementation record (2026-04-30):\n - Audited `.github/workflows/test.yml`: `pull_request` to `main` and\n `push` to `main` still run the required Ubuntu 3.12 fast-test job,\n the additional compatibility matrix, package install, security,\n smoke/table-format, and parity lanes. The new gates exclude only\n `pull_request` events whose base branch is `develop`.\n - Audited `.github/workflows/validate-main-pr.yml`: main PRs remain\n restricted to release branches through `validate-base`.\n - Audited `.github/workflows/release.yml`: `push` tags matching `v*`\n still run dependency bounds, reproducible build, package checks,\n publish, GitHub release creation, and post-publish installation\n across Ubuntu/macOS/Windows and Python 3.10/3.12/3.13 coverage.\n - No release workflow edits were needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Preserve main PR and tag-release validation semantics", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w2", "w3", "w4"], "notes": "Update `CLAUDE.md` and `AGENTS.md` only where the workflow contract is\nwrong or incomplete. The guidance should say:\n - Develop required checks are already minimal: `lint` and\n `test (ubuntu-latest, 3.12)`.\n - Strict-base is already off for `develop`; do not describe Step 3 as\n a strict-base deployment.\n - Heavy non-required lanes moved out of routine develop PR execution\n remain available through nightly/workflow_dispatch or explicit labels.\n - `main` PRs and tag releases retain release-grade validation.\n - `make pr-refresh` remains an escape hatch for genuine branch drift;\n do not deprecate it solely because strict-base was already off.\n\nImplementation record (2026-04-30):\n - Updated `CLAUDE.md` and `AGENTS.md` to state that develop required\n checks are intentionally minimal and strict-base is already off.\n - Documented the split between routine develop PR gates and broad\n nightly/workflow_dispatch validation.\n - Preserved `make pr-refresh` as an escape hatch rather than a\n deprecated path.\n - Documented that main PRs and tag releases keep release-grade\n validation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update agent docs for the replanned CI contract", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-3-ci-rebalance-and-strict-base-off", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Run focused workflow/TODO validation first, then the repo preflight\nappropriate to the code touched. For workflow-only changes, at minimum:\n - YAML validation on edited workflow and TODO files\n - `uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph`\n - `make pr-preflight`\n - `/code review W6` or equivalent adversarial review of the diff\n - `make pr-open`\n\nThe PR description must state that no branch-protection or ruleset PATCH\nis part of Step 3 because Step 0 already proved the desired required\ncontexts and strict-base setting are live on `develop`.\n\nImplementation record (2026-04-30):\n - Rebased the implementation branch onto `origin/develop` after the\n Step 3 replan PR merged.\n - Validated edited YAML files with `pre-commit` `check-yaml`.\n - Ran `todo_cli.py check-graph`; graph validation passed.\n - Reconfirmed the live `develop-squash-only` ruleset still requires\n only `lint` and `test (ubuntu-latest, 3.12)` with strict-base off.\n - Reconfirmed classic branch protection remains absent.\n - Ran `make pr-preflight`; CI lint checks passed and fast tests\n reported `21268 passed, 6 skipped`.\n - Final branch review found no blocking issues.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify locally, review the change, and open the PR", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "apply branch-protection changes before the workflow PR lands", "instead": "merge workflow changes first, then patch branch protection/rulesets", "why": "the required check name will not exist on develop yet"}, {"dont": "assume docs.yml protects develop docs PRs", "instead": "add develop content validation or classify docs/** as code", "why": "it currently triggers PR checks for main only"}, {"dont": "classify unknown paths as non-code", "instead": "default unknown paths to full PR-tier", "why": "new top-level areas can affect builds/tests and a false green weakens branch protection"}, {"dont": "duplicate the path ruleset between CI and local pr-preflight", "instead": "share via `.github/path-filters.yml` or a single shell helper", "why": "they will drift and the local skip will lie to agents"}, {"dont": "make ci-required-result the only required check while required lint remains isolated in lint.yml", "instead": "move the develop PR lint gate under the umbrella first", "why": "the result job cannot aggregate another workflow"}, {"dont": "mark .gitignore or .gitattributes as safe content-only", "instead": "route them through full PR-tier or a blocking gitignore-lint aggregation", "why": "they affect repository behavior and tracked-file conflict guardrails"}, {"dont": "mark Makefile changes as non-code", "instead": "keep Makefile in the code path list", "why": "Makefile edits change developer-facing surface and have caused real regressions"}, {"dont": "pin `dorny/paths-filter@v3` by tag", "instead": "pin by SHA", "why": "tags are mutable and a compromised release could exfil secrets"}, {"dont": "skip all validation on content-only PRs", "instead": "run a cheap content guard while skipping Python fast tests", "why": "markdown/YAML/TODO/docs errors are relevant and PR #66/#67 surfaced exactly that class of issue"}, {"dont": "skip the smoke-test (w4)", "instead": "fork-test before enabling on develop", "why": "a buggy umbrella that always greens would make develop unprotected and you would not notice until something breaks"}, {"dont": "use top-level `paths:` / `paths-ignore:` filters on the workflow `on:` block", "instead": "use job-level `if:` with an umbrella instead", "why": "GitHub treats workflows-not-run as 'pending' for required-checks, leaving the PR unmergeable"}], "approach": "Minimum implementation surface: `.github/path-filters.yml`\n(ruleset), the PR-tier workflow (`test.yml` or `pr.yml`) that adds\n`ci-paths`, `content-guard`, `code-lint`, `code-test`, and\n`ci-required-result`, plus a small local helper for `make pr-preflight`\nand the pre-push hook.\n\nFinal implementation uses `scripts/path_filter_decision.py`, a stdlib-only\nhelper, instead of a third-party path-filter action. That keeps CI and local\n`make pr-preflight` on the same code path, avoids a mutable action tag, and\nlets content-only PRs classify paths before dependency installation.\n\nRequired-check aggregation has one hard GitHub Actions constraint:\n`needs:` only works inside one workflow. If the required `lint` gate\nstays in `.github/workflows/lint.yml`, `ci-required-result` cannot\nobserve it. Move/duplicate the required develop PR lint/type work into\nthe umbrella workflow before making `ci-required-result` the only\nrequired check.\n\nFor the umbrella aggregation step, the explicit pass/fail script\n(rather than relying on `needs:` propagation alone) is intentional:\nit makes the \"content-only can pass without Python tests\" semantic visible in the workflow\nlog, and it correctly handles the case where `needs.lint.result` is\nthe literal string `\"skipped\"` (which evaluates truthy in some\ncomparison contexts).\n\nSequence the rollout: workflow changes first (w6 merged to develop),\nTHEN branch-protection/ruleset change (w7). Reverse order leaves develop\nunmergeable for whatever interval because the required umbrella check name\nwill not exist on the base branch yet.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 144, "reason": "Post-merge runs are the safety net; skipping them based on paths could mask a code regression that only manifests when a doc-only commit lands on top of an existing code change. Keep post-merge always-runs.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Deliberately rejected: post-merge CI is kept always-on as the regression safety net, explicitly not path-skipped.", "summary": "Path-based skip on Step 4 develop-post-merge.yml"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 145, "reason": "Nightly is scheduled, not PR-triggered. The 'doc-only' question doesn't apply \u2014 nightly runs the full matrix regardless of recent commits.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly states path-based skip does not apply to nightly.yml since it's schedule-triggered and always runs full matrix.", "summary": "Path-based skip on nightly.yml"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 146, "reason": "Premature optimization. dorny/paths-filter completes in seconds; caching adds complexity for negligible gain.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly called premature optimization (dorny/paths-filter already completes in seconds); no need signal.", "summary": "Cache the path-filter ruleset across steps for performance"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-3a-path-based-ci-skip", "id": 147, "reason": "Bigger redesign than the umbrella. Step 3a only separates safe content-only PRs from everything else. Revisit only if Step 5 metrics show it's worth the complexity.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Per-path CI matrix beyond content-vs-code gated on Step 5 metrics; dev-loop-step-5-measurement-window is still open/active, condition unevaluated.", "summary": "Per-path matrix dispatch beyond content-vs-code (e.g., only run explorer tests for results-explorer changes)"}], "deps": ["dev-loop-step-1-local-loop-cleanup", "dev-loop-step-3-ci-rebalance-and-strict-base-off"], "description": "Companion to Step 3. After Step 3 removes broad non-required lanes\nfrom routine develop PR CI while preserving the existing lint +\nUbuntu 3.12 fast-test required contexts, this step closes the remaining waste:\nPRs that touch zero code (TODOs, blind-spots, decisions, blog drafts,\nagent-doc edits) still run the surviving 2-3 jobs unnecessarily.\n\nEvidence: PR #66 and PR #67 \u2014 both pure `_project/TODO/**` changes \u2014\ntriggered ~16 jobs each (full matrix + integration + parity + PySpark)\npre-Step-3, and would still trigger ~3 jobs post-Step-3. Both rounds\nof waste are unnecessary; nothing in those PRs could possibly affect\nthe test suite.\n\nApproach: introduce a single always-reporting required check\n(`ci-required-result`) that classifies the PR's diff, runs a cheap\ncontent guard for content-only PRs, and dispatches the Step-3 code\nPR-tier (`lint`/type + Ubuntu 3.12 fast tests) only when code paths\nchanged. For TODO/blog/agent-doc-only PRs, the required check should\nreport green in <60 seconds without running Python fast tests.\n\nBranch protection contract: `ci-required-result` becomes the required\nstatus check on develop. Any jobs whose failures should block merge\nmust be in the same workflow dependency graph as the result job, because\nGitHub Actions `needs:` cannot aggregate jobs from separate workflows.\nThe current `lint.yml` workflow therefore moves its develop PR gate\nunder the umbrella. Leave `lint.yml` for main/push/manual use, or for\nnon-required reporting only.\n\nSingle PR via `make pr-open`. Independently shippable AFTER Step 3\nlands so the PR-tier workflow shape is stable. The starting required-checks\nlist is already established by the live `develop` ruleset; Step 3a is the\nlater step that changes it to the umbrella result check.", "id": "dev-loop-step-3a-path-based-ci-skip", "preserves": ["Branch-protection contract \u2014 required check always reports a status; the umbrella is the new required check, not a removal of all required checks", "Code-touching PRs continue to run full PR-tier CI as today (post-Step-3 narrowed) \u2014 the skip applies only when every changed path is in the safe content-only allowlist", "Content validation \u2014 content-only PRs skip Python fast tests but still run cheap TODO/markdown/docs hygiene relevant to the changed paths", "Existing PR-tier code checks from Step 3 \u2014 Step 3a gates them, it does not remove lint/type or Ubuntu 3.12 fast tests for code/infra/unknown-path PRs", "Fail-closed path behavior \u2014 unknown paths, .gitignore, .gitattributes, workflow/config/tooling changes run full PR-tier unless explicitly and safely reclassified", "Single source of truth for the path ruleset \u2014 `path-filters.yml` is referenced by both CI and local pr-preflight; do not duplicate the rules", "Step 4's develop-post-merge.yml safety net is unchanged \u2014 post-merge auto-revert still runs on every landed SHA, doc-only or not"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/develop-post-merge.yml (Step 4 owns this; post-merge runs on every SHA, doc-only or not)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/nightly.yml (Step 3 owns this; nightly always runs the full matrix)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml (release CI is unchanged)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets \u2014 Step 3a w7 IS the rule edit, but the edit is to the required-checks list only; do not change strict-base, dismiss-stale-reviews, enforce-admins, etc."}, {"kind": "do_not_modify", "path_glob": "Makefile pr-fanout target (Step 1 owns this)"}, {"kind": "do_not_modify", "path_glob": "Makefile worktree-add, worktree-claim, worktree-pool-*, worktree-release targets (Step 2 owns these)"}, {"kind": "do_not_modify", "path_glob": "tests/ (no test changes; only CI orchestration and a tiny local helper)"}, {"kind": "only_modify", "path_glob": ".claude/commands/pr.md (one-shot PR command summary)"}, {"kind": "only_modify", "path_glob": ".github/path-filters.yml (new path-filter ruleset)"}, {"kind": "only_modify", "path_glob": ".github/workflows/lint.yml (remove/disable duplicated develop PR trigger after its required gate moves under the umbrella, or leave as non-required reporting only)"}, {"kind": "only_modify", "path_glob": ".github/workflows/test.yml or .github/workflows/pr.yml (whichever Step 3 ended up with as PR-tier; add umbrella/result/content/code jobs)"}, {"kind": "only_modify", "path_glob": ".pre-commit-config.yaml (pr-preflight-fast-tests hook \u2014 add path-filter shortcut)"}, {"kind": "only_modify", "path_glob": "AGENTS.md (umbrella + local skip paragraph)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (umbrella + local skip paragraph)"}, {"kind": "only_modify", "path_glob": "CONTRIBUTING.md (required-check and preflight prose)"}, {"kind": "only_modify", "path_glob": "Makefile (pr-preflight target \u2014 add path-filter shortcut)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-path-filter-smoke-test-*.md (smoke-test evidence)"}, {"kind": "only_modify", "path_glob": "scripts/path_filter_decision.sh or scripts/path_filter_decision.py (new local helper)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 3a: path-based CI skip for content-only PRs", "verifications": [{"command": "test -f .github/path-filters.yml && grep -cE 'safe[-_]?content|content[-_]?only|needs[-_]?code|code[-_]?ci|unknown' .github/path-filters.yml", "description": "Path filter ruleset exists and has safe-content plus fail-closed code categories", "expected": ">= 1", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE 'ci-paths:|ci-required-result:|content-guard:' .github/workflows/test.yml .github/workflows/pr.yml 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "ci-required umbrella/result jobs and content guard are present in PR-tier workflow", "expected": ">= 3", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -cE \"if:.*needs\\.ci-paths\\.outputs\\.needs-code-ci.*true\" .github/workflows/test.yml .github/workflows/pr.yml 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "Code jobs gate on the fail-closed classifier output", "expected": ">= 2", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -cE 'needs: \\[.*code-lint|code-lint:|ci-required-result:' .github/workflows/test.yml .github/workflows/pr.yml 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "Develop PR lint/type failures are aggregated by the required result check", "expected": ">= 2", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 4}, {"command": "repo=$(gh repo view --json nameWithOwner --jq .nameWithOwner)\nfound=0\nfor ruleset_id in $(gh api \"repos/${repo}/rulesets\" --jq '.[] | select(.target == \"branch\" and ((.conditions.ref_name.include // []) | any(. == \"refs/heads/develop\"))) | .id'); do\n found=1\n gh api \"repos/${repo}/rulesets/${ruleset_id}\" --jq '[.rules[] | select(.type == \"required_status_checks\") | .parameters.required_status_checks[]?.context]'\ndone\ntest \"$found\" = \"1\"\n", "description": "Branch-protection required check is the umbrella, not the subordinate jobs", "expected": "list contains 'ci-required-result' and does NOT contain bare 'lint' or 'test (ubuntu-latest, 3.12)'", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -cE 'TODO.only|docs/|under 60 seconds|unknown.*full|gitignore|code-lint|code-test|content guard' _project/decisions/dev-loop-path-filter-smoke-test-*.md", "description": "Smoke-test decision record proves content-only skip, code fallback, unknown fallback, and gitignore handling", "expected": ">= 6", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 6}, {"command": "make pr-preflight 2>&1 | grep -cE 'No code changes detected|skipping fast tests' || echo 'requires a doc-only diff to demonstrate'", "description": "Local pr-preflight skips only fast tests for content-only diffs", "expected": ">= 1 when run on a doc-only branch; absent on a code-changed branch", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 7}, {"command": "grep -cE 'ci-required-result|path-filters\\.yml|content-only|path.*filter|skipping fast tests' CLAUDE.md AGENTS.md CONTRIBUTING.md .claude/commands/pr.md", "description": "Agent and contributor docs document the new required check and path behavior", "expected": ">= 4", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 8}, {"command": "! grep -R \"dorny/paths-filter@v3\" .github/workflows", "description": "No mutable dorny/paths-filter v3 tag is present", "expected": "no matches", "item_id": "dev-loop-step-3a-path-based-ci-skip", "last_result": null, "last_run": null, "seq": 9}], "work": [{"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": [], "notes": "Decide what is explicitly safe to treat as content-only. The rule is\nfail-closed: a PR can skip code CI only when EVERY changed path is in\nthe safe content-only allowlist. Any path outside that allowlist is a\ncode/infra path and runs the full post-Step-3 PR-tier.\n\nSafe content-only paths (skip Python fast tests; run content guard):\n - `_project/TODO/**`, `_project/DONE/**`\n - `_project/blind-spots/**`\n - `_project/decisions/**`\n - `_project/handoffs/**`\n - `_project/notes/**`, `_project/research/**`, `_project/planning/**`\n - `_blog/**`\n - `docs/**` ONLY if the content guard includes docs validation for\n develop PRs; `docs.yml` currently runs PR checks for `main`, not\n `develop`, so do not rely on it as-is.\n - `*.md` at repo root (README.md, CONTRIBUTING.md, etc.)\n - `CLAUDE.md`, `AGENTS.md`\n\nExamples that MUST run full PR-tier (non-exhaustive; unknown paths\nalso run full PR-tier):\n - `benchbox/**`, `tests/**`, `scripts/**`, `examples/**`\n - `landing/**`, `results-data/**`, `results-explorer/**`, `docker/**`\n - `_binaries/**`, `_sources/**`\n - `_project/scripts/**`, `_project/config/**`, `_project/compat/**`\n - `pyproject.toml`, `uv.lock`, `Makefile`\n - `.github/workflows/**`, `.pre-commit-config.yaml`\n - `.claude/commands/**` (slash-command behavior affects agents)\n - `.gitignore`, `.gitattributes` (repo behavior and tracked-file\n conflict guardrails; `.gitignore` also has gitignore-lint)\n\nEdge case: a single PR that touches BOTH categories runs full\nPR-tier (failsafe). The filter is \"all paths safe \u2192 skip code CI\";\nanything else runs code CI.\n\nDocument the ruleset in `.github/path-filters.yml` (a reusable\nfilter file referenced by the umbrella and local helper) so it has\na single source of truth.\n\nImplementation record (2026-04-30):\n - Added `.github/path-filters.yml` with a `safe-content` allowlist\n covering TODO/DONE records, blind spots, decisions, handoffs, notes,\n research/planning records, blog drafts, docs, root markdown, and\n agent docs.\n - Added `content-guard` patterns matching the safe content surface so\n content-only PRs still run targeted validation before the umbrella\n can report green.\n - Added explicit `code-ci` examples for code, tests, tooling, workflow,\n Makefile, config, lockfile, git behavior, and agent-command paths.\n - Kept unknown paths implicit and fail-closed so a new top-level path\n runs full PR-tier CI instead of silently skipping code checks.\n - W1 review issue fixed: removed a broad `**` code-ci pattern because\n it would make path-filter tooling report code CI for every content\n change and defeat the skip.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define a fail-closed path-filter ruleset", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w1"], "notes": "Edit the workflow that runs on `pull_request` to develop (Step 3\nmay have split test.yml; if so, edit the new pr-tier workflow;\notherwise edit test.yml). Add:\n\n - `ci-paths`: always runs, checks out the PR, uses\n `dorny/paths-filter@` (not `@v3`) or the shared helper\n to emit:\n - `safe-content-only: true|false`\n - `needs-code-ci: true|false`\n - `changed-paths` / optional debug summary\n - optional metric line/artifact: content-only?, skipped code\n CI?, estimated runner minutes saved (for Step 5)\n - `content-guard`: runs when a PR contains safe content paths. This\n is intentionally cheap and targeted: TODO YAML validation,\n markdown/YAML hygiene, docs validation for `docs/**` develop PRs,\n and gitignore-lint if `.gitignore` is ever routed through this\n workflow. It must catch the kind of markdown/YAML errors that\n mattered on PR #66/#67.\n - `code-lint`: the post-Step-3 lint/type gate, gated on\n `needs-code-ci == 'true'`.\n - `code-test`: Ubuntu 3.12 fast tests, gated on\n `needs-code-ci == 'true'`.\n - `ci-required-result`: always runs, aggregates all of the above,\n and is the required check name.\n\nDo not try to aggregate the existing `.github/workflows/lint.yml`\n`lint` job from `ci-required-result`: `needs:` cannot cross workflow\nboundaries. Move/duplicate the required develop PR lint/type gate\ninto the umbrella workflow before making `ci-required-result` the\nsole required check. Leave `lint.yml` for main/push/manual surfaces\nor non-required reporting only.\n\nRequired behavior:\n 1. `ci-paths` and `ci-required-result` always run (no `if:`\n condition on those jobs).\n 2. Content-only PRs run `content-guard`, skip `code-lint` and\n `code-test`, and pass only if `content-guard` succeeds.\n 3. Code/infra/unknown-path PRs run `content-guard` as applicable\n plus `code-lint` and `code-test`.\n 4. If `needs-code-ci == true`, `ci-required-result` fails unless\n both `code-lint` and `code-test` succeeded.\n 5. If `needs-code-ci == false`, `ci-required-result` fails unless\n `content-guard` succeeded or no content guard was needed.\n\nConcrete shape (sketch):\n\n jobs:\n ci-paths:\n runs-on: ubuntu-latest\n outputs:\n needs-code-ci: ${{ steps.classify.outputs.needs-code-ci }}\n safe-content-only: ${{ steps.classify.outputs.safe-content-only }}\n steps:\n - uses: actions/checkout@v4\n - uses: dorny/paths-filter@\n id: filter\n with:\n filters: .github/path-filters.yml\n - id: classify\n run: scripts/path_filter_decision.sh\n\n content-guard:\n needs: ci-paths\n if: needs.ci-paths.outputs.safe-content-only == 'true' ||\n needs.ci-paths.outputs.needs-code-ci == 'true'\n ...\n\n code-lint:\n needs: ci-paths\n if: needs.ci-paths.outputs.needs-code-ci == 'true'\n ...\n\n code-test:\n needs: ci-paths\n if: needs.ci-paths.outputs.needs-code-ci == 'true'\n ...\n\n ci-required-result:\n needs: [ci-paths, content-guard, code-lint, code-test]\n if: always()\n runs-on: ubuntu-latest\n steps:\n - run: |\n set -euo pipefail\n if [ \"${{ needs.ci-paths.outputs.needs-code-ci }}\" = \"false\" ]; then\n if ! { test \"${{ needs.content-guard.result }}\" = \"success\" ||\n test \"${{ needs.content-guard.result }}\" = \"skipped\"; }; then\n echo \"content-guard=${{ needs.content-guard.result }}\"\n exit 1\n fi\n echo \"Content-only PR \u2014 skipped Python code CI.\"\n exit 0\n fi\n if [ \"${{ needs.code-lint.result }}\" = \"success\" ] && \\\n [ \"${{ needs.code-test.result }}\" = \"success\" ]; then\n exit 0\n fi\n echo \"code-lint=${{ needs.code-lint.result }} code-test=${{ needs.code-test.result }}\"\n exit 1\n\nThe required check on develop branch protection becomes\n`ci-required-result`, NOT bare `lint` or `test` directly, only after\nthe workflow has landed on develop.\n\nImplementation record (2026-04-30):\n - Added `.github/workflows/pr.yml` for `pull_request` events targeting\n `develop`; `.github/workflows/test.yml` and\n `.github/workflows/lint.yml` now keep main/push surfaces while this\n workflow owns the develop PR gate.\n - Added always-reporting `ci-paths` and `ci-required-result` jobs.\n `ci-paths` writes a JSON decision artifact and exposes\n `safe-content-only`, `needs-code-ci`, `content-guard-needed`,\n changed paths, code paths, and unknown paths as job outputs.\n - Added `content-guard` for content paths. It validates changed YAML\n with `check-yaml`, markdown with the existing markdownlint hook,\n TODO/DONE YAML with `validate_todo.py` plus `todo_cli.py check-graph`,\n and `docs/**` changes with `make docs-validate`.\n - Added gated `code-lint` and `code-test` jobs. Their displayed check\n names remain `lint` and `test (ubuntu-latest, 3.12)` so the current\n pre-W7 ruleset still receives the required contexts until the\n post-merge ruleset edit switches to `ci-required-result`.\n - Added `scripts/path_filter_decision.py`, a stdlib-only classifier\n that reads `.github/path-filters.yml`, fails closed on unknown\n paths, writes GitHub outputs, emits helper path lists, and supports\n local/CI callers without duplicating rules.\n - W2 review issue fixed: added explicit `git fetch` steps for the base\n branch before diff classification so Actions runners do not depend\n on checkout-specific remote-ref behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add ci-required umbrella, content guard, and gated code jobs", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w1"], "notes": "Step 1 made pre-push hooks opt-in via BENCHBOX_PREPUSH=1. When\nhumans/agents DO opt in (e.g. running `make pr-preflight`), the\nfast-test lane still runs even on content-only changes \u2014 same waste,\njust locally instead of in CI.\n\nAdd a check at the top of `pr-preflight` and the\npr-preflight-fast-tests pre-commit hook that runs the same path\ndecision against `git diff --name-only origin/develop...HEAD`. If\nno code paths matched, run the cheap content guard, print\n\"No code changes detected; skipping fast tests.\", and skip only the\nPython fast-test lane.\n\nReuse the path-filter ruleset from `.github/path-filters.yml`\nvia a small helper (`scripts/path_filter_decision.sh` or Python if\nYAML parsing is needed) so the rule lives in exactly one place.\n\nImplementation record (2026-04-30):\n - Updated `make pr-preflight` so lint/type checks still run first,\n then the Python fast-test lane delegates to `make\n pr-preflight-fast-tests`.\n - Added `pr-preflight-fast-tests`, which classifies\n `origin/develop...HEAD` with `scripts/path_filter_decision.py`.\n Code/infra/unknown-path diffs still run `uv run -- python -m pytest\n -m fast -q`; content-only diffs run the cheap content guard and\n print `No code changes detected; skipping fast tests.`\n - Added `pr-content-guard` so local preflight mirrors CI content\n validation for YAML, markdown, TODO/DONE graph, and docs reference\n checks.\n - Updated the opt-in `pr-preflight-fast-tests` pre-push hook to call\n the same Make target when `BENCHBOX_PREPUSH=1`, avoiding a separate\n hook-only path decision.\n - Extended `scripts/path_filter_decision.py` with `--json-in` and\n `--check` so Makefile logic can consume the same classifier output\n without duplicating JSON parsing in shell.\n - W3 review found no actionable issues after checking quoting,\n content-only guard behavior, and code-path fallback behavior.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add the equivalent local skip in pr-preflight", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w2"], "notes": "In a fork or via `act` locally, simulate two PR diffs:\n 1. Touch only `_project/TODO/foo.yaml` \u2014 confirm\n `ci-required-result` reports green in under 60 seconds, the\n content guard runs, and code-lint/code-test do not run.\n 2. Touch `docs/development/foo.md` \u2014 confirm docs/content\n validation runs for develop PRs without Python fast tests.\n 3. Touch `benchbox/cli/run.py` \u2014 confirm full PR-tier runs as\n today and umbrella aggregates correctly.\n 4. Touch an unknown top-level path (e.g., `quality/example.txt`)\n \u2014 confirm the classifier fails closed and runs full PR-tier.\n 5. Touch `.gitignore` \u2014 confirm full PR-tier or gitignore-lint is\n included in the blocking aggregation.\nDocument the test in `_project/decisions/dev-loop-path-filter-\nsmoke-test-.md` with workflow run URLs and timings.\n\nImplementation record (2026-04-30):\n - Checked for `act`; it is not installed in this environment, so a\n local classifier/content-guard smoke run was used before the workflow\n PR exists on GitHub.\n - Added `_project/decisions/dev-loop-path-filter-smoke-test-2026-04-30.md`\n with the five required cases: TODO-only, `docs/**`, code,\n unknown top-level path, and `.gitignore`.\n - Confirmed TODO-only and docs paths classify as safe content-only,\n run content validation, and skip Python fast tests.\n - Confirmed code, unknown, and `.gitignore` paths classify as\n `needs_code_ci=true`, routing through `code-lint` and `code-test`.\n - W4 review caveat recorded: GitHub workflow run URLs are not available\n until the workflow PR is opened; W6 must use the branch PR run as the\n first hosted workflow evidence.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a tiny test that proves the skip path works", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w2", "w3"], "notes": "Update every place that currently hard-codes the old\n`lint` + `test (ubuntu-latest, 3.12)` contract:\n - `CLAUDE.md`\n - `AGENTS.md`\n - `CONTRIBUTING.md`\n - `.claude/commands/pr.md`\n\nRequired prose:\n - Develop PRs report through `ci-required-result`.\n - Content-only PRs run content validation and skip Python fast\n tests.\n - Code/infra/unknown-path PRs run the post-Step-3 lint/type +\n Ubuntu 3.12 fast-test gate.\n - The shared path ruleset is `.github/path-filters.yml`.\n - Local `make pr-preflight` uses the same classifier.\n\nImplementation record (2026-04-30):\n - Updated `CLAUDE.md`, `AGENTS.md`, `CONTRIBUTING.md`, and\n `.claude/commands/pr.md` to describe `ci-required-result` as the\n develop PR umbrella.\n - Documented `.github/path-filters.yml` as the shared path ruleset.\n - Documented that content-only PRs run content validation and skip\n Python fast tests, while code/infra/workflow/tooling/unknown paths run\n the post-Step-3 lint/type + Ubuntu 3.12 fast-test gate.\n - Documented that local `make pr-preflight` uses the same classifier\n and skips only the Python fast-test lane on content-only branches.\n - W5 review found no stale docs that still describe the develop\n required gate as bare `lint` + `test (ubuntu-latest, 3.12)`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the umbrella in agent and contributor docs", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "`make lint format typecheck`; `BENCHBOX_PREPUSH=1 make pr-preflight`;\n`make pr-open`. The PR description must state that the branch\nprotection/ruleset edit happens AFTER this PR merges, because the\nnew required check name does not exist on develop until then.\n\nInclude smoke-test fork run URLs and confirmation that PR\n#66/#67-style TODO-only PRs would now skip Python code CI while\nstill running content validation.\n\nImplementation record (2026-04-30):\n - Rebased the branch onto `origin/develop` after PR #78 landed.\n Resolved the Makefile conflict by preserving upstream worktree-pool\n targets and Step 3a's `pr-preflight-fast-tests` / `pr-content-guard`\n targets in the same phony lists.\n - Ran `make lint format typecheck`; ruff passed, format left files\n unchanged, and `ty check` exited 0 while reporting the repo's existing\n 426 diagnostics.\n - Ran `BENCHBOX_PREPUSH=1 make pr-preflight`; CI lint checks passed,\n timing policy reported 0 violations, dependency audit reported 0\n violations, and the path-aware fast-test lane ran because this branch\n touches code/workflow paths.\n - Fast tests reported `21268 passed, 6 skipped, 41 warnings, 4 subtests\n passed in 63.30s`.\n - Final W6 review confirmed the branch classifies as\n `needs_code_ci=true`, develop PR checks moved out of `lint.yml` and\n `test.yml`, and `.github/workflows/pr.yml` supplies `lint`,\n `test (ubuntu-latest, 3.12)`, and `ci-required-result`.\n - Hosted PR run #25186848736 exposed one W6 issue:\n `content-guard` ran `pre-commit` without first syncing dev\n dependencies. Fixed by adding `uv sync --group dev` to the\n `content-guard` job before YAML/markdown validation.\n - Hosted push runs exposed a second W6 issue: a reusable rules file\n under `.github/workflows/` is treated by GitHub as a workflow file.\n Moved the shared ruleset to `.github/path-filters.yml` and updated\n the classifier/docs to avoid an invalid workflow named after the\n rules file.\n - Hosted PR run #25187095898 exposed a third W6 issue: the project\n environment did not include the `pre-commit` executable, so\n `uv run -- pre-commit` resolved only on machines with a global\n install. Added `pre-commit` to the dev dependency group and refreshed\n `uv.lock` so CI and local content guard runs use the locked CLI;\n also documented/allowlisted it as a CLI-only dev dependency.\n - W6 follow-up for PR creation: after this commit, run `make pr-open`\n and set the PR body to state that the required-check ruleset edit is\n W7 and happens only after this workflow PR lands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Open and merge the workflow PR before touching branch protection", "wid": "w6"}, {"evidence": null, "item_id": "dev-loop-step-3a-path-based-ci-skip", "needs": ["w6"], "notes": "Apply via `gh api -X PATCH repos/joeharris76/BenchBox/branches/\ndevelop/protection` (or the Step-0-found ruleset). Required\nchecks before this step (post-Step-3): `lint`, `test (ubuntu-latest, 3.12)`.\nRequired checks after this step: `ci-required-result`.\n\nThe workflow changes from w6 MUST already be on develop. Do not make\nthis branch-protection edit a prerequisite for opening or merging\nthe workflow PR.\n\nRecord the before/after JSON in `_project/decisions/\ndev-loop-path-filter-smoke-test-.md` or a follow-up decision\nrecord for audit. Do not change strict-base, dismiss-stale-reviews,\nenforce-admins, or review settings in this step.\n\nImplementation record (2026-04-30):\n - Confirmed PR #79 merged to `develop` at `2026-04-30T20:29:55Z`\n as squash commit `9c82b61f5be83cf018af216a7cf99866b7a4da04`.\n - Captured ruleset `develop-squash-only` (`15611785`) before the edit:\n required checks were `lint` and `test (ubuntu-latest, 3.12)`;\n `strict_required_status_checks_policy` was already `false`.\n - Updated the repository ruleset with a full `PUT` body derived from\n the live ruleset JSON, changing only the required status check list\n to `ci-required-result`.\n - Re-read the live ruleset after the update and confirmed required\n checks now contain only `ci-required-result`; squash-only review\n settings, strict-base policy, linear history, non-fast-forward, and\n deletion rules were preserved.\n - Recorded the before/after JSON in\n `_project/decisions/dev-loop-path-filter-smoke-test-2026-04-30.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Post-merge: update branch-protection required checks", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "auto-merge the revert PR", "instead": "open it but require manual approval", "why": "a human should decide whether to revert vs fix-forward"}, {"dont": "cancel-in-progress on develop-post-merge", "instead": "every landed SHA must be verified independently", "why": "fast PR landings would skip safety checks on intermediate SHAs"}, {"dont": "chain to Step 5 work in this PR", "instead": "keep the surface small", "why": "metrics emission is the deliverable here; aggregation/threshold-checking is Step 5"}, {"dont": "page or alert on every red", "instead": "over-paging trains agents to ignore the channel", "why": "the auto-revert PR IS the alert"}, {"dont": "skip the fork smoke test (w6)", "instead": "30 minutes in a fork is cheap insurance", "why": "a buggy auto-revert on production develop would be loud and embarrassing"}], "approach": "Build the workflow changes incrementally:\n 1. cancel-in-progress (w1) is a small, low-risk YAML edit\n 2. Auto-revert (w2) is the substantive change \u2014 use peter-evans/\n create-pull-request@v6 (mature, well-tested) rather than rolling\n a custom revert script\n 3. Metrics (w3) is independent of auto-revert; both can run in\n parallel jobs in the same workflow\n 4. Make target (w5) is the human-facing surface for w3\n 5. Docs (w4) once the behavior is confirmed\n 6. Smoke test in fork (w6) BEFORE merging \u2014 auto-revert that\n opens spurious PRs would be very noisy to roll back\n\nFor revert conflicts: the fallback to \"open an issue\" is intentional.\nAn auto-revert that conflicts probably means later commits depend on\nthe broken one, so a human needs to decide whether to revert the\nwhole stack or fix forward.\n\nFor metrics: keep it simple. JSON files in workflow artifacts, no\ndatabase, no service. Step 5 reads the last 30 days and aggregates.\nIf we end up wanting longer retention, push to a `metrics` orphan\nbranch instead \u2014 but that's a Step 5 decision, not Step 4.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-4-post-merge-safety-net", "id": 148, "reason": "Out of scope; the auto-revert PR is the notification. Add only if Step 5 shows reverts are missed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Slack/Discord notifications gated on Step 5 showing reverts are being missed; no such evidence indicated, condition unmet.", "summary": "Slack/Discord notifications on develop-red"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-4-post-merge-safety-net", "id": 149, "reason": "Intentional human gate. A human decides revert vs fix-forward. Revisit if revert PRs accumulate unmerged.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Intentional permanent human-gate design decision, not deferred work; revisit only if revert PRs pile up, no evidence of that.", "summary": "Auto-merge of revert PRs after green CI"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-4-post-merge-safety-net", "id": 150, "reason": "JSON artifacts + make dev-loop-metrics is sufficient for Step 5. Build a dashboard only if the metrics become a routine inspection target.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Dashboard/Grafana only warranted 'if metrics become a routine inspection target'; no evidence that threshold was reached.", "summary": "Metrics dashboard / Grafana"}], "deps": ["dev-loop-step-3-ci-rebalance-and-strict-base-off"], "description": "Strengthens `develop-post-merge.yml` so that relaxing strict-base in\nStep 3 has a credible safety net, not just a notification. When\ndevelop goes red post-merge, the workflow automatically opens a revert\nPR for the offending SHA (or an issue if revert conflicts), and emits\nbasic dev-loop metrics that feed Step 5's measurement window.\n\nSequence: this MUST land after Step 3 (relaxed strict-base), because\nthe auto-revert is the mitigation for the increased post-merge risk\nintroduced by Step 3.\n\nSingle PR via `make pr-open`.", "id": "dev-loop-step-4-post-merge-safety-net", "preserves": ["Existing develop-post-merge.yml job definitions and their pass/fail signal \u2014 Step 4 ADDS auto-revert, does not replace existing checks", "GITHUB_TOKEN scope: auto-revert needs `contents: write` and `pull-requests: write` \u2014 confirm permissions block in workflow declares these", "Issue and PR labels are namespaced (incident:*) so they don't conflict with existing labels", "Metrics artifacts are JSON, not human-readable text \u2014 Step 5 aggregator depends on machine-parseable shape"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/nightly.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/release.yml (release CI is unchanged)"}, {"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml (Step 3 owns this)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (Step 3 owns these)"}, {"kind": "do_not_modify", "path_glob": "Makefile pr-open, pr-fanout, pr-refresh targets (Steps 1 and 3 own these)"}, {"kind": "do_not_modify", "path_glob": "Makefile worktree-add, worktree-claim, worktree-pool-*, worktree-release targets (Step 2 owns these)"}, {"kind": "only_modify", "path_glob": ".github/workflows/develop-post-merge.yml"}, {"kind": "only_modify", "path_glob": "AGENTS.md (failure flow and dev-loop metrics sections)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (failure flow, dev-loop metrics, and Pre-approved Commands sections)"}, {"kind": "only_modify", "path_glob": "Makefile (dev-loop-metrics target only)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-auto-revert-fork-test-*.md (new fork validation evidence)"}], "state": "done", "title": "Dev-loop simplification \u2014 Step 4: develop-post-merge auto-revert + dev-loop metrics", "verifications": [{"command": "grep -A 3 'concurrency:' .github/workflows/develop-post-merge.yml | grep -c 'cancel-in-progress: false'", "description": "develop-post-merge.yml uses cancel-in-progress: false", "expected": ">= 1", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE '(auto-revert|revert.*failure)' .github/workflows/develop-post-merge.yml", "description": "Auto-revert job exists in develop-post-merge.yml", "expected": ">= 1", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -cE 'name: metrics' .github/workflows/develop-post-merge.yml", "description": "Metrics job emits an artifact named 'metrics'", "expected": ">= 1", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 3}, {"command": "make dev-loop-metrics 2>&1 | grep -cE 'P50|P95|red rate|runner'", "description": "make dev-loop-metrics produces a non-empty summary", "expected": ">= 4", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 4}, {"command": "grep -cE 'dev-loop-metrics|P50|P95|post-merge red|runner minutes' CLAUDE.md AGENTS.md", "description": "dev-loop-metrics is documented for agents and pre-approved in CLAUDE.md", "expected": ">= 3", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -cE 'incident:develop-red|auto-revert' CLAUDE.md AGENTS.md", "description": "Failure flow is documented in agent-facing docs", "expected": ">= 2", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 6}, {"command": "grep -cE 'Fork validation|failing SHA|revert PR|conflict issue|workflow run' _project/decisions/dev-loop-auto-revert-fork-test-*.md", "description": "PR description includes fork-test evidence", "expected": ">= 4", "item_id": "dev-loop-step-4-post-merge-safety-net", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": [], "notes": "Currently the workflow likely has a `concurrency:` block that\nmay cancel in-progress runs when a new push lands. Change:\n concurrency:\n group: develop-post-merge-${{ github.sha }}\n cancel-in-progress: false\nGroup key on `github.sha` (not `github.ref`) so each landed SHA\ngets its own concurrency lane and runs to completion. This is\nnecessary so fast-landing PRs don't cause the workflow to skip\nchecking intermediate SHAs.\n\nImplementation record (2026-04-30):\n - Updated `.github/workflows/develop-post-merge.yml` so the\n concurrency group is `develop-post-merge-${{ github.sha }}`.\n - Set `cancel-in-progress: false` so a later push to `develop`\n cannot cancel the post-merge safety check for an earlier landed SHA.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Set cancel-in-progress: false on develop-post-merge.yml", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w1"], "notes": "New job `auto-revert-on-failure` triggered by\n`workflow_run.conclusion == 'failure'` (or as a continuation\njob in the same workflow with `if: failure()`). Steps:\n 1. Identify the failing SHA (`github.sha` of the post-merge run)\n 2. Identify the PR that introduced the SHA via\n `gh pr list --state merged --search \"\"`\n 3. Create branch `auto-revert/` from develop\n 4. `git revert --no-edit ` (single SHA, no merge commits)\n 5. If revert succeeds: push branch, open PR with title\n `Revert (#) \u2014 develop went red post-merge`,\n body includes failing run URL + originating PR link, label\n `incident:develop-red`, request review from PR author\n 6. If revert fails (conflict): open issue with same metadata,\n label `incident:develop-red-revert-conflict`, no PR\nUse `peter-evans/create-pull-request@v6` action or a small\ngh-cli script.\n\nImplementation record (2026-04-30):\n - Added `auto-revert-on-failure` to\n `.github/workflows/develop-post-merge.yml` with\n `needs: [lint, fast-test]` and an `always()` guard that runs only\n when one of those post-merge checks fails.\n - Added job-level permissions `contents: write`,\n `pull-requests: write`, and `issues: write`, which are the minimum\n scopes needed to push a revert branch and open either a PR or issue.\n - Used an inline `gh` script so the same job can identify the\n associated squash-merge PR, create idempotent incident labels, open\n an `auto-revert/` PR when `git revert --no-edit` succeeds,\n and request review from the originating PR author when possible.\n - Added conflict fallback: if the revert does not apply cleanly, the\n job aborts the revert and opens an\n `incident:develop-red-revert-conflict` issue with the failing SHA,\n run URL, originating PR URL, and attempted branch name.\n - W2 review issue addressed during implementation: the job uses\n `needs.*.result` rather than job-level `failure()` alone, so skipped\n or successful post-merge jobs do not accidentally trigger a revert.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add auto-revert PR generation on red", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": [], "notes": "Add a `metrics` job that runs with `if: always()` for every push to\ndevelop, so both green and red post-merge runs emit a machine-readable\nrecord. Computes and writes JSON artifact:\n {\n \"merged_at\": \"\",\n \"pr_number\": N,\n \"pr_open_to_merged_seconds\": N,\n \"post_merge_red\": false,\n \"develop_red_detected_at\": null,\n \"auto_revert_pr_opened_at\": null,\n \"time_to_revert_pr_seconds\": null,\n \"next_green_develop_run_at\": null,\n \"time_to_recover_red_seconds\": null,\n \"conflict_on_merge\": ,\n \"orphaned_failed_pr\": false,\n \"ci_runner_minutes\": N\n }\nOn failure, the auto-revert job should update or emit the same shape\nwith `post_merge_red: true`, `develop_red_detected_at`, and\n`auto_revert_pr_opened_at`. `next_green_develop_run_at` and\n`time_to_recover_red_seconds` may be null at artifact creation time;\n`make dev-loop-metrics` must fill them by scanning later successful\ndevelop-post-merge runs. Artifact retention: 90 days (covers the\nStep 5 measurement window). One file per merge. Step 5 aggregates\nthese records plus the explicit PR audit sources named there.\n\nImplementation record (2026-04-30):\n - Added a `metrics` job to `.github/workflows/develop-post-merge.yml`\n with `if: always()` and `needs: [lint, fast-test,\n auto-revert-on-failure]`, so every develop push emits metrics after\n the selected post-merge checks and any auto-revert attempt finish.\n - The metrics job uses read-only `actions`/`contents` permissions,\n queries the associated squash-merge PR through GraphQL, and writes\n `dev-loop-metrics/.json`.\n - The JSON includes the Step 4 shape: merge timestamp, PR number,\n PR-open-to-merge seconds, post-merge red status, red detection time,\n auto-revert PR timestamp when one was opened, recovery placeholders,\n conflict/orphan flags, and summed completed-job runner minutes.\n - Added a 90-day artifact upload named `metrics` for Step 5's\n measurement window.\n - W3 review issue addressed during implementation: the metrics job\n has only read permissions, and the GitHub jobs query requests one\n bounded page (`per_page=100`) so the captured payload is valid JSON.\n - W3 review timing issue fixed: `develop_red_detected_at` is derived\n from the failed `lint`/`fast-test` job completion timestamp instead\n of the later metrics-job start time, so `time_to_revert_pr_seconds`\n cannot go negative when auto-revert finishes before metrics runs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit per-PR dev-loop metrics as workflow artifact", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w2"], "notes": "Add a section \"If develop goes red\":\n - The post-merge workflow detects the failure within ~5 min\n - It auto-opens a revert PR (label: incident:develop-red) OR\n an issue if revert conflicts\n - Original PR author gets review-requested on the revert\n - Agents do NOT need to monitor CI proactively\n - To repair: `make worktree-claim BRANCH=fix/`,\n fix, resubmit\nReplace any older \"monitor your PR until green\" guidance.\n\nImplementation record (2026-04-30):\n - Updated `CLAUDE.md` with an \"If develop goes red\" flow: the\n post-merge workflow opens an `incident:develop-red` revert PR, or an\n `incident:develop-red-revert-conflict` issue when the revert\n conflicts.\n - Updated `AGENTS.md` with the same operating guidance and removed\n the placeholder language that Step 4 would expand the safety net\n later.\n - Documented that agents do not need to monitor CI proactively after\n auto-merge; the generated revert PR or conflict issue is the alert.\n - Documented the repair path:\n `make worktree-claim BRANCH=fix/`, fix, preflight,\n and resubmit with `make pr-open`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the new failure flow in CLAUDE.md and AGENTS.md", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w3"], "notes": "`make dev-loop-metrics` downloads the last N (default 30) days\nof metrics artifacts and prints a summary:\n - P50/P95 PR-to-merged time\n - Post-merge red rate\n - Conflict rate\n - Runner minutes total\nUsed by Step 5 reviewer (manual or scripted) to check thresholds.\nImplementation: `gh run list --workflow develop-post-merge.yml\n--json conclusion,databaseId | gh run download -n metrics`,\nthen aggregate the JSON files.\nDocument this target in CLAUDE.md and AGENTS.md near the post-merge\nsafety-net / dev-loop workflow text. In CLAUDE.md, add it to the\npre-approved read-only PR/Worktree or metrics bucket; the command\nreads workflow artifacts through `gh` and does not mutate repo state.\n\nImplementation record (2026-04-30):\n - Added `make dev-loop-metrics` with `DEV_LOOP_METRICS_DAYS` (default\n 30) and `DEV_LOOP_METRICS_LIMIT` (default 100). The target lists\n recent `develop-post-merge.yml` push runs on `develop`, downloads\n available `metrics` artifacts, and aggregates their JSON files.\n - The summary prints metrics artifact count, P50/P95 PR-to-merged\n seconds, post-merge red rate, conflict rate, and total runner\n minutes. If no artifacts exist yet, it still prints a non-empty\n `n/a` summary instead of failing.\n - Documented the target in `CLAUDE.md` and `AGENTS.md` near the\n post-merge safety-net flow.\n - Added `make dev-loop-metrics` to the CLAUDE.md pre-approved\n read-only PR/Worktree command bucket because it only reads workflow\n artifacts through `gh`.\n - W5 review issues addressed during implementation: the target handles\n an empty metrics-artifact window without failing, and P50/P95 use\n nearest-rank percentiles instead of lower-rank indexing.\n - Final Step 4 verification issue addressed: the summary prints\n `runner minutes total` with lowercase `runner` so the TODO\n verification command passes as written.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make dev-loop-metrics target for ad-hoc inspection", "wid": "w5"}, {"evidence": null, "item_id": "dev-loop-step-4-post-merge-safety-net", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Before merging this PR, validate the auto-revert path end-to-end\nin a fork: cherry-pick a commit that fails the post-merge tests,\npush to a fork's develop, confirm the workflow opens a revert PR\nwith correct title/body/labels. If revert conflicts, confirm the\nfallback issue is opened correctly.\nDocument the test in PR description AND write\n`_project/decisions/dev-loop-auto-revert-fork-test-.md` with\nfork repo, failing SHA, workflow run URL, revert PR URL or conflict\nissue URL, and screenshots/CLI excerpts sufficient for review.\n\nImplementation record (2026-04-30):\n - A true same-account fork was impossible: GitHub rejected\n `joeharris76/BenchBox` -> `joeharris76/BenchBox-dev-loop-smoke-*`\n because one user account cannot own both a parent and a fork. No\n organization target was available, so W6 used public disposable\n mirror `joeharris76/benchbox-dev-loop-smoke-20260430`.\n - Configured the mirror and canonical repository workflow setting with\n `default_workflow_permissions=read` and\n `can_approve_pull_request_reviews=true`, which is required for\n `GITHUB_TOKEN` to create pull requests when the job requests\n `pull-requests: write`.\n - First smoke SHA `35dd478645e5ed34b5b21d33e625bab8e88ddef5`\n exposed a workflow-file edge case: GitHub rejected the revert branch\n push because the reverted commit changed `.github/workflows/*` and\n `GITHUB_TOKEN` lacks `workflows` permission.\n - W6 review issue addressed in `.github/workflows/develop-post-merge.yml`:\n push failures and `gh pr create` failures now open a manual-action\n issue labeled `incident:develop-red-revert-conflict` instead of\n failing without an incident artifact.\n - Final smoke SHA `cff5c591941c989cd0c3003d552142d93489e238`\n validated the normal path: run\n `https://github.com/joeharris76/benchbox-dev-loop-smoke-20260430/actions/runs/25189439117`\n opened revert PR\n `https://github.com/joeharris76/benchbox-dev-loop-smoke-20260430/pull/1`\n from `auto-revert/cff5c591941c` to `develop` with label\n `incident:develop-red`.\n - Evidence written to\n `_project/decisions/dev-loop-auto-revert-fork-test-2026-04-30.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run smoke test against a known-bad SHA in a fork", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "begin Step 6 planning during the measurement window", "instead": "wait for the data", "why": "that primes the conclusion"}, {"dont": "redefine thresholds during evaluation", "instead": "if thresholds were wrong, that's a separate REASSESS conclusion, not a silent edit", "why": "that's confirmation bias dressed as analysis"}, {"dont": "shorten the measurement window because it feels long", "instead": "wait the full window", "why": "30 days captures within-month variability (releases, vacations, etc.) that shorter windows miss"}, {"dont": "skip writing the decision record", "instead": "a verbal 'we measured and it was fine' loses information", "why": "future re-evaluation needs to know what was decided and why"}], "approach": "This is a deliberately low-effort TODO. The work is \"wait, then look\nat numbers and write down the answer.\" The goal is a forcing\nfunction: without this gate, Step 6 might get built reflexively\n(\"we said we would\") rather than because it's needed.\n\nBe honest about what the metrics show. If the numbers are clearly\nbelow threshold (e.g., P95 PR-to-merged is 8 minutes), DO NOT BUILD\nis the correct call. If they're clearly above, BUILD. If they're\nambiguous, REASSESS \u2014 don't force a binary decision on noisy data.\n\nFor \"sustained\" interpretation: use 70% of days above threshold as\nthe rule. One bad week shouldn't trigger a queue; sustained pain\nshould. Adjust if 70% turns out wrong empirically.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-5-measurement-window", "id": 18, "reason": "That's Step 6, gated on this step's recommendation. Keeping the decision/build separation prevents 'might as well build it while I'm here.'", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Explicitly named as Step 6, which is already an open tracked item gated on Step 5's recommendation. (tracked by dev-loop-step-6-queue-decision-gate)", "summary": "Building infrastructure based on measurement findings"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-5-measurement-window", "id": 19, "reason": "If measurement reveals we want continuous tracking, that's a follow-up. For Step 5, the 90-day artifact retention from Step 4 is sufficient.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Speculative follow-up gated on 30-day measurement findings; parent Step 5 item is still open/active, so the condition hasn't been evaluated. (tracked by dev-loop-step-5-measurement-window)", "summary": "Long-term metrics retention beyond 90-day artifact window"}], "deps": ["dev-loop-step-1-local-loop-cleanup", "dev-loop-step-2-worktree-pool", "dev-loop-step-3-ci-rebalance-and-strict-base-off", "dev-loop-step-3a-path-based-ci-skip", "dev-loop-step-4-post-merge-safety-net"], "description": "Decision-gate item that decides whether Step 6 (queue) is needed.\nRuns after Steps 1-4 plus Step 3a have been merged for ~30 days.\nAggregates the metrics emitted by Step 4 against pre-defined thresholds\nand produces a written go/no-go recommendation.\n\nNo code changes during the window itself; the deliverable is a\ndecision record. If all thresholds are below trigger, Step 6 is\ncancelled and the simplification effort is closed. If one or more\ntrigger, Step 6 is unblocked.\n\nThis TODO is intentionally LOW-effort but HIGH-leverage \u2014 it prevents\nbuilding queue infrastructure for a problem that doesn't exist at\nscale.\n", "id": "dev-loop-step-5-measurement-window", "preserves": ["Existing dev-loop metrics emission \u2014 if Step 4 metrics break mid-window, fix forward; do not roll back", "Steps 1-4 implementation untouched during measurement \u2014 no code changes during the window", "Threshold definitions from this TODO \u2014 if measurement is ambiguous, use REASSESS rather than redefining thresholds post-hoc"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml, pr.yml, develop-post-merge.yml, nightly.yml, release.yml (no code changes during the measurement window)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (no further changes during the window)"}, {"kind": "do_not_modify", "path_glob": "Makefile (no target changes during the window)"}, {"kind": "do_not_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-{0,1,2,3,3a,4}-*.yaml (Steps 1-4 plus Step 3a implementations are frozen during measurement)"}, {"kind": "do_not_modify", "path_glob": "tests/conftest.py (Step 1 helper stays as-is)"}, {"kind": "only_modify", "path_glob": "AGENTS.md (one paragraph)"}, {"kind": "only_modify", "path_glob": "CLAUDE.md (one paragraph)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml (status/deps only, or move to DONE if DO NOT BUILD)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-measurement-*.md (new)"}], "state": "active", "title": "Dev-loop simplification \u2014 Step 5: 30-day measurement window with queue decision thresholds", "verifications": [{"command": "ls _project/decisions/dev-loop-measurement-*.md && grep -cE '^### (P95|Post-merge|Time-to-recover|Cross-PR|Orphaned|Runner)' _project/decisions/dev-loop-measurement-*.md", "description": "Measurement record exists with all six metrics evaluated", "expected": ">= 6", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE '^Recommendation: (BUILD QUEUE|DO NOT BUILD QUEUE|REASSESS)' _project/decisions/dev-loop-measurement-*.md", "description": "Recommendation section is present and unambiguous", "expected": "1", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -E '^status:' _project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml", "description": "Step 6 TODO status updated to reflect decision", "expected": "Either 'Not Started' (if BUILD), 'Blocked' (if REASSESS), or moved to DONE (if DO NOT BUILD)", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -cE 'Dev-loop status \\(as of|P95 PR-to-merged|post-merge red rate|BUILD QUEUE|DO NOT BUILD QUEUE|REASSESS' CLAUDE.md AGENTS.md", "description": "Agent docs include current dev-loop status summary", "expected": ">= 3", "item_id": "dev-loop-step-5-measurement-window", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": [], "notes": "The measurement window is 30 days of normal usage AFTER both\nStep 3a and Step 4 have landed. Prior data is contaminated by the\nold workflow. Mark this TODO Blocked until that date.\n\nTo start the timer: when both Step 3a and Step 4 PRs have merged,\nset the window start timestamp to the later of `step3a_merged_at`\nand `step4_merged_at`, comment on this TODO with the merge dates,\nand update `metadata.due_date` to start_date + 30 calendar days.\n\nImplementation record (2026-04-30):\n - Step 3a PR #80 merged to `develop` at `2026-04-30T20:36:27Z`.\n - Step 4 PR #81 merged to `develop` at `2026-04-30T21:33:45Z`.\n - Step 4 is the later prerequisite, so the measurement window\n starts at `2026-04-30T21:33:45Z`.\n - Set `metadata.due_date` to `2026-05-30`; the existing date\n remains correct because Step 4 merged after Step 3a.\n - Set this TODO status to `Blocked` and left W1 `in_progress`\n until the due date. No code or workflow changes are allowed\n during the measurement window.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "in_progress", "summary": "Wait for 30 calendar days after Step 3a and Step 4 have both merged before starting w2", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w1"], "notes": "Run `make dev-loop-metrics DEV_LOOP_METRICS_DAYS=30\nDEV_LOOP_METRICS_LIMIT=500`. The target downloads `metrics`\nartifacts from `develop-post-merge.yml` runs in the window and\nprints PR-to-merge P50/P95, post-merge red rate, conflict rate,\nand runner-minutes total. Capture stdout to the measurement\nrecord below.\n\nOutput is plain text only; the target does not emit JSON. To\ncompute additional aggregates beyond the printed lines, download\nthe raw artifacts and aggregate with `jq`:\n\n gh run list --workflow develop-post-merge.yml --branch develop \\\n --event push --created \">=\" --limit 500 \\\n --json databaseId --jq '.[].databaseId' \\\n | xargs -I{} gh run download {} -n metrics -D /tmp/metrics/{}\n jq -s '...' /tmp/metrics/*/*.json\n\nIf artifact coverage is below 80% of merges (i.e., emission\nfailures), extend the window 7 days and retry.\n\nCross-run derivation needed for Step 4 placeholder fields:\n`next_green_develop_run_at` and `time_to_recover_red_seconds` are\nemitted as `null` in each per-SHA metrics file because recovery\ncan only be observed on a *later* SHA's run. W2 must pair each\n`post_merge_red=true` artifact with the next chronologically\nnewer artifact whose `lint` and `fast-test` jobs succeeded, and\nderive `time_to_recover_red_seconds` from the gap between\n`develop_red_detected_at` and the next-green run's `merged_at`.\nStore the derived series in the measurement record alongside the\nraw printout.\n\nMetrics must combine:\n - Step 4 metrics artifacts (PR open->merged, post-merge red,\n time-to-revert-PR, runner minutes; recovery time derived as\n above)\n - PR audit for the same window:\n `gh pr list --state all --base develop --search \"updated:>=\"`\n and labels/comments containing `queue:conflict`,\n `conflict`, `pr-refresh`, `auto-revert`, or\n `incident:develop-red`\n - Post-merge workflow runs:\n `gh run list --workflow develop-post-merge.yml --created\n .. --json conclusion,createdAt,updatedAt,url`\nSave raw aggregation to `_project/decisions/\ndev-loop-measurement-.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Aggregate dev-loop metrics for the 30-day window", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w2"], "notes": "Apply the trigger matrix from the plan:\n | Metric | Trigger threshold |\n | P95 PR open \u2192 merged | sustained > 1h |\n | Post-merge red rate on develop | sustained > 5% |\n | Time-to-recover red develop (P95) | > 30 min |\n | Cross-PR conflict repairs / week | > 2/week |\n | Orphaned failed PRs (no resub <24h) | any pattern |\n | Runner minutes total | budget signal |\nFor each metric: PASS (below trigger) / FAIL (at or above).\n\"Sustained\" means >= 70% of days in the window above threshold;\nnot just one outlier week.\n\nData-source rules:\n - Use Step 4 artifacts for PR open->merged, post-merge red,\n time-to-revert-PR, and runner minutes.\n - Use the cross-run derivation from w2 for time-to-recover red\n develop. The per-SHA `time_to_recover_red_seconds` field is\n emitted as `null` by Step 4 by design; the value is computed\n in w2 by pairing red runs with the next green run.\n - Use the PR audit from w2 for cross-PR conflict repairs/week.\n - Count orphaned failed PRs from PRs labeled/commented as failed,\n conflicted, auto-reverted, or develop-red with no new commit,\n review response, or replacement PR within 24h.\n - If a metric cannot be computed because Step 4 did not emit the\n required fields, mark REASSESS and create a follow-up to fix\n metrics emission; do not silently drop the metric.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Evaluate against queue-trigger thresholds", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w3"], "notes": "Single markdown file appended to the measurement record from w2.\nSections:\n - Window: to \n - Per-metric PASS/FAIL with raw numbers\n - Recommendation: BUILD QUEUE | DO NOT BUILD QUEUE | REASSESS\n - If BUILD: which queue path (native vs steward), rationale\n tied to Step 0 w6 finding\n - If DO NOT BUILD: cancel Step 6 TODO, close the simplification\n effort\n - If REASSESS: extend window 30 days, identify which\n measurements were ambiguous, and what would resolve them\nCommit the record. Update Step 6 TODO status (unblock or close).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Write decision record with go/no-go for Step 6", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-5-measurement-window", "needs": ["w4"], "notes": "Add a one-paragraph \"Dev-loop status (as of )\" note in\nboth files that documents:\n - Whether the queue is being built or not\n - Current P95 PR-to-merged time\n - Current post-merge red rate\nFuture agents read this to understand the state of the merge\nmachinery without re-running the analysis.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update CLAUDE.md / AGENTS.md with measurement findings", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add both Path A and Path B implementation units to this TODO", "instead": "create exactly one selected-path TODO", "why": "the flat work DAG cannot mark the unchosen path skipped"}, {"dont": "add full matrix to queue-tier CI", "instead": "keep queue light, heavy stays nightly", "why": "that's exactly the over-engineering Steps 3-5 prevented"}, {"dont": "begin queue work without Step 5 saying BUILD", "instead": "premature start wastes effort", "why": "Step 5 might say DO NOT BUILD"}, {"dont": "remove develop-post-merge.yml just because the queue runs queue-tier CI", "instead": "both protect different failure modes", "why": "queue-tier and post-merge check different SHAs"}, {"dont": "skip trust lanes", "instead": "`pull_request_target` is forbidden anywhere in steward workflows", "why": "public repos with write-token automation are an exfiltration risk"}, {"dont": "use a long-lived PAT for the steward", "instead": "use App with scoped permissions", "why": "rotation burden and audit trail are worse than a GitHub App"}], "approach": "Lock path selection first, then generate the selected implementation TODO.\nDo not put both implementations into this item's `work[]`; the local TODO\nmodel cannot represent branch choice without a skipped status. If the\ngenerated TODO needs to differ from the locked spec, cite the Step 5\nmeasurement or Step 0 availability evidence that forced the change.\n\nPath A is significantly simpler if available. Path B is a real software\nproject (workflow + bot credentials + state machine) and needs fork smoke\ntesting before enabling on develop.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-6-queue-decision-gate", "id": 20, "reason": "Only one selected-path TODO should be created. The rejected path is documented in the decision record to avoid a deadlocked flat DAG.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: This is the explicitly rejected/unselected queue path; by design only the selected Step 6 path (already open) is tracked.", "summary": "Unselected queue implementation path"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-6-queue-decision-gate", "id": 21, "reason": "Org transfer is a separate decision with broader implications (URLs, integrations, billing). If Step 5 says BUILD and Path A requires transfer, escalate that decision separately.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Native MQ/org-transfer escalation is a conditional sub-note of the still-open, active parent queue-decision-gate item. (tracked by dev-loop-step-6-queue-decision-gate)", "summary": "Native MQ migration via org transfer"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "dev-loop-step-6-queue-decision-gate", "id": 22, "reason": "Labels + comments are sufficient for one-person ops. Add dashboard if queue volume justifies it.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: This deferral's own from_item IS the open tracked item dev-loop-step-6-queue-decision-gate; not a separate untracked task.", "summary": "Queue dashboard / observability beyond labels"}], "deps": ["dev-loop-step-5-measurement-window"], "description": "Deferred queue gate and spec handoff. This TODO does NOT implement a\nqueue directly. If Step 5 measurement returns BUILD QUEUE, this item\nchooses the viable path and creates exactly one follow-up implementation\nTODO from the locked spec below:\n\n - Path A: native GitHub Merge Queue\n - Path B: BenchBox Merge Steward\n\nThis shape is intentional. The TODO work-unit graph only supports\n`pending`, `in_progress`, and `done`, so mutually exclusive Path A/Path B\nimplementation units cannot live in the same flat DAG without deadlocking\ndownstream work. The generated follow-up TODO becomes the executable\nimplementation item; this gate records the path and prevents re-debating\nsettled design decisions.\n\nStatus starts as Blocked (on Step 5). Three resolution paths:\n - Step 5 -> DO NOT BUILD: this TODO moves to DONE with completion note\n \"queue not needed; closed by Step 5 measurement \"\n - Step 5 -> BUILD QUEUE: this TODO unblocks, status -> Not Started,\n work units below create the selected implementation TODO\n - Step 5 -> REASSESS: stays Blocked, deps point to next measurement\n", "id": "dev-loop-step-6-queue-decision-gate", "preserves": ["Existing pr-open semantics during deprecation period \u2014 do not break in-flight workflows mid-rollout", "Heavy validation stays in nightly \u2014 do not move it back to the queue path", "Only one path implementation TODO exists \u2014 no flat-DAG deadlock from mutually exclusive Path A/Path B work", "Step 3's lightweight PR-tier CI \u2014 the queue does not promote develop to release-grade gating", "develop-post-merge.yml safety net from Step 4 remains active even with a queue"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/test.yml, develop-post-merge.yml, nightly.yml, release.yml (implementation TODO territory)"}, {"kind": "do_not_modify", "path_glob": "CLAUDE.md, AGENTS.md, CONTRIBUTING.md (implementation TODO will own queue-flow docs and trust lanes)"}, {"kind": "do_not_modify", "path_glob": "GitHub branch protection / rulesets (implementation TODO toggles merge-queue rule)"}, {"kind": "do_not_modify", "path_glob": "Makefile (implementation TODO will own pr-submit and queue handoff)"}, {"kind": "do_not_modify", "path_glob": "tests/, .pre-commit-config.yaml (orthogonal to queue path selection)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6-queue-decision-gate.yaml (status/completion updates)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6a-native-merge-queue.yaml (new, Path A only)"}, {"kind": "only_modify", "path_glob": "_project/TODO/main/planning/dev-loop-step-6b-merge-steward.yaml (new, Path B only)"}, {"kind": "only_modify", "path_glob": "_project/decisions/dev-loop-queue-path-*.md (new decision record)"}], "state": "active", "title": "Dev-loop simplification \u2014 Step 6: queue implementation gate \u2014 DEFERRED until Step 5 justifies", "verifications": [{"command": "grep -cE '^Selected path: (A|B) \\(' _project/decisions/dev-loop-queue-path-*.md", "description": "Path A or Path B chosen with rationale recorded", "expected": "1", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 1}, {"command": "test \"$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml 2>/dev/null | wc -l | tr -d ' ')\" = \"1\" && echo ok", "description": "Exactly one selected-path implementation TODO exists", "expected": "ok", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 2}, {"command": "f=$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml); uv run --project _project/scripts -- python _project/scripts/validate_todo.py \"$f\" --strict", "description": "Generated implementation TODO validates strictly", "expected": "valid", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 3}, {"command": "f=$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml); ! grep -qE 'ONLY if|mark this .* skipped|needs: \\[w2, w3\\]' \"$f\" && echo ok", "description": "Generated implementation TODO has no mutually exclusive DAG branches", "expected": "ok", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 4}, {"command": "f=$(ls _project/TODO/main/planning/dev-loop-step-6[ab]-*.yaml); grep -cE 'queue-tier|pr-submit|queue:approved|queue:abandoned|pull_request_target|develop-post-merge|CLAUDE.md|AGENTS.md|CONTRIBUTING.md|Pre-approved' \"$f\"", "description": "Common queue policy is present in generated TODO", "expected": ">= 10", "item_id": "dev-loop-step-6-queue-decision-gate", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": [], "notes": "Re-read Step 0 w6 finding (native MQ availability) and the Step 5\ndecision record. Path A (native MQ) requires repo/ruleset support for\nGitHub Merge Queue. Path B (steward) is the fallback.\n\nIf Step 0 confirmed user-owned repo and GitHub APIs still do not\nexpose merge-queue rules for this repo, Path B is mandatory unless\nthe user has since transferred the repo to an org. If org transfer\nis on the table, evaluate it as a separate decision (one-time\nfriction vs ongoing steward maintenance), not inside implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Confirm BUILD decision and choose native MQ vs steward", "wid": "w1"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w1"], "notes": "Create `_project/decisions/dev-loop-queue-path-.md` with:\n - Selected path: A or B\n - Step 0 evidence used\n - Step 5 BUILD QUEUE metrics that justified starting\n - Why the unselected path was rejected\n - Whether org transfer was considered and why it is/is not part\n of the queue rollout\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Write queue path decision record", "wid": "w2"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w2"], "notes": "Create ONE of these files, not both:\n - Path A: `_project/TODO/main/planning/dev-loop-step-6a-native-merge-queue.yaml`\n - Path B: `_project/TODO/main/planning/dev-loop-step-6b-merge-steward.yaml`\n\nThe generated TODO must be a normal implementation item with flat\n`work[]` units and no mutually exclusive branches.\n\nPath A locked spec:\n 1. Enable \"Require merge queue\" on develop branch protection or\n the ruleset that targets develop.\n 2. Add `merge_group:` trigger to the workflows that report\n queue-tier checks.\n 3. Keep Step 3's lightweight PR-tier checks; queue-tier is also\n light (lint + Ubuntu 3.12 fast tests), not release-grade.\n 4. Confirm `gh pr merge --auto --squash` remains the submission\n mechanism for develop PRs.\n 5. Update CLAUDE.md/AGENTS.md to describe the queue-managed flow.\n 6. Update CLAUDE.md pre-approved command buckets for any new\n queue-facing commands or labels.\n\nPath B locked spec:\n 1. Implement BenchBox Merge Steward as a single-concurrency\n workflow triggered by `queue:ready`.\n 2. Concurrency group: `benchbox-develop-integrator`,\n `cancel-in-progress: false`.\n 3. Loop oldest-first by `queue:ready` label timestamp.\n 4. For each PR: reset to current origin/develop, apply PR branch\n onto that tip, run queue-tier checks, then merge or eject.\n 5. Conflict: label `queue:conflict`, comment with files, continue.\n 6. Test fail: label `queue:failed`, comment with job, continue.\n 7. Use a GitHub App with `contents: write` and\n `pull_requests: write`; no long-lived PAT.\n 8. Smoke test in a fork with clean, conflicting, failing, and\n untrusted PRs before enabling on develop.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Create exactly one selected-path implementation TODO", "wid": "w3"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w3"], "notes": "The selected implementation TODO must include these common work units\nor guardrails:\n - Queue-tier CI is light: lint + Ubuntu 3.12 fast tests only.\n Heavy validation stays in nightly.\n - Add `make pr-submit` as pr-open + queue handoff + hygiene:\n refuse dirty worktree, capture immutable head SHA, skip drafts,\n and bypass auto-queue for workflow/steward changes.\n - Keep `make pr-open` as a deprecated alias for one release.\n - Update CLAUDE.md and AGENTS.md with the submitted/queued flow,\n including when agents use `make pr-submit` vs deprecated\n `make pr-open`, whether they should wait, and how queue\n ejections are repaired.\n - Update CLAUDE.md Pre-approved Commands so `make pr-submit`\n and any safe queue-status commands are explicitly classified;\n queue admin/reset commands must not be hidden in broad wildcards.\n - Define trusted vs untrusted lanes. Untrusted PRs require manual\n `queue:approved`; `pull_request_target` is forbidden in steward\n workflows.\n - Update CONTRIBUTING.md with the external contributor path:\n untrusted PRs require manual `queue:approved`; PR-controlled\n workflow/steward changes bypass auto-queue.\n - Define bounce ownership and retry budget: original author owns\n `queue:conflict` / `queue:failed`; 3 ejections max, then\n `queue:abandoned`.\n - Keep develop-post-merge.yml active as Step 4's safety net.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add common queue policy requirements to the selected TODO", "wid": "w4"}, {"evidence": null, "item_id": "dev-loop-step-6-queue-decision-gate", "needs": ["w2", "w3", "w4"], "notes": "Run strict validation on the generated TODO and `todo_cli.py\ncheck-graph`. The generated TODO is the implementation item to work\nnext. This gate is complete once:\n - Exactly one implementation TODO exists\n - The decision record exists\n - The generated TODO is valid, graph-safe, and has runnable\n verification commands\n - The unselected path is documented as rejected/deferred\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Validate the generated TODO and close this gate", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "revert alias normalization or make the run platform key the suffixed `-df` name to work around the mismatch (breaks result labeling and `platforms status/enable/install`).", "instead": "(not specified)", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Usability / Config", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-07T00:00:00Z", "deferrals": [], "deps": [], "description": "RESOLVED by merged PR #1054 \"fix(cli): key DataFrame platform options by base\nname so run can reach them\" (branch fix/dataframe-platform-option-keying,\nmerged 2026-07-08) plus review follow-up PR #1062 \"fix: forward DataFrame\nplatform options to adapter construction (#1054 review)\". This TODO fell out\nof the PR #1042 (fix/datafusion-df-mode-erasure) audit; the in-flight session\ntask_ce5bd997 that was implementing it landed as #1054. No re-implementation\nwas needed. See the \"resolution\" block below for the empirical confirmation.\n\n---\n\nAll DataFrame-platform `--platform-option` specs are registered under\nsuffixed keys (`datafusion-df`, `polars-df`, `pandas-df`, `modin-df`,\n`cudf-df`, `dask-df`) in the `_OPTION_SPEC_ROWS` table in\nbenchbox/platforms/__init__.py (~lines 768-786). But `benchbox run` parses\nplatform options with the POST-alias platform key:\nbenchbox/cli/commands/run.py `_parse_plat_bench_options` (:584) calls\n`PlatformHookRegistry.parse_options(s.platform_key, ...)` where\n`s.platform_key` has already been normalized by PLATFORM_ALIASES in\nbenchbox/cli/platform.py (datafusion-df -> datafusion, polars-df -> polars,\netc.). The specs are registered under the suffixed key, so the base-name\nlookup finds nothing.\n\nResult: `benchbox run --platform datafusion-df --platform-option\ntarget_partitions=4` fails with\n\"Unknown platform option 'target_partitions' for platform 'datafusion'\".\nVerified empirically on develop 2026-07-07. Every DataFrame platform option\n(streaming, rechunk, n_rows, dtype_backend, engine, device_id,\ntarget_partitions, repartition_joins, parquet_pushdown, batch_size,\nmemory_limit, temp_dir, n_workers, ...) is unreachable from the run CLI.\n\nSame key mismatch affects the sibling lookups in run.py:\n`PlatformHookRegistry.get_default_options(s.platform_key)` (:645) and\n`list_option_specs(platform_key)` (:658). Note an asymmetry to preserve or\nreconcile: `_describe_platform_options` (:453-455) lowercases the raw name\nWITHOUT alias normalization, so `--describe-platform-options datafusion-df`\nDOES list the options \u2014 while `run --platform datafusion-df` then rejects\nthem. Describe and parse must agree after the fix.\n\nDistinct from qpc-16 (that TODO is about plan_max_depth /\nplan_capture_timeout_seconds in benchbox/core/results/platform_options.py, a\ndifferent allowlist). This one is the cli.platform_hooks PlatformHookRegistry\nused by `run`.\n\nIN FLIGHT: a background session (spawned task task_ce5bd997) is actively\nimplementing this fix as of 2026-07-07. If that lands a PR, complete this\nTODO against it rather than starting fresh work.\n\nContext: PR #1042 (fix/datafusion-df-mode-erasure) added\n`_apply_dataframe_suffix_mode` in run.py so `-df` names resolve DataFrame\nmode while the platform key stays the base name by design. DO NOT undo that;\nthe option-spec fix must keep the base-name platform key and instead make\nthe specs reachable under it.", "id": "df-suffix-platform-options-unreachable", "preserves": ["PR #1042 behavior: `-df` names resolve DataFrame mode and the platform key stays the base name.", "Registered default values for DataFrame options unchanged when the option is not set.", "SQL-platform --platform-option behavior (e.g. databricks, postgresql) unchanged."], "prior_art": [{"concept": "benchbox/cli/commands/run.py _parse_plat_bench_options:584 \u2014 parse_options(s.platform_key) with normalized base name (the consumer that fails)", "decision": "reuse", "path": "benchbox/cli/commands/run.py"}, {"concept": "benchbox/cli/platform.py PLATFORM_ALIASES:55-63 + _apply_dataframe_suffix_mode (PR #1042) \u2014 where `-df` is normalized to base name; the option key must match this", "decision": "reuse", "path": "benchbox/cli/platform.py"}, {"concept": "benchbox/core/hooks/platform_hooks.py PlatformHookRegistry.register_option_specs / parse_options / get_default_options / describe_options \u2014 the registry keying", "decision": "reuse", "path": "benchbox/core/hooks/platform_hooks.py"}, {"concept": "benchbox/platforms/__init__.py _OPTION_SPEC_ROWS:768-786 \u2014 DataFrame option specs keyed by `-df`; EXTEND: re-key to base name or register under both", "decision": "extend", "path": "benchbox/platforms/__init__.py"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/hooks/platform_hooks.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "DataFrame -df platform options are unreachable from `benchbox run`", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/test_run_command_branches.py tests/unit/cli/test_platform.py -q -k 'option or dataframe or df'", "description": "DataFrame -df platform options are accepted by run and reach adapter config", "expected": "all pass, including new datafusion-df / polars-df --platform-option acceptance tests", "item_id": "df-suffix-platform-options-unreachable", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reproduce and pin: assert `run --platform datafusion-df --platform-option target_partitions=4 --dry-run` currently rejects the option; capture the failing message as the regression baseline.", "wid": "w1"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide keying: re-key DataFrame specs in _OPTION_SPEC_ROWS to base names, or register under both keys. Confirm adapter_factory._get_dataframe_adapter config passthrough unaffected.", "wid": "w2"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement chosen keying so parse_options/get_default_options/list_option_specs resolve DataFrame options under the base key used by run.py.", "wid": "w3"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile describe/parse asymmetry: --describe-platform-options and run --platform-option must accept the same names for `-df` and base names.", "wid": "w4"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regression tests: datafusion-df target_partitions parses and reaches adapter config; base and suffixed paths agree; cover a second DataFrame platform (polars-df streaming).", "wid": "w5"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Adversarial `code` review; fix Critical/Required findings; re-verify.", "wid": "w6"}, {"evidence": null, "item_id": "df-suffix-platform-options-unreachable", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Open PR to develop.", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "hardcode date ranges", "instead": "use the parameter override pattern from tpch/dataframe_queries.py", "why": "FlightData uses parameterized {start_date}/{end_date} based on available data"}, {"dont": "implement queries as SQL strings passed to DataFrame engines", "instead": "implement using native DataFrame API (col/lit/join/groupby)", "why": "that defeats the purpose of DataFrame benchmarking"}], "approach": "Follow the read_primitives/dataframe_queries.py pattern: single file with\nexpression_impl and pandas_impl functions for each query, registered in a\nQueryRegistry. Use the get_dataframe_queries() fallback method on the benchmark\nclass (no hard-coded dispatch needed).\n\nReference benchbox/core/flightdata/queries.py for the SQL source - each query's\nSQL maps directly to DataFrame operations. Start with the 5 on-time performance\nqueries (simplest aggregation patterns), then delay analysis, route analytics,\ntemporal patterns, and carrier comparisons.\n\nUse DataFrameQuery categories: JOIN, AGGREGATE, GROUP_BY, FILTER, SORT for\nappropriate classification. Parameterize date ranges via a module-level defaults\ndict with set_parameter_overrides() pattern from tpch/dataframe_queries.py.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-flightdata", "id": 381, "reason": "The get_dataframe_queries() fallback method is sufficient. Hard-coded dispatch only needed for stream_id permutation support (TPC-H/TPC-DS compliance).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason states the existing fallback method is sufficient and hard-coded dispatch is only needed for out-of-scope stream_id permutation support.", "summary": "Add FlightData dataframe queries to hard-coded dispatch in dataframe_runner.py"}], "deps": [], "description": "Add dual-family DataFrame query implementations (expression + pandas) for all 20\nFlightData OLAP queries, enabling execution on Polars, PySpark, DataFusion, Pandas,\nModin, cuDF, and Dask platforms.\n\nFlightData is a 3-table benchmark (flights fact table + airlines/airports dimensions)\nwith 20 queries across 5 categories: on-time performance (5), delay analysis (4),\nroute analytics (4), temporal patterns (4), and carrier comparisons (3). The queries\nuse standard OLAP patterns - GROUP BY with aggregations, LEFT JOINs to dimension\ntables, CASE expressions, HAVING clauses, date filtering, and ROUND/AVG/SUM/COUNT.\n\nThis is one of the simpler benchmarks to port because the schema is small (3 tables),\njoins are straightforward (single-key LEFT JOINs to dimension tables), and query\npatterns are standard aggregation/filter operations that map cleanly to both\nexpression-family and pandas-family APIs.\n\nMotivation: FlightData uses real-world aviation data and is popular for OLAP\ndemos. DataFrame support enables benchmarking DataFrame platforms on realistic\nanalytics workloads beyond TPC-H/TPC-DS.", "id": "df-support-flightdata", "preserves": ["FlightDataBenchmark SQL execution path remains unchanged - all existing SQL queries and parameterized date ranges work identically", "FlightDataDownloader data generation produces files loadable by both SQL and DataFrame paths", "benchbox run --platform duckdb --benchmark flightdata still works (SQL mode unaffected)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/flightdata/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/flightdata/"}], "state": "done", "title": "Add DataFrame execution support to FlightData benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.flightdata.dataframe_queries import get_dataframe_queries; r = get_dataframe_queries(); print(f'{len(r)} queries registered'); assert len(r) == 20\"", "description": "All 20 DataFrame queries are registered in the QueryRegistry", "expected": "20 queries registered", "item_id": "df-support-flightdata", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['flightdata']['supports_dataframe'])\"", "description": "FlightData benchmark reports supports_dataframe=True", "expected": "True", "item_id": "df-support-flightdata", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/flightdata/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-flightdata", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-flightdata", "needs": [], "notes": "Create benchbox/core/flightdata/dataframe_queries.py with expression_impl\nfunctions for all 20 queries. Follow the pattern in read_primitives/dataframe_queries.py.\n\nKey translation patterns:\n- LEFT JOIN airlines/airports -> ctx.get_table(\"flights\").join(ctx.get_table(\"airlines\"), ...)\n- CASE WHEN -> col(\"x\").when(...).then(...).otherwise(...) (Polars) or F.when() (PySpark)\n- ROUND(AVG(...), 2) -> col(\"x\").mean().round(2)\n- date filtering -> col(\"flight_date\") >= lit(start_date)\n- HAVING COUNT(*) >= N -> filter after group_by\n- Integer division for dep_hour -> (col(\"crs_dep_time\") / lit(100)).cast(int)\n\nQueries use parameterized date ranges ({start_date}, {end_date}) - implement\na parameters dict similar to TPCH_DEFAULT_PARAMS in tpch/dataframe_queries.py.\n\nAll 20 queries should be implementable; no SQL-only constructs present.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for all 20 FlightData queries", "wid": "w1"}, {"evidence": null, "item_id": "df-support-flightdata", "needs": ["w1"], "notes": "Add pandas_impl functions to each DataFrameQuery in dataframe_queries.py.\n\nKey translation patterns:\n- LEFT JOIN -> pd.merge(flights, airlines, left_on=\"reporting_airline\", right_on=\"code\", how=\"left\")\n- CASE WHEN -> np.where() or df[\"col\"].apply()\n- GROUP BY + agg -> df.groupby([...], as_index=False).agg(...)\n- HAVING -> filter after groupby\n- ROUND -> .round(2)\n- ORDER BY ... LIMIT N -> .sort_values(...).head(N)\n- Integer division -> df[\"crs_dep_time\"] // 100\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create pandas-family implementations for all 20 FlightData queries", "wid": "w2"}, {"evidence": null, "item_id": "df-support-flightdata", "needs": ["w1"], "notes": "Integration steps:\n1. Add get_dataframe_queries() method to FlightDataBenchmark in benchmark.py\n that imports and returns the QueryRegistry from dataframe_queries.py\n2. Set supports_dataframe: True in benchmark_registry.py for flightdata entry\n3. Ensure FlightData data loader works with DataFrame platforms (CSV/Parquet loading)\n4. Add any skip lists if needed (unlikely - all queries are standard OLAP)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up FlightData benchmark class and registry for DataFrame dispatch", "wid": "w3"}, {"evidence": null, "item_id": "df-support-flightdata", "needs": ["w2", "w3"], "notes": "Create tests/unit/core/flightdata/test_dataframe_queries.py:\n- Test query registration (all 20 queries registered, correct IDs)\n- Test expression_impl functions execute without error on small synthetic data\n- Test pandas_impl functions execute without error on small synthetic data\n- Test result shape/columns match expectations\n- Test parameterized date ranges\n- Mark tests with @pytest.mark.fast where possible\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for FlightData DataFrame queries", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "assume table aliases from SQL map to DataFrame variables", "instead": "use descriptive variable names (e.g., info_type_genre, info_type_rating for the double it join in 12a)", "why": "IMDB schema has self-referential patterns"}, {"dont": "attempt to replicate SQL optimizer stress-testing in DataFrame mode", "instead": "focus on correct multi-join execution and performance measurement", "why": "DataFrame APIs define explicit join order, there is no optimizer to stress"}, {"dont": "use nested loops or iterative joins", "instead": "use chained .join() calls that the DataFrame engine can optimize", "why": "that destroys performance"}], "approach": "Follow the read_primitives/dataframe_queries.py single-file pattern with QueryRegistry.\nUse get_dataframe_queries() fallback method on the benchmark class.\n\nStart with 4-5 table queries (w1) to establish the multi-join translation pattern,\nthen scale to 7-9 table queries. The central translation challenge is converting\nimplicit cross-joins (comma-separated FROM with WHERE conditions) into explicit\n.join() chains. Use the title table as the hub for most queries since most\njunction tables reference title.id.\n\nFor the expression family, the join pattern is:\n title.join(movie_companies, left_on=\"id\", right_on=\"movie_id\")\n .join(company_type, left_on=\"company_type_id\", right_on=\"id\")\n .filter(col(\"kind\") == lit(\"production companies\"))\n .select(col(\"title\").min().alias(\"movie_title\"))\n\nUse QueryCategory.JOIN for all queries, plus AGGREGATE and FILTER as appropriate.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-joinorder", "id": 382, "reason": "Research question, not implementation work. Could inform future variant generation for DataFrame join ordering.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicitly framed as a research question, not implementation work; best captured under the future-ideas backlog if ever pursued. (tracked by future-ideas-backlog)", "summary": "Benchmark join order sensitivity - measure if different .join() sequences change performance"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-joinorder", "id": 383, "reason": "The full JOB has 113 queries; current benchmark embeds 13 representative ones. Extending is separate scope.", "resolution": "promoted", "resolved_item": "add-remaining-job-queries-beyond-the-core-13", "resolved_reason": null, "summary": "Add remaining JOB queries beyond the core 13"}], "deps": [], "description": "Add dual-family DataFrame query implementations (expression + pandas) for all 13\nJoinOrder benchmark queries, enabling execution on Polars, PySpark, DataFusion,\nPandas, Modin, cuDF, and Dask platforms.\n\nJoinOrder is based on \"How Good Are Query Optimizers, Really?\" (VLDB 2015) and uses\na 21-table IMDB-like schema to stress-test query optimizer join ordering. The 13\nqueries (1a through 12a, plus 1b) feature complex multi-table joins (5-9 tables),\nLIKE predicates, IN clauses, cross-table filtering, and MIN aggregations.\n\nThis is the most challenging benchmark to port to DataFrames because:\n1. Multi-table joins (5-9 tables per query) require careful chaining of .join() calls\n2. Implicit cross-joins in FROM clauses (comma-separated tables with WHERE join\n conditions) must be translated to explicit .join() sequences\n3. LIKE patterns with wildcards need string method equivalents\n4. Join order matters for performance, and DataFrame APIs may not preserve the\n optimizer stress-testing intent - document this caveat\n\nThe purpose of DataFrame JoinOrder is NOT to test optimizer join ordering (DataFrames\ndefine explicit join order), but to measure multi-join execution performance across\nplatforms with identical join sequences.", "id": "df-support-joinorder", "preserves": ["JoinOrderBenchmark SQL execution path unchanged - embedded queries and file-loaded queries work identically", "JoinOrderGenerator synthetic data generation produces files loadable by both SQL and DataFrame paths", "get_queries_by_complexity() classification still works for SQL mode"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/joinorder/"}], "state": "done", "title": "Add DataFrame execution support to JoinOrder benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.joinorder.dataframe_queries import get_dataframe_queries; r = get_dataframe_queries(); print(f'{len(r)} queries registered'); assert len(r) == 13\"", "description": "All 13 DataFrame queries are registered", "expected": "13 queries registered", "item_id": "df-support-joinorder", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['joinorder']['supports_dataframe'])\"", "description": "JoinOrder benchmark reports supports_dataframe=True", "expected": "True", "item_id": "df-support-joinorder", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/joinorder/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-joinorder", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-joinorder", "needs": [], "notes": "Create benchbox/core/joinorder/dataframe_queries.py with a QueryRegistry.\nStart with the simpler queries that join 4-5 tables:\n\n- 1a: 5 tables (company_type, info_type, movie_companies, movie_info_idx, title)\n MIN aggregations, LIKE/NOT LIKE on mc.note, equality joins\n- 1b: Same structure as 1a with different predicates\n- 2a: 5 tables (company_name, keyword, movie_companies, movie_keyword, title)\n Equality joins, country_code filter\n- 3a: 4 tables (keyword, movie_info, movie_keyword, title)\n LIKE on keyword, IN clause on mi.info, production_year filter\n- 4a: 5 tables (info_type, keyword, movie_info_idx, movie_keyword, title)\n String comparison (mi_idx.info > '2.0'), production_year filter\n\nTranslation pattern for implicit cross-joins:\n FROM t1, t2, t3 WHERE t1.id = t2.fk AND t2.id = t3.fk\n -> t1.join(t2, left_on=\"id\", right_on=\"fk\").join(t3, left_on=\"id\", right_on=\"fk\")\n\nFor LIKE patterns: col(\"note\").str.contains(\"(co-production)\")\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for queries 1a-4a (4-5 table joins)", "wid": "w1"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w1"], "notes": "More complex queries with 5-8 table joins:\n\n- 5a: 5 tables - European movies with theatrical release notes\n- 6a: 5 tables - Superhero movies with specific actor (LIKE + IN)\n- 7a: 8 tables (aka_name, cast_info, info_type, link_type, movie_link, name,\n person_info, title) - most complex join chain, BETWEEN on production_year,\n gender filter, name_pcode_cf LIKE\n- 8a: 7 tables - Japanese dubbed movies, role_type filter, NOT LIKE\n- 9a: 8 tables - American voice actresses, BETWEEN on year, IN on ci.note,\n person_role_id join to char_name\n\nKey challenge: 7a and 9a have 8-table joins requiring careful join sequencing.\nChoose join order that matches natural FK relationships in the IMDB schema.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for queries 5a-9a (5-8 table joins)", "wid": "w2"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w1"], "notes": "The most complex queries:\n\n- 10a: 7 tables - American producers, company_type + role_type filters\n- 11a: 9 tables (company_name, company_type, keyword, link_type, movie_companies,\n movie_info, movie_keyword, movie_link, title) - most tables in any query,\n country_code != '[pl]', LIKE on cn.name, IS NULL check on mc.note\n- 12a: 8 tables - Drama/horror movies with high ratings, double info_type join\n (it1 for genres, it2 for rating), compound conditions\n\n11a is the most complex query in the benchmark - plan the join sequence carefully\naround the title table as the central fact table.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create expression-family implementations for queries 10a-12a (7-9 table joins)", "wid": "w3"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w1", "w2"], "notes": "Add pandas_impl functions for queries 1a, 1b, 2a, 3a, 4a, 5a, 6a (7 queries).\nKey differences from expression family:\n\n- Joins: pd.merge(left, right, left_on=..., right_on=..., how=\"inner\")\n- LIKE: df[\"col\"].str.contains(\"pattern\", na=False)\n- NOT LIKE: ~df[\"col\"].str.contains(\"pattern\", na=False)\n- IN: df[\"col\"].isin([...])\n- MIN agg: df.agg({\"col\": \"min\"}) or df[\"col\"].min()\n- String comparison: df[df[\"info\"] > \"2.0\"]\n- IS NULL: df[\"col\"].isna()\n\nMulti-table joins chain: reduce(lambda acc, rhs: pd.merge(acc, rhs, ...), tables)\nor sequential merge calls.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create pandas-family implementations for queries 1a-6a (simpler joins)", "wid": "w4"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w2", "w3"], "notes": "Add pandas_impl functions for queries 7a, 8a, 9a, 10a, 11a, 12a (6 queries).\nSame translation patterns as w4.\n\n7a-9a have 7-8 table joins - the sequential pd.merge() chain will be long.\nUse descriptive intermediate variable names for readability:\n merged_title_mc = pd.merge(title, movie_companies, ...)\n merged_with_cn = pd.merge(merged_title_mc, company_name, ...)\n\n11a (9 tables) is the most complex - test this one carefully.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create pandas-family implementations for queries 7a-12a (complex joins)", "wid": "w5"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w4", "w5"], "notes": "Integration:\n1. Add get_dataframe_queries() method to JoinOrderBenchmark in benchmark.py\n2. Set supports_dataframe: True in benchmark_registry.py for joinorder entry\n3. Verify JoinOrderGenerator synthetic data works with DataFrame data loading\n (CSV/Parquet format compatibility)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up JoinOrder benchmark class and registry for DataFrame dispatch", "wid": "w6"}, {"evidence": null, "item_id": "df-support-joinorder", "needs": ["w6"], "notes": "Create tests/unit/core/joinorder/test_dataframe_queries.py:\n- Test all 13 queries registered with correct IDs\n- Test expression_impl on small synthetic IMDB-like data (build minimal\n fixtures with a few rows per table that satisfy join conditions)\n- Test pandas_impl on same fixtures\n- Test result shapes are non-empty (queries with valid data should return rows)\n- Test LIKE/IN/NOT LIKE predicates filter correctly\n- Mark tests @pytest.mark.fast where possible\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for JoinOrder DataFrame queries", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "attempt to implement INFORMATION_SCHEMA queries in DataFrame mode", "instead": "use platform-native introspection", "why": "DataFrames access metadata through native API (df.schema, spark.catalog), not SQL catalog tables"}, {"dont": "force 100% query parity with SQL mode", "instead": "document the expected coverage gap clearly", "why": "ACL, view hierarchy, and constraint queries are inherently SQL-only"}, {"dont": "rewrite the existing 1821-line module from scratch", "instead": "audit and complete what exists", "why": "significant work is already done"}], "approach": "This benchmark has 1821 lines of existing DataFrame operations code. The work is\naudit + integration + gap-filling, not greenfield.\n\nFollow the write_primitives operations manager pattern for PySpark catalog/lakehouse\noperations (stateful, sequential). For schema introspection operations that are\nindependent and stateless (df.schema, df.describe()), consider using the QueryRegistry\npattern instead - this enables the standard DataFrame runner to execute them.\n\nA hybrid approach may be best: QueryRegistry for Tier 1 schema introspection\n(works on all platforms), operations manager for Tier 2/3 catalog+lakehouse\n(PySpark only). The benchmark class exposes both via get_dataframe_queries()\n(returns Tier 1 registry) and get_dataframe_operations() (returns Tier 2/3 manager).\n\nExpect ~25-30% coverage in DataFrame mode vs SQL mode. This is acceptable because\nmany SQL metadata queries (INFORMATION_SCHEMA, ACL, views) are inherently SQL\nconstructs with no DataFrame equivalent.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-metadata-primitives", "id": 384, "reason": "Requires PySpark + Delta Lake runtime. Add to live_integration suite.", "resolution": "promoted", "resolved_item": "integration-tests-with-real-pyspark-delta-lake-catalog", "resolved_reason": null, "summary": "Integration tests with real PySpark + Delta Lake catalog"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-metadata-primitives", "id": 385, "reason": "DDL generation (CREATE TABLE with 1000 columns) is SQL-specific. DataFrame equivalent (constructing wide DataFrames) is a separate design problem.", "resolution": "promoted", "resolved_item": "complexity-stress-testing-wide-tables-large-catalogs-in-data", "resolved_reason": null, "summary": "Complexity stress testing (wide tables, large catalogs) in DataFrame mode"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-metadata-primitives", "id": 386, "reason": "DataFrame APIs don't have GRANT/REVOKE equivalents. These 11 queries are permanently SQL-only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason states ACL/permission testing is 'permanently SQL-only' for DataFrame APIs; not applicable.", "summary": "ACL/permission testing in DataFrame mode"}], "deps": [], "description": "Complete and integrate the existing DataFrame operations module for the Metadata\nPrimitives benchmark, enabling schema introspection benchmarking on Polars, PySpark,\nPandas, and other DataFrame platforms.\n\nMetadata Primitives tests database metadata introspection - INFORMATION_SCHEMA queries,\nSHOW/DESCRIBE/PRAGMA commands, and catalog operations - using 62 SQL queries across\n10 categories: schema (9), column (8), stats (6), query (4), wide table (5), view\nhierarchy (4), complex type (5), large catalog (6), constraint (4), and ACL (11).\n\nA substantial dataframe_operations.py already exists (1821 lines) with:\n- MetadataOperationType enum (13+ operation types across 4 categories)\n- MetadataOperationCategory enum (SCHEMA, CATALOG, LAKEHOUSE, COMPLEXITY)\n- OPERATION_CATEGORIES mapping dict\n- Platform-specific support levels (all platforms for schema, PySpark for catalog,\n PySpark+Delta/Iceberg for lakehouse metadata)\n\nUnlike SQL metadata introspection (which queries INFORMATION_SCHEMA), DataFrame\nmetadata operations use native DataFrame API: df.schema, df.dtypes, df.describe(),\ndf.columns, len(df), and platform-specific catalog APIs (spark.catalog.listTables()).\n\nThe benchmark has 3 tiers of DataFrame support:\n1. Schema introspection: ALL platforms (Polars, Pandas, PySpark, DataFusion, etc.)\n2. Catalog operations: PySpark with configured catalog only\n3. Lakehouse metadata: PySpark + Delta Lake/Iceberg only", "id": "df-support-metadata-primitives", "preserves": ["Complexity testing DDL generation (wide tables, view hierarchies, large catalogs) still works for SQL mode", "MetadataPrimitivesBenchmark SQL execution path unchanged - all 62 SQL queries still work", "Platform-specific SQL variants (ClickHouse, Snowflake, BigQuery, DataFusion) unchanged"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/metadata_primitives/"}], "state": "done", "title": "Add DataFrame execution support to Metadata Primitives benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['metadata_primitives']['supports_dataframe'])\"", "description": "Metadata Primitives reports supports_dataframe=True", "expected": "True", "item_id": "df-support-metadata-primitives", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.metadata_primitives.dataframe_operations import MetadataOperationType, MetadataOperationCategory; print(f'{len(MetadataOperationType)} ops, {len(MetadataOperationCategory)} categories')\"", "description": "Operations module loads without error", "expected": "13 ops, 4 categories", "item_id": "df-support-metadata-primitives", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/metadata_primitives/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-metadata-primitives", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-metadata-primitives", "needs": [], "notes": "Read benchbox/core/metadata_primitives/dataframe_operations.py (1821 lines) thoroughly.\nCompare against the 62 SQL queries in catalog/queries.yaml.\n\nCreate a mapping table across 3 tiers:\n\nTier 1 - Schema introspection (ALL platforms):\n- list_columns -> df.columns\n- get_dtypes -> df.dtypes / df.schema\n- describe_stats -> df.describe()\n- row_count -> len(df) / df.count()\n- column_count -> len(df.columns)\nMap which of the 62 SQL queries correspond to these operations.\n\nTier 2 - Catalog operations (PySpark only):\n- list_databases -> spark.catalog.listDatabases()\n- list_tables -> spark.catalog.listTables()\n- table_exists -> spark.catalog.tableExists()\nMap SQL INFORMATION_SCHEMA queries to catalog API calls.\n\nTier 3 - Lakehouse metadata (PySpark + Delta/Iceberg):\n- table_history -> DeltaTable.history()\n- table_detail -> DeltaTable.detail()\nMap SQL queries to format-specific metadata APIs.\n\nIdentify:\n- Which operations are fully implemented?\n- Which are stubs?\n- Which SQL queries have no DataFrame equivalent (e.g., ACL queries)?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit existing dataframe_operations.py - map SQL queries to DataFrame operations, identify gaps", "wid": "w1"}, {"evidence": null, "item_id": "df-support-metadata-primitives", "needs": ["w1"], "notes": "Ensure Tier 1 operations work across all platforms:\n\nExpression family (Polars, DataFusion, PySpark):\n- Schema access via ctx.get_table(name).schema\n- Column listing via ctx.get_table(name).columns\n- Statistics via describe() or platform-specific stats methods\n\nPandas family (Pandas, Modin, cuDF, Dask):\n- df.dtypes, df.describe(), df.info(), df.columns, len(df)\n- df.memory_usage() for size statistics\n- df.nunique() for cardinality\n\nThese are the most broadly supported operations and should work on all 8\nDataFrame platforms. Implement as DataFrameQuery objects with expression_impl\nand pandas_impl, even though these are metadata operations - the QueryRegistry\npattern enables integration with the standard DataFrame runner.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Complete schema introspection operations for all DataFrame platforms", "wid": "w2"}, {"evidence": null, "item_id": "df-support-metadata-primitives", "needs": ["w1"], "notes": "Tier 2 (PySpark catalog):\n- Implement using spark.catalog.* API\n- listDatabases(), listTables(), listColumns(), tableExists()\n- These require a SparkSession with a configured catalog\n\nTier 3 (PySpark + Delta/Iceberg):\n- DeltaTable.forPath(spark, path).history() / .detail()\n- Iceberg snapshot listing via table properties\n\nDefine comprehensive skip lists:\n- SKIP_FOR_DATAFRAME: ACL queries (GRANT/REVOKE timing), view hierarchy queries\n (DataFrames don't have views), constraint queries (FK/PK are schema-level)\n- SKIP_FOR_POLARS/PANDAS: Catalog and lakehouse operations\n- SKIP_FOR_EXPRESSION_FAMILY: Catalog operations (except PySpark)\n\nMany of the 62 SQL queries are SQL-specific (INFORMATION_SCHEMA, SHOW commands)\nand will be skipped. Document the expected coverage: ~15-20 operations in\nDataFrame mode vs 62 SQL queries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Complete catalog and lakehouse operations for PySpark and define skip lists", "wid": "w3"}, {"evidence": null, "item_id": "df-support-metadata-primitives", "needs": ["w2", "w3"], "notes": "Integration:\n1. Add get_dataframe_queries() or get_dataframe_operations() to\n MetadataPrimitivesBenchmark - decide which pattern based on w1 audit.\n If operations are independent (stateless introspection), use QueryRegistry.\n If operations are sequential/stateful, use operations manager.\n2. Set supports_dataframe: True in benchmark_registry.py\n3. Ensure TPC-H schema tables are loaded before metadata introspection runs\n (metadata queries inspect the loaded tables)\n4. Handle complexity testing (wide tables, large catalogs) - may need to\n generate test structures in DataFrame mode too\n\nTests (tests/unit/core/metadata_primitives/test_dataframe_operations.py):\n- Test MetadataOperationType enum values\n- Test per-platform capability detection\n- Test schema introspection operations on small synthetic DataFrames\n- Test skip list correctness (Polars skips catalog ops, etc.)\n- Test operation categories mapping\n- Mark tests @pytest.mark.fast\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up MetadataPrimitivesBenchmark class, registry, and add unit tests", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "create trivial variants (e.g., just renaming aliases)", "instead": "each variant must use a genuinely different execution strategy", "why": "the point is structural diversity"}, {"dont": "create variants that produce different results", "instead": "every variant must produce identical output for the same input data and parameters", "why": "semantic equivalence is the core property"}, {"dont": "modify tpch/dataframe_queries.py", "instead": "import base implementations and build variants on top", "why": "TPC-Havoc extends TPC-H, not modifies it"}], "approach": "Create a dataframe_queries/ subdirectory (like clickbench) with per-query modules\n(q01.py through q22.py). Each module exports 10 DataFrameQuery objects.\n\nImport the base TPC-H DataFrame implementation as v1 (baseline) for each query from\ntpch/dataframe_queries.py, then define v2-v10 as structural alternatives.\n\nA central registry.py aggregates all queries into a single QueryRegistry with\nIDs like \"Q1v1\", \"Q1v2\", etc. The benchmark's get_dataframe_queries() returns\nthis registry.\n\nDesign the variant categories based on actual TPC-H query structure - not all\nvariant types apply to all queries (e.g., join-reorder only applies to multi-join\nqueries). Document which variant strategies apply to which queries in\nDATAFRAME_VARIANTS.md.\n\nUse the existing TPC-H parameter override system (set_parameter_overrides) for\nseed-based reproducibility.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-tpchavoc", "id": 387, "reason": "Full equivalence testing is expensive (220 queries x 2 families). Start with Q1-Q3 equivalence in unit tests, defer comprehensive testing to integration suite.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: A 220-variant equivalence gate already exists in CI (tests/integration/test_tpchavoc_variant_equivalence.py: 'full gate sweeps all 220 variants'); already done.", "summary": "Automated variant equivalence testing across all 22 queries at SF=0.01"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-tpchavoc", "id": 388, "reason": "Analysis tooling is follow-up work after the implementations exist and have been benchmarked.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: 'Variant performance analysis tooling - fastest DataFrame patterns per platform' is a vague future-analysis idea with no concrete design, scope, or committed demand.", "summary": "Variant performance analysis tooling - identify which DataFrame patterns are fastest per platform"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-tpchavoc", "id": 389, "reason": "The first complete registry covers Q16-Q22 with canonical execution plus final projection/order replay to preserve equivalence. Deeper variants for Q19 condition unions, Q20 semi-join ordering, Q21 anti-join alternatives, and Q22 materialized average balance should be developed with broader equivalence coverage.", "resolution": "promoted", "resolved_item": "replace-q16-q22-result-replay-variants-with-deeper-hand-writ", "resolved_reason": null, "summary": "Replace Q16-Q22 result-replay variants with deeper hand-written structural rewrites"}], "deps": [], "description": "Add DataFrame variant implementations for TPC-Havoc, enabling execution of\nstructurally diverse TPC-H query variants on Polars, PySpark, DataFusion, Pandas,\nModin, cuDF, and Dask platforms.\n\nTPC-Havoc generates 10 structural SQL variants per TPC-H query (22 queries x 10\nvariants = 220 total). Each variant is semantically equivalent but uses different\nSQL constructs (CTEs, scalar subqueries, window functions, UNION, etc.) to stress\ndifferent optimizer components.\n\nFor DataFrame execution, the concept translates differently: instead of varying SQL\nsyntax, DataFrame variants vary the API call sequence - different join orders,\ndifferent aggregation strategies (pre-filter vs post-filter), lazy vs eager\nevaluation patterns, and different window function formulations. The goal shifts\nfrom \"stress the SQL optimizer\" to \"measure how different DataFrame API patterns\naffect execution performance on the same logical query.\"\n\nTPC-H already has DataFrame queries (tpch/dataframe_queries.py) with 22 queries\nin both expression and pandas families. TPC-Havoc DataFrame extends these with\nalternative implementations that use structurally different DataFrame API patterns.\n\nThis is architecturally unique - no other benchmark has the variant concept in\nDataFrame mode. Requires designing a new variant registration pattern for\nDataFrameQuery objects.", "id": "df-support-tpchavoc", "preserves": ["ResultValidator variant equivalence checking still works for SQL mode", "TPCHBenchmark base class DataFrame queries (tpch/dataframe_queries.py) are NOT modified - TPC-Havoc imports them as baselines", "TPCHavocBenchmark SQL variant generation unchanged - all 220 SQL variants still work via VariantGenerator/StaticSQLVariant", "benchbox run --platform duckdb --benchmark tpchavoc still runs SQL variants"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/dataframe_queries.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/"}], "state": "done", "title": "Add DataFrame execution support to TPC-Havoc benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.tpchavoc.dataframe_queries import get_dataframe_queries; r = get_dataframe_queries(); print(f'{len(r)} variants registered'); assert len(r) == 220\"", "description": "All 220 DataFrame variants are registered", "expected": "220 variants registered", "item_id": "df-support-tpchavoc", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_METADATA; print(BENCHMARK_METADATA['tpchavoc']['supports_dataframe'])\"", "description": "TPC-Havoc benchmark reports supports_dataframe=True", "expected": "True", "item_id": "df-support-tpchavoc", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/tpchavoc/test_tpchavoc_dataframe_queries.py -k 'q1_equivalence' -v", "description": "Q1 variants produce equivalent results", "expected": "all tests pass, 0 failures", "item_id": "df-support-tpchavoc", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-tpchavoc", "needs": [], "notes": "Design how DataFrame variants integrate with the existing DataFrameQuery/QueryRegistry\nsystem. Key decisions:\n\n1. Query ID scheme: Use \"Q1v1\", \"Q1v2\", ... \"Q1v10\" or \"Q1-scalar-subquery\",\n \"Q1-cte-rewrite\"? The former is consistent with SQL variant IDs; the latter\n is more descriptive for DataFrame where structural names matter.\n\n2. Registry structure: One flat QueryRegistry with all 220 entries, or a nested\n structure (VariantRegistry wrapping QueryRegistry)? Flat is simpler and works\n with the existing dataframe_runner dispatch.\n\n3. Variant categories for DataFrame (map from SQL variant types):\n - join-reorder: Different .join() sequences for same result\n - pre-filter: Filter before join vs after join\n - window-alternative: Window function vs groupby+join for same logic\n - cte-inline: Intermediate DataFrame vs chained expression\n - aggregation-strategy: Multiple small groupbys vs single large groupby\n - broadcast-hint: Explicit broadcast for small tables vs default\n - lazy-eager: .collect() placement variations (expression family only)\n\n4. How variants relate to base TPC-H queries: Import from tpch/dataframe_queries.py\n as the \"v0\" baseline, then define v1-v10 as alternatives.\n\nWrite a short design doc in benchbox/core/tpchavoc/DATAFRAME_VARIANTS.md\ndocumenting the chosen approach before implementation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design DataFrame variant architecture and registration pattern", "wid": "w1"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w8", "w9"], "notes": "Integration:\n1. Add get_dataframe_queries() method to TPCHavocBenchmark\n - Must return all variants (not just base queries)\n - Consider whether to return all 220 at once or support filtering by query/variant\n2. Set supports_dataframe: True in benchmark_registry.py for tpchavoc\n3. Add variant equivalence validation for DataFrame results\n (extend ResultValidator to handle DataFrame output comparison)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up TPC-Havoc benchmark class and registry for DataFrame dispatch", "wid": "w10"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w10"], "notes": "Create tests/unit/core/tpchavoc/test_dataframe_queries.py:\n- Test variant registration (220 total, 10 per query)\n- Test all variants for Q1 produce identical results on small TPC-H data\n- Test expression_impl and pandas_impl for at least Q1-Q3 variants\n- Test variant metadata (description, categories)\n- Mark equivalence tests @pytest.mark.slow (they need TPC-H SF=0.01 data)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for TPC-Havoc DataFrame variants", "wid": "w11"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "Create benchbox/core/tpchavoc/dataframe_queries/ directory with per-query modules\n(q01.py, q02.py, q03.py). For each query, create 10 expression_impl variants.\n\nQ1 (Pricing Summary - filter + groupby + aggregation):\n- v1: Baseline (import from tpch dataframe Q1)\n- v2: Pre-compute aggregation columns before groupby\n- v3: Window functions instead of groupby (partition + distinct)\n- v4: Two-pass: compute sums first, then averages from sums\n- v5: Filter using .when().then() instead of boolean mask\n- v6: Explicit column selection before groupby (projection pushdown hint)\n- v7: Sort before groupby (tests sort-based aggregation)\n- v8: Intermediate DataFrame for filtered data (CTE equivalent)\n- v9: Compute each aggregate separately then join results\n- v10: Reverse sort order then reverse at end\n\nQ2 (Minimum Cost Supplier) and Q3 (Shipping Priority) follow similar variant\naxes but emphasize join-reorder variants due to multi-table joins.\n\nEach variant must produce identical results to the base query.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q1-Q3 (30 variants)", "wid": "w2"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "Continue with q04.py, q05.py, q06.py.\n\nQ4 (Order Priority Checking - EXISTS subquery):\n- Variant axes: semi-join vs filter-join, pre-filter lineitem dates vs post-filter\nQ5 (Local Supplier Volume - 6-table join + groupby):\n- Variant axes: join order (6 permutations), pre-filter nation/region before join\nQ6 (Forecasting Revenue Change - single-table filter + sum):\n- Variant axes: filter order, compound vs chained filters, pre-select columns\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q4-Q6 (30 variants)", "wid": "w3"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q07.py through q10.py. These are join-heavy queries where the primary variant\naxis is join order:\n- v1: Baseline join order from tpch/dataframe_queries.py\n- v2-v4: Alternative join orders (small-to-large, large-to-small, star-join)\n- v5-v7: Pre-filter variants (filter dimension tables before joining)\n- v8-v10: Aggregation strategy variants\n\nQ7 (Volume Shipping), Q8 (National Market Share), Q9 (Product Type Profit),\nQ10 (Returned Item Reporting).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q7-Q10 (40 variants)", "wid": "w4"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q11.py through q14.py. Mix of subquery-heavy and aggregation-heavy queries.\n\nQ11 (Important Stock Identification - HAVING with subquery):\n- Variant axes: subquery as intermediate DF vs inline, threshold pre-compute\nQ12 (Shipping Modes and Order Priority - CASE + groupby):\n- Variant axes: CASE as .when() chain vs boolean masks, filter-first vs agg-first\nQ13 (Customer Distribution - LEFT JOIN + groupby + groupby):\n- Variant axes: two-stage vs single-stage aggregation, NOT LIKE as anti-join\nQ14 (Promotion Effect - CASE inside SUM):\n- Variant axes: conditional sum vs filter+sum+join, pre-filter parts\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q11-Q14 (40 variants)", "wid": "w5"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q15.py through q18.py. Complex subquery patterns.\n\nQ15 (Top Supplier - subquery for max revenue):\n- Variant axes: intermediate DF vs chained, top-1 vs filter-to-max\nQ16 (Parts/Supplier Count - DISTINCT, NOT IN):\n- Variant axes: anti-join vs ~isin(), distinct before vs after join\nQ17 (Small Quantity Order Revenue - correlated subquery):\n- Variant axes: join-with-avg vs window function, filter timing\nQ18 (Large Volume Customer - HAVING):\n- Variant axes: filter-after-groupby vs pre-aggregate, join order\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q15-Q18 (40 variants)", "wid": "w6"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w1"], "notes": "q19.py through q22.py. Most complex query patterns.\n\nQ19 (Discounted Revenue - compound OR conditions):\n- Variant axes: union of filtered sets vs single complex filter, per-brand subframes\nQ20 (Potential Part Promotion - multi-level subquery):\n- Variant axes: join chain vs nested intermediates, semi-join vs filter\nQ21 (Suppliers Who Kept Orders Waiting - EXISTS/NOT EXISTS):\n- Variant axes: semi-join + anti-join vs boolean masks, join order for 4 tables\nQ22 (Global Sales Opportunity - nested subqueries):\n- Variant axes: intermediate DFs vs chained, avg pre-compute vs inline\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement expression-family variants for Q19-Q22 (40 variants)", "wid": "w7"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w2", "w3", "w4", "w5"], "notes": "Add pandas_impl variants to q01.py through q11.py, paralleling expression_impl.\nMany variant strategies translate directly (join reorder, pre-filter, etc.).\n\nReplace expression-specific variants (lazy/eager, collect placement) with\npandas-specific alternatives:\n- .apply() vs vectorized operations\n- pd.merge() vs df.join() (index-based)\n- .groupby().agg() vs .pivot_table()\n- iterative merge chain vs reduce(merge, tables)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement pandas-family variants for Q1-Q11 (110 variants)", "wid": "w8"}, {"evidence": null, "item_id": "df-support-tpchavoc", "needs": ["w6", "w7"], "notes": "Add pandas_impl variants to q12.py through q22.py.\n\nFor the complex subquery patterns (Q15-Q22), pandas variants emphasize:\n- Intermediate DataFrames vs chained operations\n- .merge() join order permutations\n- .pipe() chains vs sequential assignment\n- Index-based operations vs column-based\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement pandas-family variants for Q12-Q22 (110 variants)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "attempt to implement transactions on Polars/Pandas", "instead": "skip cleanly with informative messages", "why": "they have no ACID semantics"}, {"dont": "map SQL BEGIN/COMMIT/ROLLBACK directly", "instead": "use the Delta Lake/Iceberg native patterns documented in dataframe_operations.py", "why": "Delta Lake uses atomic writes, not explicit transaction blocks"}, {"dont": "rewrite the existing 1290-line module from scratch", "instead": "audit and complete what exists", "why": "significant work is already done"}], "approach": "This benchmark already has substantial DataFrame operations code (1290 lines).\nThe work is primarily audit + integration, not greenfield implementation.\n\nFollow the write_primitives pattern: the benchmark class exposes\nget_dataframe_operations() returning an operations manager, not\nget_dataframe_queries() returning a QueryRegistry. The dataframe_runner\nalready handles this via the operations manager path.\n\nStart with the audit (w1) to understand what's implemented vs stub. The\nexisting code uses DataFrameMaintenanceCapabilities and TransactionIsolation\nfrom benchbox/core/dataframe/maintenance_interface.py - verify these are\ncorrectly integrated.\n\nOnly PySpark+Delta/Iceberg supports DataFrame transactions. Polars, Pandas,\nand other platforms should be explicitly skipped with clear messaging.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-10T00:00:00Z", "completed_pr": null, "created_at": "2026-04-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-transaction-primitives", "id": 390, "reason": "Requires PySpark + Delta Lake runtime. Add to live_integration test suite with @pytest.mark.live_integration.", "resolution": "promoted", "resolved_item": "integration-tests-with-real-pyspark-delta-lake", "resolved_reason": null, "summary": "Integration tests with real PySpark + Delta Lake"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-transaction-primitives", "id": 391, "reason": "Iceberg support depends on PySpark + Iceberg catalog setup. Defer until Delta Lake path is validated.", "resolution": "promoted", "resolved_item": "iceberg-table-format-support-testing", "resolved_reason": null, "summary": "Iceberg table format support testing"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "df-support-transaction-primitives", "id": 392, "reason": "Some SQL operations (savepoints, explicit isolation levels) have no DataFrame equivalent. Document the parity gap rather than attempting impossible translations.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly rejects the deferred action: 'document the parity gap rather than attempting impossible translations.'", "summary": "Map all 23 SQL operations to DataFrame equivalents with full parity"}], "deps": [], "description": "Complete and integrate the existing DataFrame operations module for the Transaction\nPrimitives benchmark, enabling execution on PySpark + Delta Lake/Iceberg platforms.\n\nTransaction Primitives tests ACID transaction semantics: commit, rollback, savepoints,\nisolation levels, and multi-statement transactions using 23 SQL operations on TPC-H\nstaging tables (txn_orders, txn_lineitem, txn_customer).\n\nA substantial dataframe_operations.py already exists (1290 lines) with:\n- TransactionOperationType enum (10 operation types: ATOMIC_INSERT, ROLLBACK_TO_VERSION,\n TIME_TRAVEL_QUERY, CONCURRENT_WRITE, etc.)\n- DataFrameTransactionCapabilities dataclass (per-platform capability tracking)\n- Integration with DataFrameMaintenanceCapabilities and TransactionIsolation\n- Platform-specific support declarations (PySpark+Delta, PySpark+Iceberg)\n\nKey constraint: DataFrame transaction semantics are fundamentally different from SQL.\nSQL uses BEGIN/COMMIT/ROLLBACK; Delta Lake uses atomic writes with RESTORE for\nrollback; Iceberg uses snapshot-based isolation. Polars/Pandas have NO transaction\nsupport. This benchmark is PySpark-only in DataFrame mode.\n\nThe work is primarily integration - wiring the existing operations module into the\nbenchmark class, runner, and registry - plus filling in any incomplete operation\nimplementations and adding tests.", "id": "df-support-transaction-primitives", "preserves": ["Platform overrides for SQL mode (e.g., DuckDB savepoint skips) still work", "TPC-H staging table schema (txn_orders, txn_lineitem, txn_customer) unchanged", "TransactionPrimitivesBenchmark SQL execution path unchanged - all 23 SQL operations still work via OperationExecutor interface"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/transaction_primitives/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/transaction_primitives/"}], "state": "done", "title": "Add DataFrame execution support to Transaction Primitives benchmark", "verifications": [{"command": "uv run -- python -c \"from benchbox.core.benchmark_registry import BENCHMARK_REGISTRY; print(BENCHMARK_REGISTRY['transaction_primitives']['supports_dataframe'])\"", "description": "Transaction Primitives reports supports_dataframe=True", "expected": "True", "item_id": "df-support-transaction-primitives", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"from benchbox.core.transaction_primitives.dataframe_operations import TransactionOperationType; print(f'{len(TransactionOperationType)} operation types')\"", "description": "Operations manager loads without error", "expected": "10 operation types", "item_id": "df-support-transaction-primitives", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/transaction_primitives/ -v", "description": "Unit tests pass", "expected": "all tests pass, 0 failures", "item_id": "df-support-transaction-primitives", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "df-support-transaction-primitives", "needs": [], "notes": "Read benchbox/core/transaction_primitives/dataframe_operations.py (1290 lines) thoroughly.\nCompare against the 23 SQL operations defined in catalog/operations.yaml.\n\nCreate a mapping table:\n- Which SQL operations have DataFrame equivalents already implemented?\n- Which operations are impossible in DataFrame mode (e.g., SQL-level savepoints)?\n- Which operations need new implementations?\n- Which DataFrameTransactionCapabilities are fully implemented vs stubs?\n\nCheck integration points:\n- Does the module use get_maintenance_operations_for_platform() correctly?\n- Does it handle DataFrameMaintenanceCapabilities properly?\n- Are the TransactionIsolation levels correctly mapped?\n\nDocument findings in a comment block at the top of the file or in a separate\naudit notes file to guide w2.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit existing dataframe_operations.py - map operations to benchmark operations, identify gaps", "wid": "w1"}, {"evidence": null, "item_id": "df-support-transaction-primitives", "needs": ["w1"], "notes": "Based on the w1 audit:\n\n1. Complete any stub implementations in dataframe_operations.py\n2. Define SKIP_FOR_DATAFRAME list for operations that have no DataFrame equivalent:\n - SQL-level savepoints (SAVEPOINT/ROLLBACK TO) - no Delta Lake/Iceberg equivalent\n - Explicit isolation level setting (SET TRANSACTION ISOLATION LEVEL)\n - Operations requiring explicit BEGIN/COMMIT blocks\n3. Define SKIP_FOR_POLARS and SKIP_FOR_PANDAS as complete skip lists (all operations)\n since transaction primitives only work on PySpark+Delta/Iceberg\n4. Ensure error handling for unsupported platforms returns clear skip messages\n\nThe operations manager pattern (not QueryRegistry pattern) is correct for this\nbenchmark - transaction operations are stateful and sequential, not independent\nqueries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Complete missing operation implementations and add skip lists for unsupported operations", "wid": "w2"}, {"evidence": null, "item_id": "df-support-transaction-primitives", "needs": ["w2"], "notes": "Integration steps:\n1. Add get_dataframe_operations() method to TransactionPrimitivesBenchmark in\n benchmark.py - similar to write_primitives pattern:\n ```python\n def get_dataframe_operations(self, platform_name):\n from .dataframe_operations import get_transaction_manager\n return get_transaction_manager(platform_name)\n ```\n2. Set supports_dataframe: True in benchmark_registry.py for transaction_primitives\n3. Verify the dataframe_runner.py handles operations-based benchmarks correctly\n (it should fall through to get_dataframe_operations if no get_dataframe_queries)\n4. Add platform capability checks - only PySpark with Delta Lake or Iceberg\n should attempt this benchmark in DataFrame mode\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire up TransactionPrimitivesBenchmark class and registry for DataFrame dispatch", "wid": "w3"}, {"evidence": null, "item_id": "df-support-transaction-primitives", "needs": ["w3"], "notes": "Create tests/unit/core/transaction_primitives/test_dataframe_operations.py:\n- Test TransactionOperationType enum values match expected operations\n- Test DataFrameTransactionCapabilities for PySpark reports correct support\n- Test Polars/Pandas capabilities report all operations as unsupported\n- Test skip list generation for non-PySpark platforms\n- Test operation manager instantiation with mock platform\n- Mock-based tests (real PySpark+Delta tests belong in integration suite)\n- Mark all tests @pytest.mark.fast\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for Transaction Primitives DataFrame operations", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Project Lifecycle Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: Completed \u2705\"\n- \"**Description**: Create databases with distinct naming that reflects scale factor and tuning configurations, enabling users to understand essential configurations from the database name alone\"\n- \"**Tasks**:\"\n- \"**Files Affected**: `benchbox/platforms/base.py`, all platform adapters, `benchbox/cli/config.py`, database validation utilities, CLI modules\"\n- \"**Implementation Strategy**:\"\n- \"**Implementation Summary**:\"\n- \"**Impact**: Enables users to easily identify database configurations from names alone, facilitates database organization and management, supports parallel testing of different configurations, improves debugging and troubleshooting by making configuration visible in database listings\"", "id": "distinct-database-naming-based-on-configuration-characteristics", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Distinct Database Naming Based on Configuration Characteristics", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Documentation audit (2026-04-01) found 10 CLI commands with NO documentation\nand 13 flags on `benchbox run` that are undocumented.\n\nUndocumented commands:\n- benchbox visualize (benchbox/cli/commands/visualize.py)\n- benchbox report (benchbox/cli/commands/report.py)\n- benchbox metrics (benchbox/cli/commands/metrics.py)\n- benchbox aggregate (benchbox/cli/commands/aggregate.py)\n- benchbox datagen (benchbox/cli/commands/datagen.py)\n- benchbox download-answers (benchbox/cli/commands/download_answers.py)\n- benchbox plan-history (benchbox/cli/commands/plan_history.py)\n- benchbox show-plan (benchbox/cli/commands/show_plan.py)\n- benchbox setup (benchbox/cli/commands/setup.py)\n- benchbox run-official (benchbox/cli/commands/run_official.py - hidden)\n\nUndocumented `benchbox run` flags:\n--table-mode, --table-format, --presort, --sorted-ingestion-mode,\n--sorted-ingestion-method, --databricks-clustering-strategy,\n--liquid-clustering-columns, --mode, --no-monitoring, --no-progress,\n--ignore-memory-warnings, --global-cache, --plan-config\n\nSome commands may be intentionally hidden (run-official). For those, a brief\ninternal note is sufficient rather than full public docs.", "id": "docs-document-undocumented-cli-commands", "preserves": [], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "docs/CLI_REFERENCE.md"}, {"kind": "only_modify", "path_glob": "docs/reference/cli/"}], "state": "done", "title": "Document 10 undocumented CLI commands and 13 undocumented run flags", "verifications": [{"command": "for cmd in visualize report metrics aggregate datagen setup; do [ -f docs/reference/cli/$cmd.md ] && echo \"PASS: $cmd\" || echo \"FAIL: $cmd missing\"; done", "description": "All non-hidden commands have doc pages", "expected": "PASS for all commands", "item_id": "docs-document-undocumented-cli-commands", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -c 'table-mode\\|table-format\\|presort\\|plan-config\\|no-monitoring\\|no-progress\\|ignore-memory-warnings\\|global-cache\\|mode' docs/reference/cli/run.md | xargs -I{} test {} -ge 8 && echo 'PASS: flags documented' || echo 'FAIL: flags missing'", "description": "New run flags documented", "expected": "PASS: flags documented", "item_id": "docs-document-undocumented-cli-commands", "last_result": null, "last_run": null, "seq": 2}, {"command": "for cmd in visualize report metrics aggregate datagen setup; do grep -qi $cmd docs/CLI_REFERENCE.md && echo \"PASS: $cmd\" || echo \"FAIL: $cmd missing\"; done", "description": "CLI_REFERENCE.md indexes new commands", "expected": "PASS for all commands", "item_id": "docs-document-undocumented-cli-commands", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "docs-document-undocumented-cli-commands", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document Databricks-specific run flags (5 flags)", "wid": "w1"}, {"evidence": null, "item_id": "docs-document-undocumented-cli-commands", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document general undocumented run flags (8 flags)", "wid": "w2"}, {"evidence": null, "item_id": "docs-document-undocumented-cli-commands", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs for visualize command", "wid": "w3"}, {"evidence": null, "item_id": "docs-document-undocumented-cli-commands", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs for report and metrics commands", "wid": "w4"}, {"evidence": null, "item_id": "docs-document-undocumented-cli-commands", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs for datagen, aggregate, and setup commands", "wid": "w5"}, {"evidence": null, "item_id": "docs-document-undocumented-cli-commands", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create docs for plan-history, show-plan, and download-answers", "wid": "w6"}, {"evidence": null, "item_id": "docs-document-undocumented-cli-commands", "needs": ["w1", "w2", "w3", "w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update CLI_REFERENCE.md and index to list all commands", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Documentation audit (2026-04-01) found 3 critical accuracy errors that will\ncause user failures if followed:\n\n1. `docs/advanced/custom-benchmarks.md` has a Python syntax error (missing\n colon in loop statement) and calls a non-existent method\n `get_create_tables_sql(\"duckdb\")` with a dialect parameter that doesn't\n exist in the API.\n\n2. `docs/benchmarks/ai-primitives.md` documents a complete benchmark (16\n queries, schema, examples) that has NO implementation in the codebase.\n Users cannot run this benchmark.\n\n3. `docs/tutorials/dataframe-quickstart.md` uses `-o` as shorthand for\n `--output`, but no such short flag exists in the CLI. Also references\n deprecated `--df-tuning` and `benchbox df-tuning` commands that should\n be `benchbox tuning`.\n\nThese are the highest-impact doc bugs because they cause immediate user\nerrors when following the documentation.", "id": "docs-fix-critical-accuracy-errors", "preserves": [], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "docs/advanced/custom-benchmarks.md"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/ai-primitives.md"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/index.md"}, {"kind": "only_modify", "path_glob": "docs/tutorials/dataframe-quickstart.md"}], "state": "done", "title": "Fix critical documentation accuracy errors - broken code, phantom benchmark, wrong flags", "verifications": [{"command": "uv run -- python -c \"import ast, re, pathlib; src=pathlib.Path('docs/advanced/custom-benchmarks.md').read_text(); blocks=re.findall(r'```python\\n(.*?)```', src, re.S); [ast.parse(b) for b in blocks]; print('PASS: all code blocks parse')\"", "description": "Python code blocks in custom-benchmarks.md parse without syntax errors", "expected": "PASS: all code blocks parse", "item_id": "docs-fix-critical-accuracy-errors", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'get_create_tables_sql.*duckdb\\|get_create_tables_sql.*dialect' docs/advanced/custom-benchmarks.md || echo 'PASS: no phantom API call'", "description": "No reference to non-existent get_create_tables_sql dialect parameter", "expected": "PASS: no phantom API call", "item_id": "docs-fix-critical-accuracy-errors", "last_result": null, "last_run": null, "seq": 2}, {"command": "if [ ! -f docs/benchmarks/ai-primitives.md ]; then echo 'PASS: file removed'; else grep -q 'Not Yet Implemented\\|not yet implemented\\|Planned' docs/benchmarks/ai-primitives.md && echo 'PASS: marked as planned' || echo 'FAIL: no implementation marker'; fi", "description": "ai-primitives.md removed or clearly marked as not implemented", "expected": "PASS", "item_id": "docs-fix-critical-accuracy-errors", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -n '\\b-o\\b' docs/tutorials/dataframe-quickstart.md | grep -v '\\-\\-o' || echo 'PASS: no bare -o flags'", "description": "No bare -o flags remain in dataframe quickstart", "expected": "PASS: no bare -o flags", "item_id": "docs-fix-critical-accuracy-errors", "last_result": null, "last_run": null, "seq": 4}, {"command": "grep -rn 'df-tuning\\|df_tuning' docs/tutorials/ || echo 'PASS: no deprecated refs'", "description": "No df-tuning references remain", "expected": "PASS: no deprecated refs", "item_id": "docs-fix-critical-accuracy-errors", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "docs-fix-critical-accuracy-errors", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix syntax error and non-existent API call in custom-benchmarks.md", "wid": "w1"}, {"evidence": null, "item_id": "docs-fix-critical-accuracy-errors", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove or stub ai-primitives.md - no implementation exists", "wid": "w2"}, {"evidence": null, "item_id": "docs-fix-critical-accuracy-errors", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix incorrect -o flag and deprecated df-tuning in dataframe-quickstart.md", "wid": "w3"}], "worktree": "main"} @@ -388,229 +388,229 @@ {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Documentation audit (2026-04-01) found systemic naming inconsistencies:\n\nBenchmark naming: Docs use hyphenated filenames (tpc-h.md, tpc-ds.md,\njoin-order.md) while the code registry uses compact/underscored names\n(tpch, tpcds, joinorder). Users reading docs can't directly use the\ndocumented names in CLI commands.\n\nAffected benchmarks: tpc-h->tpch, tpc-ds->tpcds, tpc-di->tpcdi,\ntpc-havoc->tpchavoc, tpch-skew->tpch_skew, tpc-ds-obt->tpcds_obt,\njoin-order->joinorder, metadata-primitives->metadata_primitives,\nread-primitives->read_primitives, write-primitives->write_primitives,\ntransaction-primitives->transaction_primitives, tsbs-devops->tsbs_devops.\n\nPlatform naming: Docs use different conventions than registration names\n(pg_duckdb.md vs registered pg-duckdb, etc.)\n\nResolution approach: Don't rename files (URLs would break). Instead,\nensure each doc prominently shows the CLI-usable name near the top.", "id": "docs-fix-naming-inconsistencies", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "docs/benchmarks/"}, {"kind": "only_modify", "path_glob": "docs/platforms/pg_duckdb.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/pg_mooncake.md"}], "state": "done", "title": "Standardize benchmark and platform naming across documentation", "verifications": [{"command": "total=$(ls docs/benchmarks/*.md | grep -v index.md | wc -l); found=$(grep -l 'CLI name' docs/benchmarks/*.md | grep -v index.md | wc -l); echo \"$found/$total benchmarks have CLI name callout\"; [ \"$found\" -eq \"$total\" ] && echo 'PASS' || echo 'FAIL'", "description": "All benchmark docs (excl. index) contain CLI name callouts", "expected": "20/20 benchmarks have CLI name callout - PASS", "item_id": "docs-fix-naming-inconsistencies", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -q 'CLI Name\\|CLI name\\|cli.name' docs/benchmarks/index.md && echo 'PASS: index has CLI names' || echo 'FAIL: index missing CLI names'", "description": "Index page has CLI name column", "expected": "PASS: index has CLI names", "item_id": "docs-fix-naming-inconsistencies", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "docs-fix-naming-inconsistencies", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add CLI name callout to TPC-family benchmark docs (7 files)", "wid": "w1"}, {"evidence": null, "item_id": "docs-fix-naming-inconsistencies", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add CLI name callout to non-TPC benchmark docs (8 files)", "wid": "w2"}, {"evidence": null, "item_id": "docs-fix-naming-inconsistencies", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add CLI name callout to primitives benchmark docs (4 files)", "wid": "w3"}, {"evidence": null, "item_id": "docs-fix-naming-inconsistencies", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update benchmark index page with CLI names", "wid": "w4"}, {"evidence": null, "item_id": "docs-fix-naming-inconsistencies", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add CLI name callout to platform docs with naming mismatches", "wid": "w5"}, {"evidence": null, "item_id": "docs-fix-naming-inconsistencies", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Clarify TPC-DS table count (24 vs 25 - DBGEN_VERSION)", "wid": "w6"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Documentation audit (2026-04-01) found that the two foundational design docs\nare severely outdated:\n\n1. `docs/design/architecture.md` describes a non-existent `.run()` method\n and Template Method pattern. The actual execution model is lifecycle-based\n via `ExecutionPhases` and `BenchmarkResults` models. Missing entirely:\n `DataFrameContext`, `ExpressionFamilyAdapter`, `PandasFamilyAdapter`\n classes and the whole DataFrame execution path.\n\n2. `docs/design/structure.md` completely omits `benchbox/core/` (30+\n subsystems) and `benchbox/platforms/` (40+ database adapters). A user\n reading this doc cannot understand the actual project organization.\n\nThese are the docs developers read FIRST when trying to understand the\ncodebase. Getting them wrong cascades into confusion everywhere.", "id": "docs-rewrite-architecture-and-structure", "preserves": [], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "docs/design/architecture.md"}, {"kind": "only_modify", "path_glob": "docs/design/structure.md"}], "state": "done", "title": "Rewrite architecture.md and structure.md to match current codebase", "verifications": [{"command": "grep -n 'Template Method\\|\\.run()' docs/design/architecture.md || echo 'PASS: no stale pattern refs'", "description": "No references to non-existent .run() Template Method pattern", "expected": "PASS: no stale pattern refs", "item_id": "docs-rewrite-architecture-and-structure", "last_result": null, "last_run": null, "seq": 1}, {"command": "for d in core platforms benchmarks cli; do grep -q \"benchbox/$d\" docs/design/structure.md && echo \"PASS: $d\" || echo \"FAIL: $d missing\"; done", "description": "Key subsystems mentioned in structure.md", "expected": "PASS for all four directories", "item_id": "docs-rewrite-architecture-and-structure", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -c 'ExpressionFamilyAdapter\\|PandasFamilyAdapter' docs/design/architecture.md | xargs -I{} test {} -ge 2 && echo 'PASS: adapter families documented' || echo 'FAIL: adapter families missing'", "description": "DataFrame adapter families documented in architecture.md", "expected": "PASS: adapter families documented", "item_id": "docs-rewrite-architecture-and-structure", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "docs-rewrite-architecture-and-structure", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Map current execution model, key classes, and data flow", "wid": "w1"}, {"evidence": null, "item_id": "docs-rewrite-architecture-and-structure", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rewrite architecture.md with current execution model", "wid": "w2"}, {"evidence": null, "item_id": "docs-rewrite-architecture-and-structure", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rewrite structure.md with complete directory map", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release High Priority Tasks (COMPLETED)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Critical\"\n- \"**Status**: Completed \u2705\"\n- \"**Description**: CLI documentation is severely inadequate, covering only basic usage while the CLI has extensive functionality including subcommands, platform-specific options, and advanced features\"\n- \"**Context**: Poor CLI documentation creates barriers to adoption and prevents users from utilizing powerful features\"\n- \"**Tasks**:\"\n- \"**Files Affected**:\"\n- \"**Success Metrics**:\"\n- \"**Impact**: CRITICAL - Poor CLI documentation prevents effective tool usage and adoption\"", "id": "document-cli-comprehensively", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Document CLI Comprehensively", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "place in docs/usage/ alongside existing guides", "dont": "create a new top-level docs section for this", "why": "it's a usage guide"}, {"INSTEAD": "describe observable behavior (version was reverted) and the fix, not the mechanism", "dont": "over-explain uv internals", "why": "most users don't care"}], "approach": "New file: docs/usage/driver-version-management.md\n\nStructure:\n 1. How BenchBox selects a driver version (no pin = use installed, pin = validate/install)\n 2. Why uv run may not respect manually-installed versions (uv.lock sync behavior)\n 3. Three reliable patterns:\n a. driver_version + driver_auto_install=true (BenchBox manages the install)\n b. uv run --with \"duckdb==X\" (inline uv override, no lock mutation)\n c. uv add \"duckdb==X\" --allow-prereleases (update the lock file)\n 4. Verifying the actual version used: result JSON platform.client_version and\n execution.driver_version_actual fields\n\nCross-link from: docs/reference/cli/configuration.md, docs/platforms/duckdb.md\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-23T00:00:00Z", "completed_pr": null, "created_at": "2026-02-23T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox's driver version selection logic is non-obvious to users who approach it from\nthe Python packaging angle (e.g., \"I installed version X, why didn't it use X?\"). The\nbehavior is correct but the interaction with uv.lock and uv run is undocumented.\n\nCurrent documentation gaps:\n 1. No explanation of how BenchBox selects a driver version when none is specified\n (it uses whatever is installed, as reported by importlib.metadata)\n 2. No explanation that `uv run` syncs the environment to uv.lock before launching,\n which can silently revert manually-installed versions\n 3. No dedicated \"How to test a specific driver version\" guide - examples exist in\n --help-topic examples but not in docs/\n 4. The docs/reference/cli/configuration.md driver_version/driver_auto_install entries\n describe the flags but not the underlying selection algorithm or the uv.lock\n interaction\n\nTarget: a clear, concise explanation in docs/concepts/ (or docs/usage/) that answers:\n - How does BenchBox decide which driver version to use?\n - Why might my manually-installed version not be respected?\n - How do I reliably benchmark a specific version (including dev/pre-release)?\n - How do I verify which version actually ran?\n\nThis should reference the result JSON's driver_version_actual field as the source of truth.", "id": "document-driver-version-selection-behavior", "preserves": ["No existing documentation files are removed or renamed (only additions and cross-links)", "docs/ build succeeds: uv run -- python -m sphinx docs/ docs/_build/html -b html"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "docs/platforms/duckdb.md"}, {"kind": "only_modify", "path_glob": "docs/reference/cli/configuration.md"}, {"kind": "only_modify", "path_glob": "docs/usage/"}], "state": "done", "title": "Document how BenchBox selects driver versions and how to reliably use a specific version", "verifications": [{"command": "wc -l docs/usage/driver-version-management.md", "description": "New guide file exists and is non-empty", "expected": "File exists with at least 50 lines", "item_id": "document-driver-version-selection-behavior", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m sphinx docs/ docs/_build/html -b html -q 2>&1 | tail -5", "description": "Docs build succeeds with new file (sphinx is in [dependency-groups].dev at pyproject.toml line 460 - available in standard dev install)", "expected": "Build succeeded with 0 errors", "item_id": "document-driver-version-selection-behavior", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep 'driver-version-management' docs/platforms/duckdb.md", "description": "duckdb.md contains cross-link to new guide", "expected": "At least one reference to driver-version-management", "item_id": "document-driver-version-selection-behavior", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "document-driver-version-selection-behavior", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write docs/usage/driver-version-management.md: version selection algorithm, uv.lock revert behavior, three reliable version-pinning patterns, and verifying via result JSON", "wid": "w1"}, {"evidence": null, "item_id": "document-driver-version-selection-behavior", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs/reference/cli/configuration.md driver_version and driver_auto_install entries to cross-link to the new guide", "wid": "w2"}, {"evidence": null, "item_id": "document-driver-version-selection-behavior", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a brief 'Testing a specific version' callout to docs/platforms/duckdb.md that links to the new guide", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use the same versions as pyproject.toml optional extras", "dont": "add version numbers that don't match the current optional-extra floors", "why": "it will confuse users"}, {"INSTEAD": "add only a short example + link", "dont": "duplicate the full driver-version-management guide content in each platform doc", "why": "it will go stale"}], "approach": "For each platform doc, locate the \"Platform Options\" or \"Configuration\" section and\nadd a new subsection immediately after the options table:\n\n### Testing a Specific [Platform Name] Version\n\n```bash\nbenchbox run --platform [platform] --benchmark tpch \\\n --platform-option driver_version=X.Y.Z \\\n --platform-option driver_auto_install=true\n```\n\nSee {ref}`driver-version-management` for the full guide, including why `uv run` may\nrevert a manually-installed version and how to work around it.\n\nUse the exact same wording as duckdb.md lines 79-88 for consistency, substituting\nthe correct platform name and a realistic version number (e.g., polars 1.36.1,\nclickhouse-connect 0.10.0, psycopg2-binary 2.9.11).\n\nFor driver-version-management.md, add one example per platform to the\n\"Pattern 1: driver_auto_install\" block - a brief comment line + the benchbox run\ncommand is sufficient (no new sections needed).\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-23T00:00:00Z", "completed_pr": null, "created_at": "2026-02-23T00:00:00Z", "deferrals": [], "deps": [], "description": "The DuckDB platform doc (docs/platforms/duckdb.md) was updated in the previous TODO\n(document-driver-version-selection-behavior) to include a \"Testing a Specific DuckDB Version\"\nsection with a concrete example and a cross-reference to the driver-version-management guide.\n\nThree other platform docs are missing equivalent coverage:\n\n docs/platforms/polars.md\n - No mention of driver versioning. After the polars extra is made optional, users\n will need to know how to install a specific polars version.\n\n docs/platforms/clickhouse-cloud.md\n - No driver versioning section. The platform uses clickhouse-connect; users may want\n to pin a specific connector release for reproducibility.\n\n docs/platforms/postgresql.md\n - Has basic install info but no guidance on driver version selection or the uv.lock\n revert issue.\n\nThe fix for each file is the same pattern used for duckdb.md:\n 1. Add a \"Testing a Specific [Platform] Driver Version\" subsection under Configuration\n or Platform Options, with a concrete --platform-option driver_version example\n 2. Add a {ref}`driver-version-management` cross-reference link\n\nAdditionally, the driver-version-management.md guide itself should reference these\nplatforms in its examples (currently it only shows duckdb and snowflake examples).", "id": "document-driver-versioning-for-polars-clickhouse-postgresql", "preserves": ["Existing content in all four docs files is not removed or restructured - only append new sections", "The {ref}`driver-version-management` cross-reference format matches what is used in duckdb.md", "driver-version-management.md remains the single authoritative guide - platform docs link to it, not duplicate it"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "docs/platforms/clickhouse-cloud.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/polars.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/postgresql.md"}, {"kind": "only_modify", "path_glob": "docs/usage/driver-version-management.md"}], "state": "done", "title": "Add driver version management callouts to Polars, ClickHouse Cloud, and PostgreSQL platform docs", "verifications": [{"command": "grep -l 'driver_version' docs/platforms/polars.md docs/platforms/clickhouse-cloud.md docs/platforms/postgresql.md", "description": "All three platform docs contain a driver_version example", "expected": "All three files listed (each contains 'driver_version')", "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -l 'driver-version-management' docs/platforms/polars.md docs/platforms/clickhouse-cloud.md docs/platforms/postgresql.md", "description": "All three platform docs contain the cross-reference link", "expected": "All three files listed", "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run sphinx-build -W docs docs/_build/html 2>&1 | grep -i 'warning\\|error' | head -20", "description": "Sphinx doc build produces no warnings for the new cross-references", "expected": "No warnings or errors related to the new cross-references", "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'Testing a Specific Polars Version' callout to docs/platforms/polars.md with driver_version + driver_auto_install example and {ref}`driver-version-management` link", "wid": "w1"}, {"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'Testing a Specific clickhouse-connect Version' callout to docs/platforms/clickhouse-cloud.md: same pattern as w1, note that the driver package is 'clickhouse-connect' not 'clickhouse'", "wid": "w2"}, {"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'Testing a Specific psycopg2 Version' callout to docs/platforms/postgresql.md: same pattern as w1, note that the driver package is 'psycopg2-binary'", "wid": "w3"}, {"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add polars and clickhouse-connect examples to docs/usage/driver-version-management.md Pattern 1 block alongside existing duckdb/snowflake examples", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "create a new top-level docs section for this", "instead": "place in docs/usage/ alongside existing guides", "why": "it's a usage guide"}, {"dont": "over-explain uv internals", "instead": "describe observable behavior (version was reverted) and the fix, not the mechanism", "why": "most users don't care"}], "approach": "New file: docs/usage/driver-version-management.md\n\nStructure:\n 1. How BenchBox selects a driver version (no pin = use installed, pin = validate/install)\n 2. Why uv run may not respect manually-installed versions (uv.lock sync behavior)\n 3. Three reliable patterns:\n a. driver_version + driver_auto_install=true (BenchBox manages the install)\n b. uv run --with \"duckdb==X\" (inline uv override, no lock mutation)\n c. uv add \"duckdb==X\" --allow-prereleases (update the lock file)\n 4. Verifying the actual version used: result JSON platform.client_version and\n execution.driver_version_actual fields\n\nCross-link from: docs/reference/cli/configuration.md, docs/platforms/duckdb.md\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-23T00:00:00Z", "completed_pr": null, "created_at": "2026-02-23T00:00:00Z", "deferrals": [], "deps": [], "description": "BenchBox's driver version selection logic is non-obvious to users who approach it from\nthe Python packaging angle (e.g., \"I installed version X, why didn't it use X?\"). The\nbehavior is correct but the interaction with uv.lock and uv run is undocumented.\n\nCurrent documentation gaps:\n 1. No explanation of how BenchBox selects a driver version when none is specified\n (it uses whatever is installed, as reported by importlib.metadata)\n 2. No explanation that `uv run` syncs the environment to uv.lock before launching,\n which can silently revert manually-installed versions\n 3. No dedicated \"How to test a specific driver version\" guide - examples exist in\n --help-topic examples but not in docs/\n 4. The docs/reference/cli/configuration.md driver_version/driver_auto_install entries\n describe the flags but not the underlying selection algorithm or the uv.lock\n interaction\n\nTarget: a clear, concise explanation in docs/concepts/ (or docs/usage/) that answers:\n - How does BenchBox decide which driver version to use?\n - Why might my manually-installed version not be respected?\n - How do I reliably benchmark a specific version (including dev/pre-release)?\n - How do I verify which version actually ran?\n\nThis should reference the result JSON's driver_version_actual field as the source of truth.", "id": "document-driver-version-selection-behavior", "preserves": ["No existing documentation files are removed or renamed (only additions and cross-links)", "docs/ build succeeds: uv run -- python -m sphinx docs/ docs/_build/html -b html"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "docs/platforms/duckdb.md"}, {"kind": "only_modify", "path_glob": "docs/reference/cli/configuration.md"}, {"kind": "only_modify", "path_glob": "docs/usage/"}], "state": "done", "title": "Document how BenchBox selects driver versions and how to reliably use a specific version", "verifications": [{"command": "wc -l docs/usage/driver-version-management.md", "description": "New guide file exists and is non-empty", "expected": "File exists with at least 50 lines", "item_id": "document-driver-version-selection-behavior", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m sphinx docs/ docs/_build/html -b html -q 2>&1 | tail -5", "description": "Docs build succeeds with new file (sphinx is in [dependency-groups].dev at pyproject.toml line 460 - available in standard dev install)", "expected": "Build succeeded with 0 errors", "item_id": "document-driver-version-selection-behavior", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep 'driver-version-management' docs/platforms/duckdb.md", "description": "duckdb.md contains cross-link to new guide", "expected": "At least one reference to driver-version-management", "item_id": "document-driver-version-selection-behavior", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "document-driver-version-selection-behavior", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write docs/usage/driver-version-management.md: version selection algorithm, uv.lock revert behavior, three reliable version-pinning patterns, and verifying via result JSON", "wid": "w1"}, {"evidence": null, "item_id": "document-driver-version-selection-behavior", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs/reference/cli/configuration.md driver_version and driver_auto_install entries to cross-link to the new guide", "wid": "w2"}, {"evidence": null, "item_id": "document-driver-version-selection-behavior", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a brief 'Testing a specific version' callout to docs/platforms/duckdb.md that links to the new guide", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add version numbers that don't match the current optional-extra floors", "instead": "use the same versions as pyproject.toml optional extras", "why": "it will confuse users"}, {"dont": "duplicate the full driver-version-management guide content in each platform doc", "instead": "add only a short example + link", "why": "it will go stale"}], "approach": "For each platform doc, locate the \"Platform Options\" or \"Configuration\" section and\nadd a new subsection immediately after the options table:\n\n### Testing a Specific [Platform Name] Version\n\n```bash\nbenchbox run --platform [platform] --benchmark tpch \\\n --platform-option driver_version=X.Y.Z \\\n --platform-option driver_auto_install=true\n```\n\nSee {ref}`driver-version-management` for the full guide, including why `uv run` may\nrevert a manually-installed version and how to work around it.\n\nUse the exact same wording as duckdb.md lines 79-88 for consistency, substituting\nthe correct platform name and a realistic version number (e.g., polars 1.36.1,\nclickhouse-connect 0.10.0, psycopg2-binary 2.9.11).\n\nFor driver-version-management.md, add one example per platform to the\n\"Pattern 1: driver_auto_install\" block - a brief comment line + the benchbox run\ncommand is sufficient (no new sections needed).\n", "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-23T00:00:00Z", "completed_pr": null, "created_at": "2026-02-23T00:00:00Z", "deferrals": [], "deps": [], "description": "The DuckDB platform doc (docs/platforms/duckdb.md) was updated in the previous TODO\n(document-driver-version-selection-behavior) to include a \"Testing a Specific DuckDB Version\"\nsection with a concrete example and a cross-reference to the driver-version-management guide.\n\nThree other platform docs are missing equivalent coverage:\n\n docs/platforms/polars.md\n - No mention of driver versioning. After the polars extra is made optional, users\n will need to know how to install a specific polars version.\n\n docs/platforms/clickhouse-cloud.md\n - No driver versioning section. The platform uses clickhouse-connect; users may want\n to pin a specific connector release for reproducibility.\n\n docs/platforms/postgresql.md\n - Has basic install info but no guidance on driver version selection or the uv.lock\n revert issue.\n\nThe fix for each file is the same pattern used for duckdb.md:\n 1. Add a \"Testing a Specific [Platform] Driver Version\" subsection under Configuration\n or Platform Options, with a concrete --platform-option driver_version example\n 2. Add a {ref}`driver-version-management` cross-reference link\n\nAdditionally, the driver-version-management.md guide itself should reference these\nplatforms in its examples (currently it only shows duckdb and snowflake examples).", "id": "document-driver-versioning-for-polars-clickhouse-postgresql", "preserves": ["Existing content in all four docs files is not removed or restructured - only append new sections", "The {ref}`driver-version-management` cross-reference format matches what is used in duckdb.md", "driver-version-management.md remains the single authoritative guide - platform docs link to it, not duplicate it"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "docs/platforms/clickhouse-cloud.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/polars.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/postgresql.md"}, {"kind": "only_modify", "path_glob": "docs/usage/driver-version-management.md"}], "state": "done", "title": "Add driver version management callouts to Polars, ClickHouse Cloud, and PostgreSQL platform docs", "verifications": [{"command": "grep -l 'driver_version' docs/platforms/polars.md docs/platforms/clickhouse-cloud.md docs/platforms/postgresql.md", "description": "All three platform docs contain a driver_version example", "expected": "All three files listed (each contains 'driver_version')", "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -l 'driver-version-management' docs/platforms/polars.md docs/platforms/clickhouse-cloud.md docs/platforms/postgresql.md", "description": "All three platform docs contain the cross-reference link", "expected": "All three files listed", "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run sphinx-build -W docs docs/_build/html 2>&1 | grep -i 'warning\\|error' | head -20", "description": "Sphinx doc build produces no warnings for the new cross-references", "expected": "No warnings or errors related to the new cross-references", "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'Testing a Specific Polars Version' callout to docs/platforms/polars.md with driver_version + driver_auto_install example and {ref}`driver-version-management` link", "wid": "w1"}, {"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'Testing a Specific clickhouse-connect Version' callout to docs/platforms/clickhouse-cloud.md: same pattern as w1, note that the driver package is 'clickhouse-connect' not 'clickhouse'", "wid": "w2"}, {"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'Testing a Specific psycopg2 Version' callout to docs/platforms/postgresql.md: same pattern as w1, note that the driver package is 'psycopg2-binary'", "wid": "w3"}, {"evidence": null, "item_id": "document-driver-versioning-for-polars-clickhouse-postgresql", "needs": ["w1", "w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add polars and clickhouse-connect examples to docs/usage/driver-version-management.md Pattern 1 block alongside existing duckdb/snowflake examples", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-15T00:00:00Z", "completed_pr": null, "created_at": "2026-01-15T00:00:00Z", "deferrals": [], "deps": ["patch-tpcds-source-stdout"], "description": "The TPC-DS stdout fix (commit 3911fd3a) applied patches to enable streaming\ndata generation but lacks documentation equivalent to what TPC-H has.\n\nTPC-H has a comprehensive PATCHES.md file documenting the stdout patch\nand crediting gregrahn/tpch-kit. TPC-DS needs the same treatment to:\n\n1. Document the patches applied for future maintainers\n2. Credit the original fix author (gregrahn/tpcds-kit commit 7992dbb)\n3. Help users understand the streaming capability exists\n\nThis was identified during code review of the stdout fix implementation.", "id": "document-stdout-patches", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Document TPC-DS stdout patches and credit tpcds-kit origin", "verifications": [], "work": [], "worktree": "tpcds-docs"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Documentation and Project Organization", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Successfully completed comprehensive 7-phase reorganization of BenchBox documentation, including 18 file moves/renames, 200+ link updates, creation of 7 navigational index files, broken link fixes, and naming standardization. All internal links verified working, zero broken references remaining.", "id": "documentation-structure-reorganization", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "\u2705 Documentation Structure Reorganization", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "invert or weaken a merged item's actual guarantees", "why": "only align the recorded text with the merged diff."}, {"INSTEAD": "(not specified)", "dont": "reopen or re-run the merged items; correct their recorded metadata only.", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-17T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up to holistic-review-followup-metadata-truth (#938), which corrected metric\nand scope truth defects (overstated success_metrics, scope_limit omissions, stale\nMakefile comments) for the specific DONE items the holistic review examined. That\nitem scoped itself to the holistic-review effort and deferred the broad sweep.\n\nDo the full audit: for every historical DONE item, verify its success_metrics,\nscope_limit.only_modify, and any counts against the item's actually-merged diff, and\ncorrect any that overstate, understate, or omit. Then systematize it so future\nclose-outs cannot silently drift \u2014 a lightweight check that flags a DONE item whose\nscope_limit does not cover its merged diff, or whose metric claims a state its diff\ndid not deliver. Metadata-only; no source-code behavior change.\n", "id": "done-item-metric-truthfulness-audit", "preserves": ["Metadata/documentation only \u2014 no source-code behavior change.", "Validator and TODO graph stay green after the DONE-YAML edits."], "prior_art": [{"concept": "{'ref': 'AGENTS.md (Default write-task close-out, guard+wiring rule added by #938)', 'note': 'House-rules home for any new close-out discipline this audit surfaces.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '_project/DONE/main/planning/holistic-review-followup-metadata-truth.yaml (#938)', 'note': 'The pattern to generalize: reword overstated metrics to the accurate scoped claim, add omitted scope_limit paths, refresh stale comments; metadata-only.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '_project/scripts/validate_todo.py', 'note': 'Where a recurrence guard (scope_limit-vs-diff / metric-vs-diff check) could live or be invoked from.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}], "state": "planning", "title": "Audit metric/scope truthfulness across all historical DONE items and add a recurrence guard", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py --all && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "Amended DONE YAMLs validate and the graph is intact", "expected": "all files valid; no cycles/dangling refs; sampled DONE metrics match their merged diffs", "item_id": "done-item-metric-truthfulness-audit", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "done-item-metric-truthfulness-audit", "needs": [], "notes": "For each DONE item, resolve the merging PR and compare the recorded metrics,\nscope_limit.only_modify, and any counts against the diff --name-only / content.\nRecord mismatches (overstated / understated / omitted).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Enumerate DONE items and diff each item's success_metrics / scope_limit / counts vs its merged PR diff", "wid": "w1"}, {"evidence": null, "item_id": "done-item-metric-truthfulness-audit", "needs": ["w1"], "notes": "Reword metrics to the accurate claim, add omitted scope paths, fix stale\ncounts/comments. No source or behavior change; re-run validate_todo on each.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Correct the mismatches (metadata-only), like #938 did for the holistic-review subset", "wid": "w2"}, {"evidence": null, "item_id": "done-item-metric-truthfulness-audit", "needs": ["w2"], "notes": "A lightweight script/guardrail (invoked from validate_todo or the todo review\naction) that catches scope_limit-vs-diff and metric-vs-diff drift on close-out,\nso this cannot silently recur.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add a recurrence guard that flags a DONE item whose scope_limit doesn't cover its diff", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "leave chdb in extras", "dont": "add chdb to core deps to compensate", "why": "that would defeat the optionality of ClickHouse"}, {"INSTEAD": "(not specified)", "dont": "remove chdb-core from extras as well", "why": "chdb depends on it transitively, and uv tree would re-add it; the only goal is dropping the redundant *core* declaration"}], "approach": "Tiny, surgical change. Remove one line from pyproject.toml, refresh\nuv.lock, run fast tests. The risk is essentially zero because chdb-core\nis reachable via the existing chdb declarations in the extras that\nactually need it. If a future code path wants to import chdb_core\ndirectly, it should declare chdb (which transitively brings chdb-core)\nrather than depending on the leaked-into-core declaration.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "drop-chdb-core-from-core-dependencies", "id": 151, "reason": "Out of scope; covered by the parent audit-imported-dependencies-for-elimination.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Parent audit-imported-dependencies-for-elimination (DONE 2026-04-15) scopes 'every declared dep... does it belong in the manifest', covering transitive-only core deps. The deferral explicitly delegated to that now-completed parent.", "summary": "Audit other transitive-only deps that may also be in core"}], "deps": [], "description": "`chdb-core>=26.1.0` is declared on line 49 of `pyproject.toml` as a CORE\nruntime dep. It has zero direct `import chdb_core` sites anywhere in\n`benchbox/`, `tests/`, `scripts/`, or `docs/`. `uv tree` shows that\n`chdb-core` is a dep of `chdb v4.1.6`, which itself lives in optional\nextras (`extras:all`, `extras:clickhouse-local`, `dep-group:dev`).\n\nNet effect today: every minimal install of benchbox pulls `chdb-core`\n(a sizeable C++-bundled package, ~tens of MB) even when the user did\nnot ask for ClickHouse local mode. Moving `chdb-core` out of core deps\ntrims the install footprint and keeps the optional/required boundary\nhonest.\n\nSource for this finding: `docs/development/dependency-inventory.md` (F1),\nproduced by `audit-imported-dependencies-for-elimination`.", "id": "drop-chdb-core-from-core-dependencies", "preserves": ["ClickHouse local extras (`extras:all`, `extras:clickhouse-local`, `dep-group:dev`) continue to install chdb (and therefore chdb-core)", "Fast test suite green after the change", "No new explicit `import chdb_core` is introduced"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "scripts/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Drop chdb-core from [project] dependencies (transitive of optional chdb)", "verifications": [{"command": "uv run -- python -c \"import tomllib, pathlib; deps = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['dependencies']; assert not any('chdb-core' in d for d in deps), 'chdb-core still in core deps: ' + str([d for d in deps if 'chdb-core' in d]); print('OK: chdb-core removed from core')\"", "description": "chdb-core no longer in [project] dependencies", "expected": "OK: chdb-core removed from core", "item_id": "drop-chdb-core-from-core-dependencies", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite passes", "expected": "no failures", "item_id": "drop-chdb-core-from-core-dependencies", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "drop-chdb-core-from-core-dependencies", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm zero direct `chdb_core` imports in shipped source: `grep -rn 'chdb_core' benchbox tests scripts docs`. Document the result in the commit message.", "wid": "w1"}, {"evidence": null, "item_id": "drop-chdb-core-from-core-dependencies", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Edit pyproject.toml: remove `chdb-core>=26.1.0` from `[project] dependencies`. Refresh uv.lock with `uv lock`.", "wid": "w2"}, {"evidence": null, "item_id": "drop-chdb-core-from-core-dependencies", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run fast suite to confirm no regressions. Verify chdb-using extras still pull chdb-core transitively: `uv tree --depth 2 | grep -B1 chdb-core`.", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "port first, remove second", "dont": "remove lxml without porting the test", "why": "`pytest tests/unit/core/tpcdi/test_etl_sources.py` would break"}], "approach": "Look-before-you-leap: read the test first, decide port vs keep based on\nwhat lxml feature it actually uses. Don't blindly remove and break.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "drop-lxml-dev-dep-or-port-test-to-stdlib", "id": 152, "reason": "Out of scope; covered by the parent audit.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Explicitly marked out of scope and covered by the parent dependency audit, which is done.", "summary": "Audit other dev deps with single-test usage"}], "deps": [], "description": "`lxml>=5.0.0` is declared in `[dependency-groups] dev`. The only import\nsite in the entire repo is `tests/unit/core/tpcdi/test_etl_sources.py:22`.\nTPC-DI ETL itself uses stdlib `xml.etree`, not lxml.\n\nTwo options:\n A. Port the single test to `xml.etree.ElementTree` and remove the dep.\n B. Keep the dep and add an inline comment in pyproject.toml explaining\n why this single test needs lxml-specific behavior.\n\nOption A is preferred unless the test relies on lxml-specific XPath /\nvalidation features that stdlib cannot replicate.\n\nSource for this finding: `docs/development/dependency-inventory.md` (F3),\nproduced by `audit-imported-dependencies-for-elimination`.", "id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "preserves": ["Fast suite green", "`benchbox/core/tpcdi/etl/` continues to use stdlib xml.etree (no lxml import added)", "`tests/unit/core/tpcdi/test_etl_sources.py` continues to pass"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_etl_sources.py"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Drop lxml dev dep (port single test to stdlib xml.etree)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_etl_sources.py -v", "description": "Test still passes", "expected": "1 or more passed", "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"import tomllib, pathlib; d = tomllib.loads(pathlib.Path('pyproject.toml').read_text()); all_specs = d['project']['dependencies'] + sum(d['project'].get('optional-dependencies', {}).values(), []) + sum((g for g in d.get('dependency-groups', {}).values()), []); has_lxml = any(isinstance(s, str) and 'lxml' in s for s in all_specs); print('LXML_PRESENT' if has_lxml else 'OK: lxml fully removed')\"", "description": "If removal path was taken, lxml not declared anywhere", "expected": "OK: lxml fully removed (or LXML_PRESENT if keep-with-rationale was chosen)", "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Read `tests/unit/core/tpcdi/test_etl_sources.py` and identify which lxml API surface (XPath / validation / parser) the test actually uses.", "wid": "w1"}, {"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide port-vs-keep based on w1 findings: stdlib-only API \u2192 port to xml.etree; lxml-specific features \u2192 keep dep with inline rationale.", "wid": "w2"}, {"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "If ported: remove `lxml>=5.0.0` from `[dependency-groups] dev` and refresh uv.lock.", "wid": "w3"}, {"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the affected test to confirm it still passes: `uv run -- python -m pytest tests/unit/core/tpcdi/test_etl_sources.py -v`.", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "fix only client.py and leave the latent imports", "why": "the user requires ALL blind spots addressed; w3 closes the next-failure vector."}, {"INSTEAD": "(not specified)", "dont": "keep pandas core and defer to v0.3.2", "why": "v0.3.1 is unreleased, so the correct minimal-core fix lands now at zero extra release cost."}, {"INSTEAD": "(not specified)", "dont": "keep the string-assert test (test_release_infrastructure.py:99-107)", "why": "it tests the manifest text, not the import contract, and will both block w6 and fail to catch the real defect."}, {"INSTEAD": "(not specified)", "dont": "remove pandas from core (w6) before the core-only-minus-pandas gate (w5) is green", "why": "that gate is the only thing preventing a silent re-regression of the v0.3.0 failure."}], "approach": "Land w1 first (lazy import) \u2014 it alone restores the minimal import surface and\nde-risks everything downstream. Then w2-w5 (structural fix, latent imports,\nreal contract test, strengthened gate) so removing pandas in w6 is safe and\nre-regression-proof. w7 is a documentation/decision pass aligning with\ndependency-inventory F1/F2. The whole sequence must land on develop BEFORE\n`make release-cut VERSION=0.3.1`, so v0.3.1 ships the minimal core, not the\npandas patch. Functional code fixes (w1-w3, w6) belong in a feature/fix PR;\ntest/gate changes (w4, w5) can ride along or in a sibling PR.\n", "blocked_reason": null, "category": "Dependency Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-30T00:00:00Z", "deferrals": [], "deps": [], "description": "PR #681 fixed the v0.3.0 clean-install failure (`import benchbox` raised\nImportError without pandas) by adding `pandas>=2.0.0` to\n`[project.dependencies]` (`pyproject.toml:50-52`). That treated the symptom,\nnot the cause.\n\nRoot cause: `import benchbox` eagerly pulls an engine-specific module that\nimports pandas at module top level. The reachable chain is\n`benchbox/__init__.py:28` -> `platforms/__init__.py:286` ->\n`adapter_factory.py:17` -> `clickhouse/__init__.py:5` -> `adapter.py:13` ->\n`setup.py:9` -> `client.py:11 import pandas as pd`. `platforms/__init__.py`\nalready lazy-loads `ClickHouseAdapter` via `__getattr__`, but\n`adapter_factory.py:17` reaches through the package `__init__`, defeating that\nlazy design. `client.py` uses pandas at exactly one runtime site\n(`client.py:144`, an empty-result `pd.DataFrame()`), so it is trivially\nlazy-able.\n\nConsequences of the patch: every install \u2014 including a DuckDB-only SQL run \u2014\nnow pays ~49 MB (pandas + pytz, tzdata, python-dateutil, six) for a library it\nmay never touch. pandas is simultaneously a hard core dep AND an optional\nextra (`pandas`, `dataframe-pandas`, `tpcdi`, `modin`, `dask`, `all`), an\nincoherent manifest. This contradicts the project's own documented thin-core\nphilosophy: `docs/development/dependency-inventory.md:159` classifies pandas\nas \"DF/BM\" (DataFrame/Benchmark-specific, not core), and finding F1\n(`:209-227`) recommends removing engine-specific libs from core for exactly\nthis reason.\n\nv0.3.1 is UNRELEASED, so the correct fix lands NOW: make the import surface\nminimal and drop pandas back to extras before the cut, rather than shipping\nthe heavy patch and deferring. Everyone who needs pandas still gets it via the\nextras and transitively via `chdb` (which requires pandas).\n\nThis TODO is a gate on `release-recovery-v0-3-1` w3 (`make release-cut\nVERSION=0.3.1`): the cut should carry the minimal-surface fix, not the patch.", "id": "drop-pandas-from-core-dependencies", "preserves": ["DataFrame mode, TPC-DI, and ClickHouse-local (chdb) runs must keep working \u2014 they receive pandas via their extras.", "Do not re-tag, force-push, or alter the immutable v0.3.0 tag/PyPI artifact.", "Do not weaken the release-required context or the core-only smoke gate to land this.", "`import benchbox` MUST succeed on a clean core install at every step; w1 satisfies this before pandas leaves core in w6."], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/nightly.yml"}, {"kind": "only_modify", "path_glob": ".github/workflows/release.yml"}, {"kind": "only_modify", "path_glob": ".github/workflows/test.yml"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder_synthetic/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/dataframe_queries/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/adapter_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/client.py"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_infrastructure.py"}], "state": "done", "title": "Drop pandas from core dependencies for v0.3.1 by fixing the eager import surface", "verifications": [{"command": "uv run -- python -c 'import sys; sys.modules[\"pandas\"]=None; import benchbox; print(benchbox.__version__)'", "description": "benchbox imports with pandas made unavailable in-process", "expected": "prints the version without ImportError", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import benchbox, sys; assert \"pandas\" not in sys.modules, sorted(m for m in sys.modules if \"pandas\" in m)'", "description": "No top-level pandas import remains reachable from import benchbox", "expected": "exit 0 (pandas not imported)", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv export --no-dev --no-emit-project 2>/dev/null | grep -c '^pandas==' || true", "description": "pandas is not in the resolved core dependency set", "expected": "0", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv pip install --dry-run '.[tpcdi]' >/dev/null && echo ok", "description": "pandas-needing extras still resolve", "expected": "ok", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run _project/scripts/todo_cli.py check-graph && uv run _project/scripts/validate_todo.py --all", "description": "TODO graph remains valid", "expected": "exit 0", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": [], "notes": "Bind the evidence this TODO rests on before changing code, and capture the\nBEFORE baseline that the size success-metric is measured against. Re-run and\nrecord a compact summary (command + PASS/FAIL + key counts) \u2014 not raw logs \u2014\nin this work unit's notes and the PR body:\n - Import surface still pulls pandas (expect pandas present today):\n `uv run -- python -c 'import benchbox, sys; print(\"pandas\" in sys.modules)'`\n -> expect True (this is the defect; expect False after w1).\n - Reachable chain unchanged at the cited lines:\n `clickhouse/client.py:11` import, `:144` sole `pd.` use;\n `adapter_factory.py` -> `clickhouse/deployment_mode`;\n `clickhouse/__init__.py` eager `.adapter/.client/.setup` re-export.\n - Core dependency-size baseline:\n `uv export --no-dev --no-emit-project | grep -cE '^(pandas|pytz|tzdata|python-dateutil|six)=='`\n and record installed size of those wheels. Pin the measured number so\n the \"~49 MB\" / \"drops ~49 MB\" claims are re-runnable, not asserted.\nIf any cited line has drifted, update the dependent work units before\nstarting.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the import chain and measure the core dependency-size baseline", "wid": "w0"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w0"], "notes": "Move `import pandas as pd` out of module scope in\n`benchbox/platforms/clickhouse/client.py:11` into the function that uses it\n(`client.py:144`). Mirror the in-function import pattern already used by\n`benchbox/core/tpch/dataframe_queries.py:1064`. This single change removes\npandas from the entire `import benchbox` surface and is sufficient to make\na clean core install import without pandas present.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Lazy-ize the one reachable eager pandas import in the ClickHouse client", "wid": "w1"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "Structural root cause. `adapter_factory.py:17` imports the pure helper\n`clickhouse/deployment_mode.py` but executing `clickhouse/__init__.py:5-11`\neagerly re-exports the heavy `adapter`/`setup`/`client` modules. Either\n(a) make `clickhouse/__init__.py` lazy via module `__getattr__`, or\n(b) have `adapter_factory.py:17` import `deployment_mode` by full path\nwithout triggering the package `__init__`. Without this, the next engine\nmodule added to the chain re-introduces the #681 failure.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Break the ClickHouse package-init eager re-export that defeats lazy loading", "wid": "w2"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "BLIND SPOT: not reachable on the import surface today, but a future wiring\nof these benchmarks recreates the exact #681 failure. Convert top-level\n`import pandas` to in-function in:\n - benchbox/core/joinorder/dataframe_queries.py:33\n - benchbox/core/ssb/dataframe_queries/queries.py:8\n - benchbox/core/joinorder_synthetic/dataframe_queries.py:17\nConfirm none are reached from `import benchbox` before and after.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Convert the latent top-level pandas imports to in-function imports", "wid": "w3"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "BLIND SPOT: `tests/unit/test_release_infrastructure.py:99-107` asserts the\nSTRING `pandas>=` is present in pyproject \u2014 it locks in the #681 patch and\nwill fail once pandas leaves core (w6). Replace it with a test that imports\n`benchbox` in a pandas-blocked / minimal-deps environment (e.g. a\n`sys.modules['pandas'] = None` guard or subprocess with pandas absent) and\nasserts success. Test the CONTRACT (import works on minimal deps), not the\nmanifest text.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace the wrong-contract pandas test with a minimal-deps import test", "wid": "w4"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w4"], "notes": "BLIND SPOT: a core-only clean-install smoke test exists\n(`.github/workflows/nightly.yml:235`, `test.yml:302`:\n`uv run --isolated --no-project --with -- python -c \"import benchbox\"`),\nbut it now PASSES because pandas is core \u2014 it no longer proves the surface\nis minimal, which is why 0.3.0 shipped broken. Strengthen it to install the\nCORE deps WITHOUT pandas (or block pandas) and assert `import benchbox`\nsucceeds. Confirm this gate runs on the release path (`release.yml:181,188`),\nnot only nightly/test, so a broken import surface blocks the tag.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the core-only smoke gate prove a minimal import surface and gate the release", "wid": "w5"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Only after w1-w5 are green. Remove `pandas>=2.0.0` from\n`[project.dependencies]` (`pyproject.toml:52`). It remains in the `tpcdi`,\n`pandas`, `dataframe-pandas`, `modin`, `dask`, and `all` extras, and arrives\ntransitively via `chdb`. Verify with `uv` that a core-only resolve excludes\npandas and that each pandas-needing extra still resolves. Resolve the\ncore-dep/extra contradiction.\nBLIND SPOT (pin): the `pandas>=2.0.0` floor was copied, not chosen (it\npredates #681 across the extras). Decide deliberately: keep `>=2.0.0` or\ntighten/loosen with a one-line rationale; do not leave it as cargo-culted.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove pandas from core dependencies and reconcile the extras/pin", "wid": "w6"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "`pyarrow` and `numpy` are reached on `import benchbox` and are core. Verify\nthat is intentional (results/Arrow handling is plausibly genuinely core) and\nrecord the decision. Cross-reference `dependency-inventory.md` F1\n(`chdb-core` flagged for removal from core) and F2 (`jsonschema`) so this\nwork aligns with the documented thin-core direction rather than fixing only\npandas in isolation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit remaining eager heavy imports on the import surface", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "break the MCP tool registration", "why": "generate_chart must still work; just remove the HTML branch and simplify the response model"}, {"INSTEAD": "(not specified)", "dont": "delete exporters.py entirely", "why": "export_ascii() and render_ascii_chart() serve the ASCII path; only remove export_figure() and Plotly imports"}, {"INSTEAD": "(not specified)", "dont": "delete result_plotter.py entirely", "why": "it contains the data loading and normalization logic (from_sources, NormalizedResult) that ASCII charts depend on"}, {"INSTEAD": "(not specified)", "dont": "leave 'plotly' in pyproject.toml extras", "why": "orphaned optional deps confuse users and bloat installs"}, {"INSTEAD": "(not specified)", "dont": "remove the chart template system (templates.py)", "why": "templates are format-agnostic and drive which charts to generate, not how to render them"}], "approach": "Work outside-in: start by deleting the leaf Plotly files (charts.py, styles.py, dependencies.py)\nthat nothing else imports, then strip Plotly code from the shared files (result_plotter.py,\nexporters.py, visualization.py) that have both Plotly and ASCII paths. After removal, unify the\ndata models by promoting ASCII models as canonical and refactoring ResultPlotter to produce them\ndirectly. Finally simplify the MCP/CLI interface by removing format branching.\n\nKey files to reference for understanding the current dual-path architecture:\n- exporters.py:render_ascii_chart() - shows the conversion between Plotly and ASCII models\n- mcp/tools/visualization.py - shows the format branching (HTML vs ASCII)\n- charts.py - the Plotly chart classes that will be deleted entirely\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": [], "description": "Remove Plotly and the entire HTML chart rendering pipeline from BenchBox. The ASCII charting\nsystem (2,255 lines, 6 chart types, zero dependencies) is the actively-used visualization\npath for CLI, MCP, and CI. The Plotly/HTML path (~1,750 lines source + ~1,260 lines tests)\ngenerates static .html files that nobody opens - there is no browser integration, no HTTP\nserver, no CI usage, and no documented user workflow that depends on HTML output.\n\nRemoving Plotly eliminates:\n- Two parallel rendering pipelines with incompatible data models\n- ~2,300 lines of source + test code (37% reduction in visualization module)\n- 46 MB optional dependency weight (plotly 43 MB + narwhals 3.3 MB)\n- The dual data model problem (BarDatum vs BarData, etc.) that forces every new chart type\n to be implemented twice\n- Three test files that use pytest.importorskip(\"plotly\") and may skip silently\n\nThis clears the path for the ASCII charting enhancement (enhance-ascii-charting.yaml) by\nestablishing a single canonical data model and removing the need for conversion layers.\n\nDecision rationale: Evaluated in depth - HTML charts are the default format but disconnected\nfrom real usage. MCP returns file paths for HTML (useless to LLMs) vs inline content for ASCII\n(immediately useful). CLI generates files users must manually locate and open. No blog workflow\nreferences chart generation. The feature is vestigial.", "id": "drop-plotly-html-charts", "preserves": ["ASCIIChartBase interface and render() -> str contract must not change", "All 6 ASCII chart types (bar, histogram, heatmap, box, line, scatter) must remain fully functional", "CLI benchbox visualize command must continue to work (ASCII output)", "Chart templates (default, flagship, head_to_head, trends, cost_optimization) must work with ASCII", "MCP generate_chart tool must continue to work (ASCII format)", "MCP suggest_charts tool must continue to work (format-agnostic logic)", "NormalizedResult and NormalizedQuery dataclasses must be preserved (shared data loading)", "ResultPlotter.from_sources() must continue loading and normalizing result JSON files", "tests/unit/visualization/ascii/test_ascii_charts.py (934 lines) must pass unchanged"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "do_not_modify", "path_glob": "tests/unit/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/visualize.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/comparison/plotter.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/benchmark_suite.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/charts.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/dependencies.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/result_plotter.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/styles.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "docs/visualization/chart-generation-guide.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/integration/visualization/test_visualization_export.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_visualization_tools.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_charts.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_result_plotter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_styles.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_visualize_cli.py"}], "state": "done", "title": "Drop Plotly dependency and HTML chart rendering - go ASCII-only", "verifications": [{"command": "uv run -- python -m pytest tests/unit/visualization/ascii/ -v", "description": "All ASCII chart tests pass unchanged", "expected": "all tests pass, 0 failures", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/mcp/test_visualization_tools.py -v", "description": "MCP visualization tool tests pass", "expected": "all tests pass (some removed, remaining pass)", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -r 'import plotly\\|from plotly\\|require_plotly' benchbox/ --include='*.py'", "description": "No references to plotly remain in source code", "expected": "no output (zero matches)", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -c 'plotly' pyproject.toml", "description": "plotly is not in any pyproject.toml dependency group", "expected": "0", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -x -q", "description": "Full test suite passes", "expected": "all tests pass, 0 failures", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run ruff check benchbox/core/visualization/", "description": "Lint and type check clean", "expected": "no lint errors", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 6}], "work": [], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "invert or weaken a merged item's actual guarantees", "instead": "(not specified)", "why": "only align the recorded text with the merged diff."}, {"dont": "reopen or re-run the merged items; correct their recorded metadata only.", "instead": "(not specified)", "why": "(unstated)"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-17T00:00:00Z", "deferrals": [], "deps": [], "description": "Follow-up to holistic-review-followup-metadata-truth (#938), which corrected metric\nand scope truth defects (overstated success_metrics, scope_limit omissions, stale\nMakefile comments) for the specific DONE items the holistic review examined. That\nitem scoped itself to the holistic-review effort and deferred the broad sweep.\n\nDo the full audit: for every historical DONE item, verify its success_metrics,\nscope_limit.only_modify, and any counts against the item's actually-merged diff, and\ncorrect any that overstate, understate, or omit. Then systematize it so future\nclose-outs cannot silently drift \u2014 a lightweight check that flags a DONE item whose\nscope_limit does not cover its merged diff, or whose metric claims a state its diff\ndid not deliver. Metadata-only; no source-code behavior change.\n", "id": "done-item-metric-truthfulness-audit", "preserves": ["Metadata/documentation only \u2014 no source-code behavior change.", "Validator and TODO graph stay green after the DONE-YAML edits."], "prior_art": [{"concept": "{'ref': 'AGENTS.md (Default write-task close-out, guard+wiring rule added by #938)', 'note': 'House-rules home for any new close-out discipline this audit surfaces.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '_project/DONE/main/planning/holistic-review-followup-metadata-truth.yaml (#938)', 'note': 'The pattern to generalize: reword overstated metrics to the accurate scoped claim, add omitted scope_limit paths, refresh stale comments; metadata-only.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': '_project/scripts/validate_todo.py', 'note': 'Where a recurrence guard (scope_limit-vs-diff / metric-vs-diff check) could live or be invoked from.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "AGENTS.md"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/scripts/"}], "state": "planning", "title": "Audit metric/scope truthfulness across all historical DONE items and add a recurrence guard", "verifications": [{"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py --all && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "Amended DONE YAMLs validate and the graph is intact", "expected": "all files valid; no cycles/dangling refs; sampled DONE metrics match their merged diffs", "item_id": "done-item-metric-truthfulness-audit", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "done-item-metric-truthfulness-audit", "needs": [], "notes": "For each DONE item, resolve the merging PR and compare the recorded metrics,\nscope_limit.only_modify, and any counts against the diff --name-only / content.\nRecord mismatches (overstated / understated / omitted).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Enumerate DONE items and diff each item's success_metrics / scope_limit / counts vs its merged PR diff", "wid": "w1"}, {"evidence": null, "item_id": "done-item-metric-truthfulness-audit", "needs": ["w1"], "notes": "Reword metrics to the accurate claim, add omitted scope paths, fix stale\ncounts/comments. No source or behavior change; re-run validate_todo on each.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Correct the mismatches (metadata-only), like #938 did for the holistic-review subset", "wid": "w2"}, {"evidence": null, "item_id": "done-item-metric-truthfulness-audit", "needs": ["w2"], "notes": "A lightweight script/guardrail (invoked from validate_todo or the todo review\naction) that catches scope_limit-vs-diff and metric-vs-diff drift on close-out,\nso this cannot silently recur.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add a recurrence guard that flags a DONE item whose scope_limit doesn't cover its diff", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add chdb to core deps to compensate", "instead": "leave chdb in extras", "why": "that would defeat the optionality of ClickHouse"}, {"dont": "remove chdb-core from extras as well", "instead": "(not specified)", "why": "chdb depends on it transitively, and uv tree would re-add it; the only goal is dropping the redundant *core* declaration"}], "approach": "Tiny, surgical change. Remove one line from pyproject.toml, refresh\nuv.lock, run fast tests. The risk is essentially zero because chdb-core\nis reachable via the existing chdb declarations in the extras that\nactually need it. If a future code path wants to import chdb_core\ndirectly, it should declare chdb (which transitively brings chdb-core)\nrather than depending on the leaked-into-core declaration.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "drop-chdb-core-from-core-dependencies", "id": 151, "reason": "Out of scope; covered by the parent audit-imported-dependencies-for-elimination.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Parent audit-imported-dependencies-for-elimination (DONE 2026-04-15) scopes 'every declared dep... does it belong in the manifest', covering transitive-only core deps. The deferral explicitly delegated to that now-completed parent.", "summary": "Audit other transitive-only deps that may also be in core"}], "deps": [], "description": "`chdb-core>=26.1.0` is declared on line 49 of `pyproject.toml` as a CORE\nruntime dep. It has zero direct `import chdb_core` sites anywhere in\n`benchbox/`, `tests/`, `scripts/`, or `docs/`. `uv tree` shows that\n`chdb-core` is a dep of `chdb v4.1.6`, which itself lives in optional\nextras (`extras:all`, `extras:clickhouse-local`, `dep-group:dev`).\n\nNet effect today: every minimal install of benchbox pulls `chdb-core`\n(a sizeable C++-bundled package, ~tens of MB) even when the user did\nnot ask for ClickHouse local mode. Moving `chdb-core` out of core deps\ntrims the install footprint and keeps the optional/required boundary\nhonest.\n\nSource for this finding: `docs/development/dependency-inventory.md` (F1),\nproduced by `audit-imported-dependencies-for-elimination`.", "id": "drop-chdb-core-from-core-dependencies", "preserves": ["ClickHouse local extras (`extras:all`, `extras:clickhouse-local`, `dep-group:dev`) continue to install chdb (and therefore chdb-core)", "Fast test suite green after the change", "No new explicit `import chdb_core` is introduced"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "scripts/"}, {"kind": "do_not_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Drop chdb-core from [project] dependencies (transitive of optional chdb)", "verifications": [{"command": "uv run -- python -c \"import tomllib, pathlib; deps = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['dependencies']; assert not any('chdb-core' in d for d in deps), 'chdb-core still in core deps: ' + str([d for d in deps if 'chdb-core' in d]); print('OK: chdb-core removed from core')\"", "description": "chdb-core no longer in [project] dependencies", "expected": "OK: chdb-core removed from core", "item_id": "drop-chdb-core-from-core-dependencies", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite passes", "expected": "no failures", "item_id": "drop-chdb-core-from-core-dependencies", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "drop-chdb-core-from-core-dependencies", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm zero direct `chdb_core` imports in shipped source: `grep -rn 'chdb_core' benchbox tests scripts docs`. Document the result in the commit message.", "wid": "w1"}, {"evidence": null, "item_id": "drop-chdb-core-from-core-dependencies", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Edit pyproject.toml: remove `chdb-core>=26.1.0` from `[project] dependencies`. Refresh uv.lock with `uv lock`.", "wid": "w2"}, {"evidence": null, "item_id": "drop-chdb-core-from-core-dependencies", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run fast suite to confirm no regressions. Verify chdb-using extras still pull chdb-core transitively: `uv tree --depth 2 | grep -B1 chdb-core`.", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "remove lxml without porting the test", "instead": "port first, remove second", "why": "`pytest tests/unit/core/tpcdi/test_etl_sources.py` would break"}], "approach": "Look-before-you-leap: read the test first, decide port vs keep based on\nwhat lxml feature it actually uses. Don't blindly remove and break.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "drop-lxml-dev-dep-or-port-test-to-stdlib", "id": 152, "reason": "Out of scope; covered by the parent audit.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Explicitly marked out of scope and covered by the parent dependency audit, which is done.", "summary": "Audit other dev deps with single-test usage"}], "deps": [], "description": "`lxml>=5.0.0` is declared in `[dependency-groups] dev`. The only import\nsite in the entire repo is `tests/unit/core/tpcdi/test_etl_sources.py:22`.\nTPC-DI ETL itself uses stdlib `xml.etree`, not lxml.\n\nTwo options:\n A. Port the single test to `xml.etree.ElementTree` and remove the dep.\n B. Keep the dep and add an inline comment in pyproject.toml explaining\n why this single test needs lxml-specific behavior.\n\nOption A is preferred unless the test relies on lxml-specific XPath /\nvalidation features that stdlib cannot replicate.\n\nSource for this finding: `docs/development/dependency-inventory.md` (F3),\nproduced by `audit-imported-dependencies-for-elimination`.", "id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "preserves": ["Fast suite green", "`benchbox/core/tpcdi/etl/` continues to use stdlib xml.etree (no lxml import added)", "`tests/unit/core/tpcdi/test_etl_sources.py` continues to pass"], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_etl_sources.py"}, {"kind": "only_modify", "path_glob": "uv.lock"}], "state": "done", "title": "Drop lxml dev dep (port single test to stdlib xml.etree)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/tpcdi/test_etl_sources.py -v", "description": "Test still passes", "expected": "1 or more passed", "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c \"import tomllib, pathlib; d = tomllib.loads(pathlib.Path('pyproject.toml').read_text()); all_specs = d['project']['dependencies'] + sum(d['project'].get('optional-dependencies', {}).values(), []) + sum((g for g in d.get('dependency-groups', {}).values()), []); has_lxml = any(isinstance(s, str) and 'lxml' in s for s in all_specs); print('LXML_PRESENT' if has_lxml else 'OK: lxml fully removed')\"", "description": "If removal path was taken, lxml not declared anywhere", "expected": "OK: lxml fully removed (or LXML_PRESENT if keep-with-rationale was chosen)", "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Read `tests/unit/core/tpcdi/test_etl_sources.py` and identify which lxml API surface (XPath / validation / parser) the test actually uses.", "wid": "w1"}, {"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide port-vs-keep based on w1 findings: stdlib-only API \u2192 port to xml.etree; lxml-specific features \u2192 keep dep with inline rationale.", "wid": "w2"}, {"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "If ported: remove `lxml>=5.0.0` from `[dependency-groups] dev` and refresh uv.lock.", "wid": "w3"}, {"evidence": null, "item_id": "drop-lxml-dev-dep-or-port-test-to-stdlib", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the affected test to confirm it still passes: `uv run -- python -m pytest tests/unit/core/tpcdi/test_etl_sources.py -v`.", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fix only client.py and leave the latent imports", "instead": "(not specified)", "why": "the user requires ALL blind spots addressed; w3 closes the next-failure vector."}, {"dont": "keep pandas core and defer to v0.3.2", "instead": "(not specified)", "why": "v0.3.1 is unreleased, so the correct minimal-core fix lands now at zero extra release cost."}, {"dont": "keep the string-assert test (test_release_infrastructure.py:99-107)", "instead": "(not specified)", "why": "it tests the manifest text, not the import contract, and will both block w6 and fail to catch the real defect."}, {"dont": "remove pandas from core (w6) before the core-only-minus-pandas gate (w5) is green", "instead": "(not specified)", "why": "that gate is the only thing preventing a silent re-regression of the v0.3.0 failure."}], "approach": "Land w1 first (lazy import) \u2014 it alone restores the minimal import surface and\nde-risks everything downstream. Then w2-w5 (structural fix, latent imports,\nreal contract test, strengthened gate) so removing pandas in w6 is safe and\nre-regression-proof. w7 is a documentation/decision pass aligning with\ndependency-inventory F1/F2. The whole sequence must land on develop BEFORE\n`make release-cut VERSION=0.3.1`, so v0.3.1 ships the minimal core, not the\npandas patch. Functional code fixes (w1-w3, w6) belong in a feature/fix PR;\ntest/gate changes (w4, w5) can ride along or in a sibling PR.\n", "blocked_reason": null, "category": "Dependency Management", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-30T00:00:00Z", "deferrals": [], "deps": [], "description": "PR #681 fixed the v0.3.0 clean-install failure (`import benchbox` raised\nImportError without pandas) by adding `pandas>=2.0.0` to\n`[project.dependencies]` (`pyproject.toml:50-52`). That treated the symptom,\nnot the cause.\n\nRoot cause: `import benchbox` eagerly pulls an engine-specific module that\nimports pandas at module top level. The reachable chain is\n`benchbox/__init__.py:28` -> `platforms/__init__.py:286` ->\n`adapter_factory.py:17` -> `clickhouse/__init__.py:5` -> `adapter.py:13` ->\n`setup.py:9` -> `client.py:11 import pandas as pd`. `platforms/__init__.py`\nalready lazy-loads `ClickHouseAdapter` via `__getattr__`, but\n`adapter_factory.py:17` reaches through the package `__init__`, defeating that\nlazy design. `client.py` uses pandas at exactly one runtime site\n(`client.py:144`, an empty-result `pd.DataFrame()`), so it is trivially\nlazy-able.\n\nConsequences of the patch: every install \u2014 including a DuckDB-only SQL run \u2014\nnow pays ~49 MB (pandas + pytz, tzdata, python-dateutil, six) for a library it\nmay never touch. pandas is simultaneously a hard core dep AND an optional\nextra (`pandas`, `dataframe-pandas`, `tpcdi`, `modin`, `dask`, `all`), an\nincoherent manifest. This contradicts the project's own documented thin-core\nphilosophy: `docs/development/dependency-inventory.md:159` classifies pandas\nas \"DF/BM\" (DataFrame/Benchmark-specific, not core), and finding F1\n(`:209-227`) recommends removing engine-specific libs from core for exactly\nthis reason.\n\nv0.3.1 is UNRELEASED, so the correct fix lands NOW: make the import surface\nminimal and drop pandas back to extras before the cut, rather than shipping\nthe heavy patch and deferring. Everyone who needs pandas still gets it via the\nextras and transitively via `chdb` (which requires pandas).\n\nThis TODO is a gate on `release-recovery-v0-3-1` w3 (`make release-cut\nVERSION=0.3.1`): the cut should carry the minimal-surface fix, not the patch.", "id": "drop-pandas-from-core-dependencies", "preserves": ["DataFrame mode, TPC-DI, and ClickHouse-local (chdb) runs must keep working \u2014 they receive pandas via their extras.", "Do not re-tag, force-push, or alter the immutable v0.3.0 tag/PyPI artifact.", "Do not weaken the release-required context or the core-only smoke gate to land this.", "`import benchbox` MUST succeed on a clean core install at every step; w1 satisfies this before pandas leaves core in w6."], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/nightly.yml"}, {"kind": "only_modify", "path_glob": ".github/workflows/release.yml"}, {"kind": "only_modify", "path_glob": ".github/workflows/test.yml"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/joinorder_synthetic/dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/ssb/dataframe_queries/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/adapter_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/clickhouse/client.py"}, {"kind": "only_modify", "path_glob": "docs/development/dependency-inventory.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_infrastructure.py"}], "state": "done", "title": "Drop pandas from core dependencies for v0.3.1 by fixing the eager import surface", "verifications": [{"command": "uv run -- python -c 'import sys; sys.modules[\"pandas\"]=None; import benchbox; print(benchbox.__version__)'", "description": "benchbox imports with pandas made unavailable in-process", "expected": "prints the version without ImportError", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import benchbox, sys; assert \"pandas\" not in sys.modules, sorted(m for m in sys.modules if \"pandas\" in m)'", "description": "No top-level pandas import remains reachable from import benchbox", "expected": "exit 0 (pandas not imported)", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv export --no-dev --no-emit-project 2>/dev/null | grep -c '^pandas==' || true", "description": "pandas is not in the resolved core dependency set", "expected": "0", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv pip install --dry-run '.[tpcdi]' >/dev/null && echo ok", "description": "pandas-needing extras still resolve", "expected": "ok", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run _project/scripts/todo_cli.py check-graph && uv run _project/scripts/validate_todo.py --all", "description": "TODO graph remains valid", "expected": "exit 0", "item_id": "drop-pandas-from-core-dependencies", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": [], "notes": "Bind the evidence this TODO rests on before changing code, and capture the\nBEFORE baseline that the size success-metric is measured against. Re-run and\nrecord a compact summary (command + PASS/FAIL + key counts) \u2014 not raw logs \u2014\nin this work unit's notes and the PR body:\n - Import surface still pulls pandas (expect pandas present today):\n `uv run -- python -c 'import benchbox, sys; print(\"pandas\" in sys.modules)'`\n -> expect True (this is the defect; expect False after w1).\n - Reachable chain unchanged at the cited lines:\n `clickhouse/client.py:11` import, `:144` sole `pd.` use;\n `adapter_factory.py` -> `clickhouse/deployment_mode`;\n `clickhouse/__init__.py` eager `.adapter/.client/.setup` re-export.\n - Core dependency-size baseline:\n `uv export --no-dev --no-emit-project | grep -cE '^(pandas|pytz|tzdata|python-dateutil|six)=='`\n and record installed size of those wheels. Pin the measured number so\n the \"~49 MB\" / \"drops ~49 MB\" claims are re-runnable, not asserted.\nIf any cited line has drifted, update the dependent work units before\nstarting.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate the import chain and measure the core dependency-size baseline", "wid": "w0"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w0"], "notes": "Move `import pandas as pd` out of module scope in\n`benchbox/platforms/clickhouse/client.py:11` into the function that uses it\n(`client.py:144`). Mirror the in-function import pattern already used by\n`benchbox/core/tpch/dataframe_queries.py:1064`. This single change removes\npandas from the entire `import benchbox` surface and is sufficient to make\na clean core install import without pandas present.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Lazy-ize the one reachable eager pandas import in the ClickHouse client", "wid": "w1"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "Structural root cause. `adapter_factory.py:17` imports the pure helper\n`clickhouse/deployment_mode.py` but executing `clickhouse/__init__.py:5-11`\neagerly re-exports the heavy `adapter`/`setup`/`client` modules. Either\n(a) make `clickhouse/__init__.py` lazy via module `__getattr__`, or\n(b) have `adapter_factory.py:17` import `deployment_mode` by full path\nwithout triggering the package `__init__`. Without this, the next engine\nmodule added to the chain re-introduces the #681 failure.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Break the ClickHouse package-init eager re-export that defeats lazy loading", "wid": "w2"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "BLIND SPOT: not reachable on the import surface today, but a future wiring\nof these benchmarks recreates the exact #681 failure. Convert top-level\n`import pandas` to in-function in:\n - benchbox/core/joinorder/dataframe_queries.py:33\n - benchbox/core/ssb/dataframe_queries/queries.py:8\n - benchbox/core/joinorder_synthetic/dataframe_queries.py:17\nConfirm none are reached from `import benchbox` before and after.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Convert the latent top-level pandas imports to in-function imports", "wid": "w3"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "BLIND SPOT: `tests/unit/test_release_infrastructure.py:99-107` asserts the\nSTRING `pandas>=` is present in pyproject \u2014 it locks in the #681 patch and\nwill fail once pandas leaves core (w6). Replace it with a test that imports\n`benchbox` in a pandas-blocked / minimal-deps environment (e.g. a\n`sys.modules['pandas'] = None` guard or subprocess with pandas absent) and\nasserts success. Test the CONTRACT (import works on minimal deps), not the\nmanifest text.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace the wrong-contract pandas test with a minimal-deps import test", "wid": "w4"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w4"], "notes": "BLIND SPOT: a core-only clean-install smoke test exists\n(`.github/workflows/nightly.yml:235`, `test.yml:302`:\n`uv run --isolated --no-project --with -- python -c \"import benchbox\"`),\nbut it now PASSES because pandas is core \u2014 it no longer proves the surface\nis minimal, which is why 0.3.0 shipped broken. Strengthen it to install the\nCORE deps WITHOUT pandas (or block pandas) and assert `import benchbox`\nsucceeds. Confirm this gate runs on the release path (`release.yml:181,188`),\nnot only nightly/test, so a broken import surface blocks the tag.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make the core-only smoke gate prove a minimal import surface and gate the release", "wid": "w5"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Only after w1-w5 are green. Remove `pandas>=2.0.0` from\n`[project.dependencies]` (`pyproject.toml:52`). It remains in the `tpcdi`,\n`pandas`, `dataframe-pandas`, `modin`, `dask`, and `all` extras, and arrives\ntransitively via `chdb`. Verify with `uv` that a core-only resolve excludes\npandas and that each pandas-needing extra still resolves. Resolve the\ncore-dep/extra contradiction.\nBLIND SPOT (pin): the `pandas>=2.0.0` floor was copied, not chosen (it\npredates #681 across the extras). Decide deliberately: keep `>=2.0.0` or\ntighten/loosen with a one-line rationale; do not leave it as cargo-culted.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove pandas from core dependencies and reconcile the extras/pin", "wid": "w6"}, {"evidence": null, "item_id": "drop-pandas-from-core-dependencies", "needs": ["w1"], "notes": "`pyarrow` and `numpy` are reached on `import benchbox` and are core. Verify\nthat is intentional (results/Arrow handling is plausibly genuinely core) and\nrecord the decision. Cross-reference `dependency-inventory.md` F1\n(`chdb-core` flagged for removal from core) and F2 (`jsonschema`) so this\nwork aligns with the documented thin-core direction rather than fixing only\npandas in isolation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit remaining eager heavy imports on the import surface", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "break the MCP tool registration", "instead": "(not specified)", "why": "generate_chart must still work; just remove the HTML branch and simplify the response model"}, {"dont": "delete exporters.py entirely", "instead": "(not specified)", "why": "export_ascii() and render_ascii_chart() serve the ASCII path; only remove export_figure() and Plotly imports"}, {"dont": "delete result_plotter.py entirely", "instead": "(not specified)", "why": "it contains the data loading and normalization logic (from_sources, NormalizedResult) that ASCII charts depend on"}, {"dont": "leave 'plotly' in pyproject.toml extras", "instead": "(not specified)", "why": "orphaned optional deps confuse users and bloat installs"}, {"dont": "remove the chart template system (templates.py)", "instead": "(not specified)", "why": "templates are format-agnostic and drive which charts to generate, not how to render them"}], "approach": "Work outside-in: start by deleting the leaf Plotly files (charts.py, styles.py, dependencies.py)\nthat nothing else imports, then strip Plotly code from the shared files (result_plotter.py,\nexporters.py, visualization.py) that have both Plotly and ASCII paths. After removal, unify the\ndata models by promoting ASCII models as canonical and refactoring ResultPlotter to produce them\ndirectly. Finally simplify the MCP/CLI interface by removing format branching.\n\nKey files to reference for understanding the current dual-path architecture:\n- exporters.py:render_ascii_chart() - shows the conversion between Plotly and ASCII models\n- mcp/tools/visualization.py - shows the format branching (HTML vs ASCII)\n- charts.py - the Plotly chart classes that will be deleted entirely\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": [], "description": "Remove Plotly and the entire HTML chart rendering pipeline from BenchBox. The ASCII charting\nsystem (2,255 lines, 6 chart types, zero dependencies) is the actively-used visualization\npath for CLI, MCP, and CI. The Plotly/HTML path (~1,750 lines source + ~1,260 lines tests)\ngenerates static .html files that nobody opens - there is no browser integration, no HTTP\nserver, no CI usage, and no documented user workflow that depends on HTML output.\n\nRemoving Plotly eliminates:\n- Two parallel rendering pipelines with incompatible data models\n- ~2,300 lines of source + test code (37% reduction in visualization module)\n- 46 MB optional dependency weight (plotly 43 MB + narwhals 3.3 MB)\n- The dual data model problem (BarDatum vs BarData, etc.) that forces every new chart type\n to be implemented twice\n- Three test files that use pytest.importorskip(\"plotly\") and may skip silently\n\nThis clears the path for the ASCII charting enhancement (enhance-ascii-charting.yaml) by\nestablishing a single canonical data model and removing the need for conversion layers.\n\nDecision rationale: Evaluated in depth - HTML charts are the default format but disconnected\nfrom real usage. MCP returns file paths for HTML (useless to LLMs) vs inline content for ASCII\n(immediately useful). CLI generates files users must manually locate and open. No blog workflow\nreferences chart generation. The feature is vestigial.", "id": "drop-plotly-html-charts", "preserves": ["ASCIIChartBase interface and render() -> str contract must not change", "All 6 ASCII chart types (bar, histogram, heatmap, box, line, scatter) must remain fully functional", "CLI benchbox visualize command must continue to work (ASCII output)", "Chart templates (default, flagship, head_to_head, trends, cost_optimization) must work with ASCII", "MCP generate_chart tool must continue to work (ASCII format)", "MCP suggest_charts tool must continue to work (format-agnostic logic)", "NormalizedResult and NormalizedQuery dataclasses must be preserved (shared data loading)", "ResultPlotter.from_sources() must continue loading and normalizing result JSON files", "tests/unit/visualization/ascii/test_ascii_charts.py (934 lines) must pass unchanged"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/ascii/"}, {"kind": "do_not_modify", "path_glob": "tests/unit/visualization/ascii/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/visualize.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/comparison/plotter.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/benchmark_suite.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/charts.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/dependencies.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/result_plotter.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/styles.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/templates.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "docs/visualization/chart-generation-guide.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "tests/integration/visualization/test_visualization_export.py"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/test_visualization_tools.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_charts.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_result_plotter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_styles.py"}, {"kind": "only_modify", "path_glob": "tests/unit/visualization/test_visualize_cli.py"}], "state": "done", "title": "Drop Plotly dependency and HTML chart rendering - go ASCII-only", "verifications": [{"command": "uv run -- python -m pytest tests/unit/visualization/ascii/ -v", "description": "All ASCII chart tests pass unchanged", "expected": "all tests pass, 0 failures", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/mcp/test_visualization_tools.py -v", "description": "MCP visualization tool tests pass", "expected": "all tests pass (some removed, remaining pass)", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -r 'import plotly\\|from plotly\\|require_plotly' benchbox/ --include='*.py'", "description": "No references to plotly remain in source code", "expected": "no output (zero matches)", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -c 'plotly' pyproject.toml", "description": "plotly is not in any pyproject.toml dependency group", "expected": "0", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -x -q", "description": "Full test suite passes", "expected": "all tests pass, 0 failures", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run ruff check benchbox/core/visualization/", "description": "Lint and type check clean", "expected": "no lint errors", "item_id": "drop-plotly-html-charts", "last_result": null, "last_run": null, "seq": 6}], "work": [], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "New Feature Requests", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Add dry run capability to CLI for configuration and query preview\"\n- \"**Tasks**:\"\n- \"**Files Affected**: CLI module, base benchmark classes\"\n- \"**Impact**: \u2705 Improves debugging and configuration validation. Comprehensive dry run system implemented with query extraction, schema preview, resource estimation, and multi-format output.\"", "id": "dry-run-feature-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Dry Run Feature \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a second generic link checker before explaining why the existing Sphinx linkcheck missed the regression", "why": "duplicated docs gates create noise without proving this failure mode is covered"}, {"INSTEAD": "(not specified)", "dont": "whitelist the originally-broken path to make linkcheck pass", "why": "that defeats the purpose of adding the check"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The agent-proxy dry-run on 2026-04-28 found that\n`docs/contributing-results.md` Prerequisites #1 linked to\n`getting-started.rst` at the wrong relative path (the file lives at\n`docs/usage/getting-started.md`, not `docs/getting-started.rst`). The\nlink itself is patched in the same PR as the retrospective. The repo\nnow has Sphinx linkcheck coverage in `.github/workflows/docs.yml` and\n`make docs-linkcheck`; this TODO exists to explain why that existing\ncoverage did not prevent the source-relative link regression, then\npatch the smallest gap in the existing docs validation surface.\n\nSee `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`\nfor the originating finding.", "id": "dry-run-followup-broken-getting-started-link", "preserves": ["Docs CI total wall-clock stays within the existing budget (linkcheck added as a parallel job, not serialized after the slow build)", "Existing relative links in `docs/` that currently resolve continue to resolve (no false positives from a poorly-tuned linkcheck config)", "External-link 404s remain advisory, not blocking -- a third-party link rotting should not block a docs PR"], "prior_art": [{"concept": ".github/workflows/docs.yml \u2014 existing blocking Sphinx linkcheck job; EXTEND before adding a separate checker", "decision": "extend", "path": ".github/workflows/docs.yml"}, {"concept": "Makefile \u2014 existing `ci-linkcheck`, `docs-linkcheck`, and `docs-check` wrappers; REUSE for local verification", "decision": "reuse", "path": "Makefile"}, {"concept": "docs/conf.py \u2014 existing `linkcheck_ignore` configuration; EXTEND only for flaky external URLs, not local relative-link misses", "decision": "extend", "path": "docs/conf.py"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "scripts/"}], "state": "done", "title": "Close the docs linkcheck gap for broken source-relative links", "verifications": [{"command": "rg -n 'sphinx-build -b linkcheck' .github/workflows/docs.yml Makefile", "description": "Existing Sphinx linkcheck remains wired into docs CI", "expected": "docs.yml and Makefile both contain linkcheck wiring", "item_id": "dry-run-followup-broken-getting-started-link", "last_result": null, "last_run": null, "seq": 1}, {"command": "! rg -n 'getting-started\\.rst|\\]\\(getting-started\\.rst\\)' docs/contributing-results.md", "description": "The originally-broken source-relative path no longer appears", "expected": "no matches", "item_id": "dry-run-followup-broken-getting-started-link", "last_result": null, "last_run": null, "seq": 2}, {"command": "make docs-linkcheck", "description": "Docs linkcheck passes after the coverage-gap fix", "expected": "exit 0", "item_id": "dry-run-followup-broken-getting-started-link", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "dry-run-followup-broken-getting-started-link", "needs": [], "notes": "Current prior art:\n - `.github/workflows/docs.yml` has a blocking `linkcheck` job.\n - `Makefile` has `ci-linkcheck`, `docs-linkcheck`, and\n `docs-check` wrappers.\n - `docs/index.rst` includes `contributing-results.md` in the\n toctree.\nReconstruct the failure mode from the 2026-04-28 dry-run context:\nwas the link fixed before linkcheck ran, was linkcheck absent from\nthe relevant PR gate at the time, did MyST/Sphinx not validate the\nsource-relative target shape, or did the path only break in the\nGitHub-rendered markdown view?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reproduce why existing Sphinx linkcheck missed the original source-relative link", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-broken-getting-started-link", "needs": ["w1"], "notes": "Prefer extending the existing Sphinx/docs validation path over\nadding a second link checker. Examples:\n - If `docs/contributing-results.md` was not in the checked source\n set at the time, add an explicit regression assertion to\n `docs-validate` or the docs CI job.\n - If Sphinx validates built links but not GitHub-rendered markdown\n source links, add a small markdown relative-link check scoped to\n repo-local targets.\n - Add `markdown-link-check` only if the Sphinx path cannot cover\n this exact failure mode.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Patch the existing docs validation surface at the narrowest failing layer", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-broken-getting-started-link", "needs": ["w2"], "notes": "Run the existing docs linkcheck and the new/extended local-link\nassertion. Fix local relative-link failures in the same PR if they\nare under `docs/`; file separate follow-ups for unrelated generated\ndocs or broad external-link cleanup.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the docs link checks and fix any other latent local-link breakage", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "pick a path and apply it", "dont": "leave the rows in with vague qualifiers", "why": "that is the failure mode this TODO exists to fix"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "**Completion note:** plans/tuning sidecars are community-submittable\ncompanion files when they sit next to the primary result JSON. The\ncontributor-facing doc now explains the opt-in flags and labels both rows\nas optional.\n\n`docs/contributing-results.md` Step 2 lists `.plans.json` and\n`.tuning.json` as bundle outputs gated on `(if captured)` /\n`(if used)`. Neither phrase is defined anywhere in the doc; nothing\nexplains how a contributor opts into plan capture or applies tuning\nbefore running. First-time contributors see the rows and wonder if\ntheir bundle is incomplete because those files don't appear.\n\nDecision needed: are plans/tuning files community-submittable, or\nmaintainer-only? Either explain the opt-in or drop the rows from\nthe contributor-facing doc and surface them in advanced docs only.\n\nSurfaced by the agent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).", "id": "dry-run-followup-bundle-table-conditionals", "preserves": ["If plans/tuning files ARE community-submittable, the validate-submission CI continues to accept them"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "docs/operations/"}], "state": "done", "title": "Clarify or drop the plans.json / tuning.json rows in the Step 2 bundle table", "verifications": [{"command": "grep -cE '\\(if captured\\)|\\(if used\\)' docs/contributing-results.md", "description": "Bundle table no longer contains undefined `(if captured)` / `(if used)` qualifiers", "expected": "0", "item_id": "dry-run-followup-bundle-table-conditionals", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dry-run-followup-bundle-table-conditionals", "needs": [], "notes": "Decision: plans.json / tuning.json sidecars are community-submittable\nwhen present next to the primary result JSON. Evidence:\n\n- `benchbox submit` discovers `.plans.json` and `.tuning.json`\n companions beside the source result, copies them into `bundle/`,\n and records per-file hashes in `manifest.companion_hashes`.\n- `scripts/validate_submission.py` treats those suffixes as companion\n files rather than primary bundles and verifies their hashes when\n declared in the manifest.\n- Unit coverage in `tests/unit/cli/commands/test_submit.py` and\n `tests/unit/scripts/test_validate_submission.py` exercises companion\n copy and hash validation behavior.\n\n`/code review W1`: no required issues. The decision is backed by the\ncurrent submission/validation contract; W2 should document the opt-in\nflags rather than removing the rows.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide whether plans/tuning files are community-submittable", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-bundle-table-conditionals", "needs": ["w1"], "notes": "Documented the accepted opt-in path in `docs/contributing-results.md`:\nuse `--capture-plans` on supported platforms for `.plans.json`,\nand use `--tuning tuned`, `--tuning auto`, or a YAML path for\nintentionally tuned submissions that may emit `.tuning.json`.\nThe Step 2 bundle table now labels both rows as optional and no longer\nuses undefined `(if captured)` / `(if used)` qualifiers.\n\n`/code review W2`: no required issues. The text preserves the\ncommunity-submittable sidecar contract from W1 while making absence of\nsidecars explicitly non-blocking for default submissions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the chosen change to docs/contributing-results.md", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "block on completing all 7 work units before landing any", "why": "each is independently shippable; ship as ready"}, {"INSTEAD": "(not specified)", "dont": "bundle the validator hash mismatch fix into this TODO", "why": "that's release-blocker scope and tracked separately as `dry-run-followup-validator-hash-contract-mismatch`"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-29T00:00:00Z", "deferrals": [], "deps": [], "description": "Seven smaller friction items surfaced by the 2026-04-29 Codex\nagent-proxy dry-run. Each work unit is independently verifiable and\nsmall; consolidated into one TODO to avoid TODO sprawl. None of these\nare blocking by themselves, but cumulatively they make the difference\nbetween a smooth and a frustrating first submission.\n\nSee `_project/handoffs/external-dry-run-retrospective-2026-04-29.md`\nfor the full report and Codex's verbatim feedback (Appendix A there).\n\nNote: the release-blocker hash-contract mismatch from the same\ndry-run is tracked separately as\n`dry-run-followup-validator-hash-contract-mismatch.yaml` (Critical).\nThis TODO is polish; that one is correctness.", "id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "preserves": ["Existing user-facing behavior on the success path -- these are output / UX tweaks only, not flow changes", "Hard-failure red \u274c for actually-unrecoverable errors -- don't make every error look amber and dilute the signal", "The canonical `uv run -- python` examples in the validation block (those are unambiguous and contributor-tested)"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "README.md (w1: invocation consistency)"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py (w4: drop Interactive header at line 706)"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py (w2: print next commands)"}, {"kind": "only_modify", "path_glob": "benchbox/cli/output.py (w3: downgrade red `Missing tables` at line 172)"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/power_test.py (w7: reword stream_id warning at lines 172-176)"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md (w1: invocation consistency)"}, {"kind": "only_modify", "path_glob": "docs/platforms/dataframe.md (w5: rename 'Platform Availability' at line 358)"}, {"kind": "only_modify", "path_glob": "docs/usage/ (w1: invocation consistency)"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/ (regression tests for w2, w3, w4, w6)"}], "state": "done", "title": "CLI UX polish + doc consistency fixes from 2026-04-29 Codex dry-run", "verifications": [{"command": "grep -rEc '^[^#]*\\buv run benchbox\\b(?! --)' docs/contributing-results.md docs/usage/ README.md 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "Doc invocation consistency: zero non-canonical `uv run benchbox` (without `--`) in contributor docs", "expected": "0 (assuming `uv run -- benchbox` is the canonical form chosen in w1)", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 1}, {"command": "rm -rf /tmp/bb-test && uv run -- benchbox submit --last --output /tmp/bb-test 2>&1 | grep -c 'published-results'", "description": "`benchbox submit` output mentions the PR target branch inline", "expected": ">= 1", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 2>&1 | grep -c 'Interactive Benchmark Runner'", "description": "`benchbox run` non-interactive does not print the interactive header", "expected": "0", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --force datagen 2>&1 | grep -E '\\[red\\][^[]*Missing tables|\u274c[^\\n]*Missing tables' | wc -l", "description": "Missing-tables auto-recovery does not print in red", "expected": "0", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 4}, {"command": "grep -cE 'timing only|per TPC-H spec' benchbox/core/tpch/power_test.py", "description": "Stream-id warning explicitly mentions 'timing only' or 'per TPC-H spec'", "expected": ">= 1", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "DECISION: `uv run -- benchbox` is the canonical form for\ncontributor-facing submission docs. Bare `benchbox` remains\nacceptable in broad reference examples that explicitly assume an\ninstalled CLI.\n\nChanges:\n - Normalized the active submission workflow in\n `docs/contributing-results.md` to `uv run -- benchbox`.\n - Updated the public-results contribution paragraph in\n `README.md` to use `uv run -- benchbox submit`.\n\n`/code review W1`: no required issues. Scope stayed focused on\ncontributor-facing submission paths rather than rewriting every\nbroad README/reference example.\n\nVerification:\n - `grep -rEc '^[^#]*\\buv run benchbox\\b(?! --)' docs/contributing-results.md docs/usage/ README.md 2>/dev/null | awk -F: '{s+=$2}END{print s}'`\n reports no matches.\n - `git diff --check` passed.\n\nCodex re-read steps to figure out which form to use. Pick one\ncanonical form for contributor-facing docs and normalize.\nRecommend `uv run -- benchbox` (matches the repo's uv-first\nconvention, matches the `uv run -- python` examples in the\nvalidation block). Bare `benchbox` only where the doc explicitly\nassumes a system install.\nFiles to scan: docs/contributing-results.md, docs/usage/,\nREADME.md, docs/development/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit doc invocation consistency: pick canonical form between `benchbox`, `uv run benchbox`, `uv run -- benchbox`", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing implementation already satisfies this work item:\n - `_print_submission_summary()` prints the concrete `cp`\n command for the bundle and manifest, the inventory command,\n the validation command, the PR title, and the\n `published-results` target branch.\n - `tests/unit/cli/commands/test_submit.py` includes the pinned\n W2 regression\n `test_submit_prints_next_steps_with_pr_target_branch`.\n\n`/code review W2`: no required issues. The code path is covered\nby the focused submit unit suite, and no production-code patch was\nneeded for this slice.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/commands/test_submit.py -q`\n passed: 28 passed.\n\nCodex's recommendation #2: after writing the bundle, print the\nexact next commands the contributor should run, with paths\nfilled in:\n - cp results-data/bundles/\n - uv run -- python scripts/generate_corpus_inventory.py --write\n - uv run -- python scripts/validate_submission.py results-data/bundles/\n - PR title: `results: sf`\n - PR target branch: `published-results`\nReduces re-read load and removes the \"where does this go?\"\nfriction. The information is already in the in-bundle\nCONTRIBUTING.md after PR #43; this surfaces it inline at submit\ntime so contributors don't have to switch contexts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make `benchbox submit` print the exact next commands", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing production output already renders missing-table details\nas yellow/recoverable while preserving red output for hard\nfailures such as inaccessible tables.\n\nChange:\n - Added\n `test_missing_tables_validation_message_is_recoverable_amber`\n in `tests/unit/cli/test_cli_output.py` to pin the amber\n missing-table branch and the red inaccessible-table branch.\n\n`/code review W3`: no required issues. The test is scoped to the\nexact formatter behavior and does not weaken hard-failure red\noutput.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/test_cli_output.py::TestConsoleResultFormatter::test_missing_tables_validation_message_is_recoverable_amber -q`\n passed: 1 passed.\n\nCodex hit `\u274c Missing tables...` printed in red, then the runner\nautomatically deleted+rebuilt the DuckDB file and the run\nsucceeded. The user-visible result was success, but the red \u274c\nmade it look like a failure mid-run. Downgrade to amber/yellow\nwith a \"rebuilding...\" spinner; reserve red \u274c for actually\nunrecoverable errors.\n\nAudit findings (2026-04-29) \u2014 emit sites:\n - `benchbox/cli/output.py:172` -- `[red]\u2022 Missing tables:[/red]`\n (this is the most likely red \u274c Codex saw)\n - `benchbox/platforms/base/result_capture.py:1372` -- already\n uses `\u26a0\ufe0f` (amber); not the offending site\n - `benchbox/platforms/base/validation.py:174` -- adds error\n to a result object; check if it propagates to the [red] path\n - `benchbox/utils/data_validation.py:426` -- emit helper\n - `benchbox/core/validation/engines.py:432` -- error append\n\nTrace the auto-recovery path: which caller catches the\nvalidation error, deletes+rebuilds the DuckDB file, and re-runs?\nDowngrade `cli/output.py:172` to amber/yellow IF the caller is\nrecoverable; keep red only for hard-fail paths.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Downgrade `benchbox run` recoverable Missing-tables from red \u274c to amber recovery message", "wid": "w3"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing production code already gates the header on\n`sys.stdin.isatty()` and the absence of explicit platform +\nbenchmark args.\n\nChange:\n - Added focused tests in\n `tests/unit/cli/test_run_command_interactive_paths.py` for\n suppressing the header with explicit run args and preserving\n it for a prompted TTY run.\n\n`/code review W4`: no required issues. The regression tests\ncover the requested suppression without removing the true\ninteractive banner.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/test_run_command_interactive_paths.py::test_interactive_header_suppressed_for_explicit_run_args tests/unit/cli/test_run_command_interactive_paths.py::test_interactive_header_shown_for_prompted_tty_run -q`\n passed: 2 passed.\n\nCodex called this out: showing 'Interactive Benchmark Runner'\nwhen running with explicit flags (no interactive prompt) is\nmisleading. Suppress the header when stdin is not a TTY OR\nwhen explicit `--platform` / `--benchmark` args are provided.\n\nAudit findings (2026-04-29) \u2014 emit site:\n - `benchbox/cli/commands/run.py:706` --\n `Text(\"BenchBox Interactive Benchmark Runner\", style=\"bold blue\")`\nSingle-site fix; gate on `sys.stdin.isatty() and not (platform\nand benchmark)` or equivalent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Drop 'Interactive Benchmark Runner' header in non-interactive `benchbox run`", "wid": "w4"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing docs already use the CLI-matching `Available Databases`\nheading in `docs/platforms/dataframe.md`, and the other audited\ndocs also align with the CLI output.\n\nEvidence:\n - `docs/platforms/dataframe.md:358` is `## Available Databases`.\n - `benchbox/cli/system.py` and `benchbox/cli/database.py` print\n or title the surface as `Available Databases`.\n - `rg -n \"Platform Availability|Available Databases|Checking Platform Availability|benchbox profile\" ...`\n found no remaining `Platform Availability` doc mismatch.\n\n`/code review W5`: no required issues. No docs patch was needed\nbeyond recording the resolved state.\n\nThe doc references a 'Platform Availability' surface; the actual\nCLI prints 'Available Databases'. Either add the doc-claimed\ntable to the CLI, or update docs to match what the CLI prints.\nLatter is cheaper.\n\nAudit findings (2026-04-29) \u2014 file pins:\n - Doc: `docs/platforms/dataframe.md:358` (`## Checking Platform\n Availability`) -- main mismatch surface\n - Doc: `docs/usage/intelligent-guidance.md:49,83` -- already\n uses 'Available Databases' (ASCII tree examples); aligned\n - CLI: `benchbox/cli/system.py:42` -- prints\n `[cyan]Available Databases:[/cyan]`\n - CLI: `benchbox/cli/database.py:395,504` --\n `Table(title=\"Available Databases\")`\nRecommended: rename the doc heading + body in\n`docs/platforms/dataframe.md:358` to 'Available Databases'\nand align the prose.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sync `benchbox profile` docs ('Platform Availability' \u2192 actual 'Available Databases')", "wid": "w5"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing implementation already routes dry-run and real submit\noutput through the shared `_print_submission_summary()` helper.\nDry-run only changes the header and the trailing\n`(dry run; no files written)` footer; the file list and next-step\ncommand shape are shared.\n\nExisting pinned test:\n - `test_submit_dry_run_matches_real_run_shape` in\n `tests/unit/cli/commands/test_submit.py`.\n\n`/code review W6`: no required issues. The shared helper avoids\ndrift between dry-run and real submit output, and the dry-run path\nstill asserts no files are written.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/commands/test_submit.py::test_submit_dry_run_matches_real_run_shape -q`\n passed: 1 passed.\n\nCodex noted the dry-run path output format and real output\nformat differ slightly. Make `--dry-run` print the same\nfile/manifest/contributing list shape as the real run, just\nwith a \"(dry run; no files written)\" footer. Today they're\nsimilar but the dry-run formatting is its own one-off.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align `benchbox submit --dry-run` output format with real submit output", "wid": "w6"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing warning text already includes the requested clarification:\nstream 0 is validated against the answer set, and streams > 0 are\nrun for timing only per TPC-H spec.\n\n`/code review W7`: no required issues. The behavior remains\nunchanged: auto-selected validation is disabled for nonzero\nstreams, while explicit validation modes are not silently altered\nby this wording-only path.\n\nVerification:\n - `grep -cE 'timing only|per TPC-H spec' benchbox/core/tpch/power_test.py`\n returned 2.\n\nCodex's run hit a warning that validation was disabled for\nstream_id != 0 and was unsure whether the result was actually\nfully validated.\n\nAudit findings (2026-04-29) \u2014 emit site:\n - `benchbox/core/tpch/power_test.py:172-176`:\n if stream_id != 0 and validation and validation_mode is None:\n actual_validation_mode = \"disabled\"\n self.logger.warning(\n \"\u26a0\ufe0f TPC-H answer sets are only available for stream 0. \"\n \"Disabling validation for stream_id != 0.\"\n )\n\nThe warning IS correct (TPC-H answer sets only exist for stream 0),\nbut Codex's confusion shows the wording leaves room for \"did my\nresult get validated or not?\" doubt. Reword to explicitly state:\n\"Stream 0 was validated against the answer set; streams > 0 are\nrun for timing only (per TPC-H spec).\" Suppression is the wrong\ncall -- the warning protects users running multi-stream from\nclaiming false validation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Clarify the `stream_id != 0 validation disabled` TPC-H warning", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "audit consumers first (w1)", "dont": "change the writer without updating every reader in the same PR", "why": "mid-flight readers will fail to find the manifest"}, {"INSTEAD": "if any exist with the old name, support both during a transition window", "dont": "silently break already-merged manifests in results-data/bundles/", "why": "that erodes trust in already-published submissions"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": ["dry-run-followup-package-canonical-contributing"], "description": "`benchbox submit` always writes the submission manifest as the literal\nfilename `submission-manifest.json` (constant\n`SUBMISSION_MANIFEST_FILENAME` in\n`benchbox/core/explorer_pipeline/pipeline.py:39`, also used in\n`benchbox/cli/commands/submit.py:338`). The doc instruction is \"copy\n`submission/submission-manifest.json` alongside the bundle files\" into\n`results-data/bundles/`. `results-data/bundles/` currently contains\nzero manifest files, but two community PRs landing in the same window\nwould conflict on the same filename. The doc names no convention for\nresolving the collision.\n\nProposal: emit `.manifest.json` so the manifest's\nfilename inherits the bundle's uniqueness (timestamp + hash). Update\ndoc, in-bundle CONTRIBUTING.md, validators, and the inventory generator\nif any of them filter by exact filename.\n\nSurfaced by the agent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).", "id": "dry-run-followup-manifest-filename-convention", "preserves": ["Backwards-compat for any already-merged community submissions that used the old filename", "The trust-label resolution path in `explorer_pipeline/pipeline.py` continues to find manifests for existing bundles"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/validate-submission.yml"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/pipeline.py"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "only_modify", "path_glob": "scripts/validate_submission.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Per-bundle manifest filenames to avoid PR collision in results-data/bundles/", "verifications": [{"command": "uv run -- benchbox submit --last --output /tmp/dry-run-mft && ls /tmp/dry-run-mft | grep -c '\\.manifest\\.json$'", "description": "Submit emits per-bundle manifest filenames", "expected": "1", "item_id": "dry-run-followup-manifest-filename-convention", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -c 'manifest.json' scripts/generate_corpus_inventory.py", "description": "Inventory generator resolves manifests by glob", "expected": ">= 1", "item_id": "dry-run-followup-manifest-filename-convention", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": [], "notes": "Grep for `submission-manifest.json` and `SUBMISSION_MANIFEST_FILENAME`\nacross:\n - benchbox/cli/commands/submit.py (writer)\n - benchbox/core/explorer_pipeline/pipeline.py (reader, trust-label resolver)\n - scripts/validate_submission.py\n - scripts/generate_corpus_inventory.py\n - .github/workflows/validate-submission.yml (if it references the filename)\nDocument everywhere the filename is hard-coded so the rename is\natomic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit consumers of submission-manifest.json filename", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": ["w1"], "notes": "Update the writer to emit `.manifest.json`. Update\nreaders to resolve manifests by glob (`*.manifest.json`) rather\nthan by literal name. Update the inventory generator's filter\nrules if it currently filters by exact filename.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rename to .manifest.json and update consumers", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": ["w2"], "notes": "`docs/contributing-results.md` Step 2 / Step 3.3 both name the\nfile. The in-bundle CONTRIBUTING.md is being rewritten in\n`dry-run-followup-package-canonical-contributing` \u2014 coordinate\nso both land consistent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs (canonical + in-bundle) to reflect the new filename", "wid": "w3"}, {"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": ["w2"], "notes": "Two consecutive `benchbox submit --output ./submission` calls\nagainst different result files should not overwrite each other's\nmanifests. Test asserts both manifests exist after the second\ncall.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test for two consecutive submits", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "ship a checklist + canonical URL instead", "dont": "remove the in-bundle file entirely", "why": "contributors offline / on planes still need a packaged copy of the steps"}, {"INSTEAD": "regenerate at build time or include a clear pointer", "dont": "silently keep the in-bundle template diverged from the canonical doc", "why": "that is the exact failure mode this TODO exists to fix"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "`benchbox submit` writes a 16-line CONTRIBUTING.md into every submission\nbundle (hardcoded as `_CONTRIBUTING_TEXT` at\n`benchbox/cli/commands/submit.py:29-46`), and the CLI's \"Next step\"\nmessage tells contributors to follow it. That file does NOT include\nthree steps the canonical `docs/contributing-results.md` requires:\n\n 1. Regenerate `corpus-inventory.json` before commit.\n 2. Target the `published-results` branch (not `main`).\n 3. Optionally validate the bundle locally with\n `scripts/validate_submission.py`.\n\nA contributor who reads only the in-bundle file will open a PR against\n`main` with a stale inventory. CI catches it, but that's wasted\nmaintainer cycles and a worse first-time UX. Surfaced by the\nagent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).\n\nFix options (pick one):\n A. Generate the in-bundle file from `docs/contributing-results.md`\n at package-build time so the two cannot drift.\n B. Replace the in-bundle file with a one-page checklist that points\n to the canonical doc URL and lists the three required commands\n inline.", "id": "dry-run-followup-package-canonical-contributing", "preserves": ["The CLI's existing 'Next step' message still points contributors at the in-bundle file", "`benchbox submit --output` continues to package a self-contained bundle directory"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/commands/test_submit.py"}], "state": "done", "title": "Make benchbox submit's packaged CONTRIBUTING.md match docs/contributing-results.md", "verifications": [{"command": "uv run -- benchbox submit --last --output /tmp/dry-run-canonical && grep -cE 'published-results|generate_corpus_inventory|validate_submission|docs\\.benchbox\\.dev' /tmp/dry-run-canonical/CONTRIBUTING.md", "description": "Packaged CONTRIBUTING.md mentions all four required items", "expected": ">= 4", "item_id": "dry-run-followup-package-canonical-contributing", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/commands/test_submit.py -q", "description": "Submit unit tests still pass", "expected": "All tests pass", "item_id": "dry-run-followup-package-canonical-contributing", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-package-canonical-contributing", "needs": [], "notes": "A vs B is a real architectural call:\n - A: harder-to-drift but introduces a build-time read of\n `docs/contributing-results.md` from `submit.py`. Requires\n packaging the doc into the wheel or reading it relative to\n the repo root.\n - B: simpler; ships a checklist that names the four required\n steps and points contributors to docs.benchbox.dev for the\n full doc. Requires updating the checklist whenever the\n canonical doc changes -- but the checklist is small and\n stable.\nDefault recommendation: option B unless wheel-packaging the doc\nis already cheap. Document the choice + rationale in a one-line\ncomment above `_CONTRIBUTING_TEXT` so future maintainers don't\nre-litigate it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pick option A (regenerate from canonical doc at build time) or option B (static checklist + canonical URL)", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-package-canonical-contributing", "needs": ["w1"], "notes": "Touch `benchbox/cli/commands/submit.py` (`_CONTRIBUTING_TEXT`).\nThe new file content MUST include all four:\n - `published-results` as the PR base branch (not `main`)\n - `generate_corpus_inventory.py --write` before commit\n - `validate_submission.py` for local pre-push check\n - canonical doc URL (https://docs.benchbox.dev/contributing-results\n or wherever the canonical doc lives at land time)\nCoordinate with `dry-run-followup-submitted-by-flag` and\n`dry-run-followup-manifest-filename-convention` since both\nwill edit the same template after this lands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the chosen option in benchbox/cli/commands/submit.py", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-package-canonical-contributing", "needs": ["w2"], "notes": "Add to `tests/unit/cli/commands/test_submit.py`: run\n`benchbox submit --last --output ` against a fixture result,\nread the resulting CONTRIBUTING.md, assert presence of\n`published-results`, `generate_corpus_inventory`,\n`validate_submission`, and the canonical doc URL. The test\npins the contract independent of which option (A or B) was\nchosen.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test asserting the in-bundle file mentions all four required items", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "keep code and doc consistent", "dont": "change the default fallback source without updating the doc table in the same PR", "why": "contributors verifying their manifest will look for whichever field the doc names"}, {"INSTEAD": "warn and continue", "dont": "make empty submitted_by a hard error", "why": "anonymous submissions are valid"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": ["dry-run-followup-package-canonical-contributing"], "description": "`docs/contributing-results.md` describes the submission manifest as\ncontaining a \"contributor\" field. The CLI does fall back to\n`git config user.name` via `_get_git_username()` at\n`benchbox/cli/commands/submit.py:49-60`, but when that returns empty\n(no git user.name configured \u2014 e.g. fresh sandbox), the manifest's\n`submitted_by` is silently set to `\"\"` with no warning.\n\nTwo improvements:\n 1. Warn loudly when the fallback returns empty, telling the\n contributor exactly how to set it.\n 2. Add `--submitted-by TEXT` for explicit override (CI, automation,\n contributors who prefer not to set git config).\n\nSurfaced by the agent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).\nThe Cowork sandbox had no `git config user.name` set, which is what\ntriggered the silent-empty path that this TODO addresses.", "id": "dry-run-followup-submitted-by-flag", "preserves": ["Existing `_get_git_username()` fallback behavior remains the default when no flag is passed", "The empty-submitted_by case stays a warning, not a hard failure -- some contributors may legitimately submit anonymously"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/commands/test_submit.py"}], "state": "done", "title": "benchbox submit: warn on empty submitted_by, add --submitted-by override", "verifications": [{"command": "uv run -- benchbox submit --help | grep -c -- '--submitted-by'", "description": "--submitted-by flag is in the help output", "expected": "1", "item_id": "dry-run-followup-submitted-by-flag", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/commands/test_submit.py -q", "description": "Submit unit tests pass", "expected": "All tests pass", "item_id": "dry-run-followup-submitted-by-flag", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-submitted-by-flag", "needs": [], "notes": "In the manifest-write path, if `submitted_by` would be empty,\nprint a warning to stderr: \"submitted_by is empty \u2014 set\n`git config user.name ` or pass --submitted-by NAME\".\nDo not fail the command; the empty field is a soft issue, not\na hard error.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Warn when _get_git_username returns empty and no --submitted-by was passed", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-submitted-by-flag", "needs": ["w1"], "notes": "Add `--submitted-by TEXT` option to `benchbox submit`. Precedence:\nexplicit flag > `git config user.name` > empty (with warning).\nDocument in `docs/contributing-results.md` and in the in-bundle\nCONTRIBUTING.md (which is being rewritten in\n`dry-run-followup-package-canonical-contributing` \u2014 coordinate\nthe doc change there).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --submitted-by TEXT flag and plumb into manifest write", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-submitted-by-flag", "needs": ["w2"], "notes": "Test cases: (a) --submitted-by overrides git config, (b) git\nconfig user.name is used when no flag, (c) warning is emitted\nwhen both are empty. Mock `_get_git_username` for determinism.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for the three precedence paths", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "check a real bundle's top-level keys before naming one", "dont": "name a different JSON key than the bundle actually emits", "why": "a contributor grepping their bundle should find what the doc says is there"}, {"INSTEAD": "add the fallback as an alternative line", "dont": "remove the `uv run` examples in favor of plain `python`", "why": "most contributors do have uv and removing it would break the canonical Python tooling pattern"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "Two small contributor-facing clarifications in\n`docs/contributing-results.md`, both surfaced by the agent-proxy\ndry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).\n\n1. The \"Running Validation Locally\" block uses `uv run -- python \u2026`\n for every example. In environments where `uv` is not installed or\n is slow to bootstrap a venv, plain `python scripts/validate_submission.py \u2026`\n works identically. Add a one-line \"no uv? use python directly\"\n note. (Cowork's sandbox specifically hit a 45s `uv run` stall\n and had to fall back to `python3` \u2014 a real-world contributor on\n a constrained machine could hit the same.)\n\n2. The \"Quality Expectations\" section says \"Schema v2 format - only\n the current schema version is accepted\". The actual JSON key is\n `version` (string, currently `\"2.1\"`), not `schema_version`. Name\n the key explicitly so contributors can grep their bundle to\n verify compliance.", "id": "dry-run-followup-uv-fallback-and-schema-key", "preserves": ["The Quality Expectations section still names schema v2 as the accepted format", "The existing `uv run -- python` examples remain in the doc as the canonical invocation -- the fallback is added as an alternative, not a replacement"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "docs/contributing-results.md"}], "state": "done", "title": "Local-validation block: document non-uv fallback and schema-version key name", "verifications": [{"command": "grep -cE 'top-level `version`|JSON key `version`|`version` field' docs/contributing-results.md", "description": "Doc names the top-level version JSON key explicitly", "expected": ">= 1", "item_id": "dry-run-followup-uv-fallback-and-schema-key", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -ciE 'without uv|no uv|don.t have uv' docs/contributing-results.md", "description": "Doc mentions a non-uv fallback for the validation block", "expected": ">= 1", "item_id": "dry-run-followup-uv-fallback-and-schema-key", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-uv-fallback-and-schema-key", "needs": [], "notes": "Add a sentence: \"If you don't have `uv`, replace `uv run -- python`\nwith `python` \u2014 the scripts have no uv-specific dependencies.\"\nPlace it once at the top of the local-validation block.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a non-uv fallback note to Running Validation Locally", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-uv-fallback-and-schema-key", "needs": [], "notes": "Change \"Schema v2 format - only the current schema version is\naccepted\" to \"Schema v2 format (top-level `version` field,\ncurrently `\\\"2.1\\\"`)\". Optionally add a one-line how-to:\n`python3 -c \"import json; print(json.load(open('bundle.json'))['version'])\"`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Name the version JSON key explicitly under Quality Expectations", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "fix the validator instead", "dont": "revert develop's writer to per-directory hash", "why": "per-file is the right contract for individually-addressable bundles"}, {"INSTEAD": "the test is the regression-prevention story for this whole class of bug", "dont": "skip the cross-branch contract test (w3)", "why": "the same drift will happen again the next time either side changes"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-29T00:00:00Z", "deferrals": [], "deps": [], "description": "The `scripts/validate_submission.py` on the `published-results` branch\n(pinned at v0.2.1, commit `0182c9556`) uses a per-DIRECTORY hash\nalgorithm (`hashlib.sha256()` over `path.encode() + file.read_bytes()`\nfor every file in `bundle_dir` excluding the manifest). The same\nscript on `develop` (post-PR #33) uses a per-FILE hash (just\n`sha256(bundle_file)` plus a `companion_hashes` map). The writer in\n`develop`'s `submit.py` also uses per-file. Result: a bundle produced\nby current `develop`'s `benchbox submit` will fail CI on\n`published-results` with `Bundle hash mismatch: manifest says XXXX,\ncomputed YYYY`.\n\nThis was confirmed empirically during the 2026-04-29 Codex\nagent-proxy dry-run:\n - Codex's manifest: `bundle_hash: 799c80a2...` (per-file SHA-256)\n - Validator on published-results computed: `8a89ab6f...` (per-dir)\n - `shasum -a 256` on the actual bundle file: `799c80a2...` (matches\n the manifest exactly)\n\nForensic and full reproduction at\n`_project/handoffs/external-dry-run-retrospective-2026-04-29.md`\n(Appendix B).\n\nThis is a release-blocker for the community submission flow: until\nthe validator on `published-results` matches the writer's contract,\nno contributor can successfully submit a result regardless of how\nclosely they follow the docs. The bug is invisible to maintainers\nwho only run the validator inside `develop`, because both sides\nagree there.", "id": "dry-run-followup-validator-hash-contract-mismatch", "preserves": ["Backwards-compat with already-merged community submissions in results-data/bundles/ (zero exist today, so this is theoretical)", "develop's per-file hash contract (per-file is the right design -- bundles are individually addressable in results-data/bundles/)"], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": "scripts/validate_submission.py (on published-results branch)"}, {"kind": "only_modify", "path_glob": "tests/fixtures/ (only if w3 vendors the validator instead of pulling live)"}, {"kind": "only_modify", "path_glob": "tests/integration/test_cross_branch_validator_contract.py (new file, w3)"}], "state": "done", "title": "Sync published-results validator with develop's per-file hash contract (release-blocker)", "verifications": [{"command": "git fetch --quiet origin published-results && \\\nTMP=$(mktemp) && \\\ngit show origin/published-results:scripts/validate_submission.py > \"$TMP\" && \\\npython3 \"$TMP\" /Users/joe/Developer/BenchBox/submission/bundle/tpch_sf001_duckdb_sql_20260428_205018_e4f24ee2.json 2>&1 | tail -3; \\\nrc=$?; rm -f \"$TMP\"; exit $rc\n", "description": "Validator on published-results passes against a develop-submitted bundle", "expected": "0 error(s), 0 warning(s)", "item_id": "dry-run-followup-validator-hash-contract-mismatch", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/integration/test_cross_branch_validator_contract.py -q", "description": "Cross-branch contract test (added by w3) passes", "expected": "All tests pass (test file is created by w3; this verification only meaningful after w3 lands)", "item_id": "dry-run-followup-validator-hash-contract-mismatch", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-validator-hash-contract-mismatch", "needs": [], "notes": "DECISION: choose path A -- update the `published-results`\nvalidator directly.\n\nRationale:\n - This is a production submission-flow blocker independent of\n the Python package release cadence: GitHub Actions executes\n `scripts/validate_submission.py` from the PR checkout of the\n `published-results` branch.\n - `develop` already has the correct per-file writer and\n validator contract. Waiting for the release-flow dry run would\n delay contributor submissions without improving the contract.\n - The branch-specific validator fix can be verified immediately\n by running the `published-results` validator against the\n Codex dry-run bundle cited in the retrospective.\n\n`/code review W1`: no required issues. The rejected path B remains\nthe longer-term release hygiene path, but it is not the fastest\nunblock for external contributors.\n\nTwo paths:\nA. Cherry-pick the post-PR-#33 `scripts/validate_submission.py`\n (and any related changes) onto `published-results`. Quickest\n unblock; doesn't depend on a release.\nB. Cut the next benchbox release (per\n `release-flow-dry-run-rc1`) which propagates the validator\n to v0.x.y, then sync `published-results` from the release tag.\nRecommend A for speed (the validator is a script, not a wheel\nartefact -- it runs from the PR's checkout of published-results\nregardless of whether benchbox is installed). B is cleaner long\nterm but blocks on the release-flow dry-run already in flight.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide path: cherry-pick develop's validator to published-results, or land a release first", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-validator-hash-contract-mismatch", "needs": ["w1"], "notes": "Applied path A in branch\n`fix/published-results-validator-manifest` and opened PR #87\nagainst `published-results`:\nhttps://github.com/joeharris76/BenchBox/pull/87\n\nScope of the branch-specific patch:\n - `scripts/validate_submission.py` now skips `*.manifest.json`\n during bundle discovery.\n - The validator prefers `.manifest.json`, while\n preserving the legacy `submission-manifest.json` fallback.\n\nVerification run against the Codex dry-run bundle cited in the\nretrospective:\n - legacy `submission-manifest.json` layout: 0 errors,\n 0 warnings.\n - per-bundle `.manifest.json` layout: 0 errors,\n 0 warnings.\n\n`/code review W2`: no required issues. The patch is branch-local,\npreserves legacy manifests, and does not touch the writer.\n\nAfter sync, re-run the validator on `published-results` against\na freshly-submitted bundle:\n cd \n python3 scripts/validate_submission.py /path/to/bundle.json\nExpected: 0 errors, 0 warnings.\nUse the artefacts Codex left at\n`/Users/joe/Developer/BenchBox/submission/` (or regenerate via\n`benchbox submit --last`) for the round-trip.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply chosen path; verify with a round-trip test", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-validator-hash-contract-mismatch", "needs": ["w2"], "notes": "Already landed before this TODO cleanup:\n - `tests/integration/test_cross_branch_validator_contract.py`\n - `tests/fixtures/published_results_validator.py`\n\nEvidence:\n - `git log -- tests/integration/test_cross_branch_validator_contract.py tests/fixtures/published_results_validator.py`\n shows the contract test and vendored validator landed in\n prior dry-run follow-up commits (#49, #51, #45).\n - `cmp -s scripts/validate_submission.py tests/fixtures/published_results_validator.py`\n returns 0, so the vendored validator matches the develop\n validator byte-for-byte.\n - The test covers writer output, companion hashes, symlink\n rejection, missing-manifest-field UX, and validator drift.\n\n`/code review W3`: no required issues. The original TODO\nsuggested fetching `origin/published-results` live; the landed\ntest intentionally vendors the validator to keep the fast lane\ndeterministic and avoid network/branch-state flake.\n\nAdd `tests/integration/test_cross_branch_validator_contract.py`\nthat validates the writer's emitted hashes match what the\nvalidator on `published-results` computes. Implementation shape:\n - Run `benchbox submit` to produce a bundle + manifest in a\n temp dir.\n - Fetch `scripts/validate_submission.py` from\n `origin/published-results` (via `git show\n origin/published-results:scripts/validate_submission.py`)\n and exec it against the freshly-emitted bundle.\n - Assert no hash mismatch.\nVendor the published-results validator into\n`tests/fixtures/` if pulling it live makes the test flaky\n(network, branch state). Prevents the same drift from\nhappening again the next time either side of the contract\nchanges.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a CI cross-branch contract test to prevent recurrence", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add a second generic link checker before explaining why the existing Sphinx linkcheck missed the regression", "instead": "(not specified)", "why": "duplicated docs gates create noise without proving this failure mode is covered"}, {"dont": "whitelist the originally-broken path to make linkcheck pass", "instead": "(not specified)", "why": "that defeats the purpose of adding the check"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-26T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The agent-proxy dry-run on 2026-04-28 found that\n`docs/contributing-results.md` Prerequisites #1 linked to\n`getting-started.rst` at the wrong relative path (the file lives at\n`docs/usage/getting-started.md`, not `docs/getting-started.rst`). The\nlink itself is patched in the same PR as the retrospective. The repo\nnow has Sphinx linkcheck coverage in `.github/workflows/docs.yml` and\n`make docs-linkcheck`; this TODO exists to explain why that existing\ncoverage did not prevent the source-relative link regression, then\npatch the smallest gap in the existing docs validation surface.\n\nSee `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`\nfor the originating finding.", "id": "dry-run-followup-broken-getting-started-link", "preserves": ["Docs CI total wall-clock stays within the existing budget (linkcheck added as a parallel job, not serialized after the slow build)", "Existing relative links in `docs/` that currently resolve continue to resolve (no false positives from a poorly-tuned linkcheck config)", "External-link 404s remain advisory, not blocking -- a third-party link rotting should not block a docs PR"], "prior_art": [{"concept": ".github/workflows/docs.yml \u2014 existing blocking Sphinx linkcheck job; EXTEND before adding a separate checker", "decision": "extend", "path": ".github/workflows/docs.yml"}, {"concept": "Makefile \u2014 existing `ci-linkcheck`, `docs-linkcheck`, and `docs-check` wrappers; REUSE for local verification", "decision": "reuse", "path": "Makefile"}, {"concept": "docs/conf.py \u2014 existing `linkcheck_ignore` configuration; EXTEND only for flaky external URLs, not local relative-link misses", "decision": "extend", "path": "docs/conf.py"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docs/"}, {"kind": "only_modify", "path_glob": "scripts/"}], "state": "done", "title": "Close the docs linkcheck gap for broken source-relative links", "verifications": [{"command": "rg -n 'sphinx-build -b linkcheck' .github/workflows/docs.yml Makefile", "description": "Existing Sphinx linkcheck remains wired into docs CI", "expected": "docs.yml and Makefile both contain linkcheck wiring", "item_id": "dry-run-followup-broken-getting-started-link", "last_result": null, "last_run": null, "seq": 1}, {"command": "! rg -n 'getting-started\\.rst|\\]\\(getting-started\\.rst\\)' docs/contributing-results.md", "description": "The originally-broken source-relative path no longer appears", "expected": "no matches", "item_id": "dry-run-followup-broken-getting-started-link", "last_result": null, "last_run": null, "seq": 2}, {"command": "make docs-linkcheck", "description": "Docs linkcheck passes after the coverage-gap fix", "expected": "exit 0", "item_id": "dry-run-followup-broken-getting-started-link", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "dry-run-followup-broken-getting-started-link", "needs": [], "notes": "Current prior art:\n - `.github/workflows/docs.yml` has a blocking `linkcheck` job.\n - `Makefile` has `ci-linkcheck`, `docs-linkcheck`, and\n `docs-check` wrappers.\n - `docs/index.rst` includes `contributing-results.md` in the\n toctree.\nReconstruct the failure mode from the 2026-04-28 dry-run context:\nwas the link fixed before linkcheck ran, was linkcheck absent from\nthe relevant PR gate at the time, did MyST/Sphinx not validate the\nsource-relative target shape, or did the path only break in the\nGitHub-rendered markdown view?\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reproduce why existing Sphinx linkcheck missed the original source-relative link", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-broken-getting-started-link", "needs": ["w1"], "notes": "Prefer extending the existing Sphinx/docs validation path over\nadding a second link checker. Examples:\n - If `docs/contributing-results.md` was not in the checked source\n set at the time, add an explicit regression assertion to\n `docs-validate` or the docs CI job.\n - If Sphinx validates built links but not GitHub-rendered markdown\n source links, add a small markdown relative-link check scoped to\n repo-local targets.\n - Add `markdown-link-check` only if the Sphinx path cannot cover\n this exact failure mode.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Patch the existing docs validation surface at the narrowest failing layer", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-broken-getting-started-link", "needs": ["w2"], "notes": "Run the existing docs linkcheck and the new/extended local-link\nassertion. Fix local relative-link failures in the same PR if they\nare under `docs/`; file separate follow-ups for unrelated generated\ndocs or broad external-link cleanup.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the docs link checks and fix any other latent local-link breakage", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "leave the rows in with vague qualifiers", "instead": "pick a path and apply it", "why": "that is the failure mode this TODO exists to fix"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "**Completion note:** plans/tuning sidecars are community-submittable\ncompanion files when they sit next to the primary result JSON. The\ncontributor-facing doc now explains the opt-in flags and labels both rows\nas optional.\n\n`docs/contributing-results.md` Step 2 lists `.plans.json` and\n`.tuning.json` as bundle outputs gated on `(if captured)` /\n`(if used)`. Neither phrase is defined anywhere in the doc; nothing\nexplains how a contributor opts into plan capture or applies tuning\nbefore running. First-time contributors see the rows and wonder if\ntheir bundle is incomplete because those files don't appear.\n\nDecision needed: are plans/tuning files community-submittable, or\nmaintainer-only? Either explain the opt-in or drop the rows from\nthe contributor-facing doc and surface them in advanced docs only.\n\nSurfaced by the agent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).", "id": "dry-run-followup-bundle-table-conditionals", "preserves": ["If plans/tuning files ARE community-submittable, the validate-submission CI continues to accept them"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "docs/operations/"}], "state": "done", "title": "Clarify or drop the plans.json / tuning.json rows in the Step 2 bundle table", "verifications": [{"command": "grep -cE '\\(if captured\\)|\\(if used\\)' docs/contributing-results.md", "description": "Bundle table no longer contains undefined `(if captured)` / `(if used)` qualifiers", "expected": "0", "item_id": "dry-run-followup-bundle-table-conditionals", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "dry-run-followup-bundle-table-conditionals", "needs": [], "notes": "Decision: plans.json / tuning.json sidecars are community-submittable\nwhen present next to the primary result JSON. Evidence:\n\n- `benchbox submit` discovers `.plans.json` and `.tuning.json`\n companions beside the source result, copies them into `bundle/`,\n and records per-file hashes in `manifest.companion_hashes`.\n- `scripts/validate_submission.py` treats those suffixes as companion\n files rather than primary bundles and verifies their hashes when\n declared in the manifest.\n- Unit coverage in `tests/unit/cli/commands/test_submit.py` and\n `tests/unit/scripts/test_validate_submission.py` exercises companion\n copy and hash validation behavior.\n\n`/code review W1`: no required issues. The decision is backed by the\ncurrent submission/validation contract; W2 should document the opt-in\nflags rather than removing the rows.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide whether plans/tuning files are community-submittable", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-bundle-table-conditionals", "needs": ["w1"], "notes": "Documented the accepted opt-in path in `docs/contributing-results.md`:\nuse `--capture-plans` on supported platforms for `.plans.json`,\nand use `--tuning tuned`, `--tuning auto`, or a YAML path for\nintentionally tuned submissions that may emit `.tuning.json`.\nThe Step 2 bundle table now labels both rows as optional and no longer\nuses undefined `(if captured)` / `(if used)` qualifiers.\n\n`/code review W2`: no required issues. The text preserves the\ncommunity-submittable sidecar contract from W1 while making absence of\nsidecars explicitly non-blocking for default submissions.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply the chosen change to docs/contributing-results.md", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "block on completing all 7 work units before landing any", "instead": "(not specified)", "why": "each is independently shippable; ship as ready"}, {"dont": "bundle the validator hash mismatch fix into this TODO", "instead": "(not specified)", "why": "that's release-blocker scope and tracked separately as `dry-run-followup-validator-hash-contract-mismatch`"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-29T00:00:00Z", "deferrals": [], "deps": [], "description": "Seven smaller friction items surfaced by the 2026-04-29 Codex\nagent-proxy dry-run. Each work unit is independently verifiable and\nsmall; consolidated into one TODO to avoid TODO sprawl. None of these\nare blocking by themselves, but cumulatively they make the difference\nbetween a smooth and a frustrating first submission.\n\nSee `_project/handoffs/external-dry-run-retrospective-2026-04-29.md`\nfor the full report and Codex's verbatim feedback (Appendix A there).\n\nNote: the release-blocker hash-contract mismatch from the same\ndry-run is tracked separately as\n`dry-run-followup-validator-hash-contract-mismatch.yaml` (Critical).\nThis TODO is polish; that one is correctness.", "id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "preserves": ["Existing user-facing behavior on the success path -- these are output / UX tweaks only, not flow changes", "Hard-failure red \u274c for actually-unrecoverable errors -- don't make every error look amber and dilute the signal", "The canonical `uv run -- python` examples in the validation block (those are unambiguous and contributor-tested)"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "README.md (w1: invocation consistency)"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py (w4: drop Interactive header at line 706)"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py (w2: print next commands)"}, {"kind": "only_modify", "path_glob": "benchbox/cli/output.py (w3: downgrade red `Missing tables` at line 172)"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/power_test.py (w7: reword stream_id warning at lines 172-176)"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md (w1: invocation consistency)"}, {"kind": "only_modify", "path_glob": "docs/platforms/dataframe.md (w5: rename 'Platform Availability' at line 358)"}, {"kind": "only_modify", "path_glob": "docs/usage/ (w1: invocation consistency)"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/ (regression tests for w2, w3, w4, w6)"}], "state": "done", "title": "CLI UX polish + doc consistency fixes from 2026-04-29 Codex dry-run", "verifications": [{"command": "grep -rEc '^[^#]*\\buv run benchbox\\b(?! --)' docs/contributing-results.md docs/usage/ README.md 2>/dev/null | awk -F: '{s+=$2}END{print s}'", "description": "Doc invocation consistency: zero non-canonical `uv run benchbox` (without `--`) in contributor docs", "expected": "0 (assuming `uv run -- benchbox` is the canonical form chosen in w1)", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 1}, {"command": "rm -rf /tmp/bb-test && uv run -- benchbox submit --last --output /tmp/bb-test 2>&1 | grep -c 'published-results'", "description": "`benchbox submit` output mentions the PR target branch inline", "expected": ">= 1", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 2>&1 | grep -c 'Interactive Benchmark Runner'", "description": "`benchbox run` non-interactive does not print the interactive header", "expected": "0", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --platform duckdb --benchmark tpch --scale 0.01 --force datagen 2>&1 | grep -E '\\[red\\][^[]*Missing tables|\u274c[^\\n]*Missing tables' | wc -l", "description": "Missing-tables auto-recovery does not print in red", "expected": "0", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 4}, {"command": "grep -cE 'timing only|per TPC-H spec' benchbox/core/tpch/power_test.py", "description": "Stream-id warning explicitly mentions 'timing only' or 'per TPC-H spec'", "expected": ">= 1", "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "DECISION: `uv run -- benchbox` is the canonical form for\ncontributor-facing submission docs. Bare `benchbox` remains\nacceptable in broad reference examples that explicitly assume an\ninstalled CLI.\n\nChanges:\n - Normalized the active submission workflow in\n `docs/contributing-results.md` to `uv run -- benchbox`.\n - Updated the public-results contribution paragraph in\n `README.md` to use `uv run -- benchbox submit`.\n\n`/code review W1`: no required issues. Scope stayed focused on\ncontributor-facing submission paths rather than rewriting every\nbroad README/reference example.\n\nVerification:\n - `grep -rEc '^[^#]*\\buv run benchbox\\b(?! --)' docs/contributing-results.md docs/usage/ README.md 2>/dev/null | awk -F: '{s+=$2}END{print s}'`\n reports no matches.\n - `git diff --check` passed.\n\nCodex re-read steps to figure out which form to use. Pick one\ncanonical form for contributor-facing docs and normalize.\nRecommend `uv run -- benchbox` (matches the repo's uv-first\nconvention, matches the `uv run -- python` examples in the\nvalidation block). Bare `benchbox` only where the doc explicitly\nassumes a system install.\nFiles to scan: docs/contributing-results.md, docs/usage/,\nREADME.md, docs/development/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit doc invocation consistency: pick canonical form between `benchbox`, `uv run benchbox`, `uv run -- benchbox`", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing implementation already satisfies this work item:\n - `_print_submission_summary()` prints the concrete `cp`\n command for the bundle and manifest, the inventory command,\n the validation command, the PR title, and the\n `published-results` target branch.\n - `tests/unit/cli/commands/test_submit.py` includes the pinned\n W2 regression\n `test_submit_prints_next_steps_with_pr_target_branch`.\n\n`/code review W2`: no required issues. The code path is covered\nby the focused submit unit suite, and no production-code patch was\nneeded for this slice.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/commands/test_submit.py -q`\n passed: 28 passed.\n\nCodex's recommendation #2: after writing the bundle, print the\nexact next commands the contributor should run, with paths\nfilled in:\n - cp results-data/bundles/\n - uv run -- python scripts/generate_corpus_inventory.py --write\n - uv run -- python scripts/validate_submission.py results-data/bundles/\n - PR title: `results: sf`\n - PR target branch: `published-results`\nReduces re-read load and removes the \"where does this go?\"\nfriction. The information is already in the in-bundle\nCONTRIBUTING.md after PR #43; this surfaces it inline at submit\ntime so contributors don't have to switch contexts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make `benchbox submit` print the exact next commands", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing production output already renders missing-table details\nas yellow/recoverable while preserving red output for hard\nfailures such as inaccessible tables.\n\nChange:\n - Added\n `test_missing_tables_validation_message_is_recoverable_amber`\n in `tests/unit/cli/test_cli_output.py` to pin the amber\n missing-table branch and the red inaccessible-table branch.\n\n`/code review W3`: no required issues. The test is scoped to the\nexact formatter behavior and does not weaken hard-failure red\noutput.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/test_cli_output.py::TestConsoleResultFormatter::test_missing_tables_validation_message_is_recoverable_amber -q`\n passed: 1 passed.\n\nCodex hit `\u274c Missing tables...` printed in red, then the runner\nautomatically deleted+rebuilt the DuckDB file and the run\nsucceeded. The user-visible result was success, but the red \u274c\nmade it look like a failure mid-run. Downgrade to amber/yellow\nwith a \"rebuilding...\" spinner; reserve red \u274c for actually\nunrecoverable errors.\n\nAudit findings (2026-04-29) \u2014 emit sites:\n - `benchbox/cli/output.py:172` -- `[red]\u2022 Missing tables:[/red]`\n (this is the most likely red \u274c Codex saw)\n - `benchbox/platforms/base/result_capture.py:1372` -- already\n uses `\u26a0\ufe0f` (amber); not the offending site\n - `benchbox/platforms/base/validation.py:174` -- adds error\n to a result object; check if it propagates to the [red] path\n - `benchbox/utils/data_validation.py:426` -- emit helper\n - `benchbox/core/validation/engines.py:432` -- error append\n\nTrace the auto-recovery path: which caller catches the\nvalidation error, deletes+rebuilds the DuckDB file, and re-runs?\nDowngrade `cli/output.py:172` to amber/yellow IF the caller is\nrecoverable; keep red only for hard-fail paths.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Downgrade `benchbox run` recoverable Missing-tables from red \u274c to amber recovery message", "wid": "w3"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing production code already gates the header on\n`sys.stdin.isatty()` and the absence of explicit platform +\nbenchmark args.\n\nChange:\n - Added focused tests in\n `tests/unit/cli/test_run_command_interactive_paths.py` for\n suppressing the header with explicit run args and preserving\n it for a prompted TTY run.\n\n`/code review W4`: no required issues. The regression tests\ncover the requested suppression without removing the true\ninteractive banner.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/test_run_command_interactive_paths.py::test_interactive_header_suppressed_for_explicit_run_args tests/unit/cli/test_run_command_interactive_paths.py::test_interactive_header_shown_for_prompted_tty_run -q`\n passed: 2 passed.\n\nCodex called this out: showing 'Interactive Benchmark Runner'\nwhen running with explicit flags (no interactive prompt) is\nmisleading. Suppress the header when stdin is not a TTY OR\nwhen explicit `--platform` / `--benchmark` args are provided.\n\nAudit findings (2026-04-29) \u2014 emit site:\n - `benchbox/cli/commands/run.py:706` --\n `Text(\"BenchBox Interactive Benchmark Runner\", style=\"bold blue\")`\nSingle-site fix; gate on `sys.stdin.isatty() and not (platform\nand benchmark)` or equivalent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Drop 'Interactive Benchmark Runner' header in non-interactive `benchbox run`", "wid": "w4"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing docs already use the CLI-matching `Available Databases`\nheading in `docs/platforms/dataframe.md`, and the other audited\ndocs also align with the CLI output.\n\nEvidence:\n - `docs/platforms/dataframe.md:358` is `## Available Databases`.\n - `benchbox/cli/system.py` and `benchbox/cli/database.py` print\n or title the surface as `Available Databases`.\n - `rg -n \"Platform Availability|Available Databases|Checking Platform Availability|benchbox profile\" ...`\n found no remaining `Platform Availability` doc mismatch.\n\n`/code review W5`: no required issues. No docs patch was needed\nbeyond recording the resolved state.\n\nThe doc references a 'Platform Availability' surface; the actual\nCLI prints 'Available Databases'. Either add the doc-claimed\ntable to the CLI, or update docs to match what the CLI prints.\nLatter is cheaper.\n\nAudit findings (2026-04-29) \u2014 file pins:\n - Doc: `docs/platforms/dataframe.md:358` (`## Checking Platform\n Availability`) -- main mismatch surface\n - Doc: `docs/usage/intelligent-guidance.md:49,83` -- already\n uses 'Available Databases' (ASCII tree examples); aligned\n - CLI: `benchbox/cli/system.py:42` -- prints\n `[cyan]Available Databases:[/cyan]`\n - CLI: `benchbox/cli/database.py:395,504` --\n `Table(title=\"Available Databases\")`\nRecommended: rename the doc heading + body in\n`docs/platforms/dataframe.md:358` to 'Available Databases'\nand align the prose.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sync `benchbox profile` docs ('Platform Availability' \u2192 actual 'Available Databases')", "wid": "w5"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing implementation already routes dry-run and real submit\noutput through the shared `_print_submission_summary()` helper.\nDry-run only changes the header and the trailing\n`(dry run; no files written)` footer; the file list and next-step\ncommand shape are shared.\n\nExisting pinned test:\n - `test_submit_dry_run_matches_real_run_shape` in\n `tests/unit/cli/commands/test_submit.py`.\n\n`/code review W6`: no required issues. The shared helper avoids\ndrift between dry-run and real submit output, and the dry-run path\nstill asserts no files are written.\n\nVerification:\n - `uv run -- python -m pytest tests/unit/cli/commands/test_submit.py::test_submit_dry_run_matches_real_run_shape -q`\n passed: 1 passed.\n\nCodex noted the dry-run path output format and real output\nformat differ slightly. Make `--dry-run` print the same\nfile/manifest/contributing list shape as the real run, just\nwith a \"(dry run; no files written)\" footer. Today they're\nsimilar but the dry-run formatting is its own one-off.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align `benchbox submit --dry-run` output format with real submit output", "wid": "w6"}, {"evidence": null, "item_id": "dry-run-followup-cli-ux-and-doc-polish-2026-04-29", "needs": [], "notes": "Existing warning text already includes the requested clarification:\nstream 0 is validated against the answer set, and streams > 0 are\nrun for timing only per TPC-H spec.\n\n`/code review W7`: no required issues. The behavior remains\nunchanged: auto-selected validation is disabled for nonzero\nstreams, while explicit validation modes are not silently altered\nby this wording-only path.\n\nVerification:\n - `grep -cE 'timing only|per TPC-H spec' benchbox/core/tpch/power_test.py`\n returned 2.\n\nCodex's run hit a warning that validation was disabled for\nstream_id != 0 and was unsure whether the result was actually\nfully validated.\n\nAudit findings (2026-04-29) \u2014 emit site:\n - `benchbox/core/tpch/power_test.py:172-176`:\n if stream_id != 0 and validation and validation_mode is None:\n actual_validation_mode = \"disabled\"\n self.logger.warning(\n \"\u26a0\ufe0f TPC-H answer sets are only available for stream 0. \"\n \"Disabling validation for stream_id != 0.\"\n )\n\nThe warning IS correct (TPC-H answer sets only exist for stream 0),\nbut Codex's confusion shows the wording leaves room for \"did my\nresult get validated or not?\" doubt. Reword to explicitly state:\n\"Stream 0 was validated against the answer set; streams > 0 are\nrun for timing only (per TPC-H spec).\" Suppression is the wrong\ncall -- the warning protects users running multi-stream from\nclaiming false validation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Clarify the `stream_id != 0 validation disabled` TPC-H warning", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the writer without updating every reader in the same PR", "instead": "audit consumers first (w1)", "why": "mid-flight readers will fail to find the manifest"}, {"dont": "silently break already-merged manifests in results-data/bundles/", "instead": "if any exist with the old name, support both during a transition window", "why": "that erodes trust in already-published submissions"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": ["dry-run-followup-package-canonical-contributing"], "description": "`benchbox submit` always writes the submission manifest as the literal\nfilename `submission-manifest.json` (constant\n`SUBMISSION_MANIFEST_FILENAME` in\n`benchbox/core/explorer_pipeline/pipeline.py:39`, also used in\n`benchbox/cli/commands/submit.py:338`). The doc instruction is \"copy\n`submission/submission-manifest.json` alongside the bundle files\" into\n`results-data/bundles/`. `results-data/bundles/` currently contains\nzero manifest files, but two community PRs landing in the same window\nwould conflict on the same filename. The doc names no convention for\nresolving the collision.\n\nProposal: emit `.manifest.json` so the manifest's\nfilename inherits the bundle's uniqueness (timestamp + hash). Update\ndoc, in-bundle CONTRIBUTING.md, validators, and the inventory generator\nif any of them filter by exact filename.\n\nSurfaced by the agent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).", "id": "dry-run-followup-manifest-filename-convention", "preserves": ["Backwards-compat for any already-merged community submissions that used the old filename", "The trust-label resolution path in `explorer_pipeline/pipeline.py` continues to find manifests for existing bundles"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/validate-submission.yml"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/pipeline.py"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "scripts/generate_corpus_inventory.py"}, {"kind": "only_modify", "path_glob": "scripts/validate_submission.py"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Per-bundle manifest filenames to avoid PR collision in results-data/bundles/", "verifications": [{"command": "uv run -- benchbox submit --last --output /tmp/dry-run-mft && ls /tmp/dry-run-mft | grep -c '\\.manifest\\.json$'", "description": "Submit emits per-bundle manifest filenames", "expected": "1", "item_id": "dry-run-followup-manifest-filename-convention", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -c 'manifest.json' scripts/generate_corpus_inventory.py", "description": "Inventory generator resolves manifests by glob", "expected": ">= 1", "item_id": "dry-run-followup-manifest-filename-convention", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": [], "notes": "Grep for `submission-manifest.json` and `SUBMISSION_MANIFEST_FILENAME`\nacross:\n - benchbox/cli/commands/submit.py (writer)\n - benchbox/core/explorer_pipeline/pipeline.py (reader, trust-label resolver)\n - scripts/validate_submission.py\n - scripts/generate_corpus_inventory.py\n - .github/workflows/validate-submission.yml (if it references the filename)\nDocument everywhere the filename is hard-coded so the rename is\natomic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit consumers of submission-manifest.json filename", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": ["w1"], "notes": "Update the writer to emit `.manifest.json`. Update\nreaders to resolve manifests by glob (`*.manifest.json`) rather\nthan by literal name. Update the inventory generator's filter\nrules if it currently filters by exact filename.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rename to .manifest.json and update consumers", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": ["w2"], "notes": "`docs/contributing-results.md` Step 2 / Step 3.3 both name the\nfile. The in-bundle CONTRIBUTING.md is being rewritten in\n`dry-run-followup-package-canonical-contributing` \u2014 coordinate\nso both land consistent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs (canonical + in-bundle) to reflect the new filename", "wid": "w3"}, {"evidence": null, "item_id": "dry-run-followup-manifest-filename-convention", "needs": ["w2"], "notes": "Two consecutive `benchbox submit --output ./submission` calls\nagainst different result files should not overwrite each other's\nmanifests. Test asserts both manifests exist after the second\ncall.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test for two consecutive submits", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "remove the in-bundle file entirely", "instead": "ship a checklist + canonical URL instead", "why": "contributors offline / on planes still need a packaged copy of the steps"}, {"dont": "silently keep the in-bundle template diverged from the canonical doc", "instead": "regenerate at build time or include a clear pointer", "why": "that is the exact failure mode this TODO exists to fix"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "`benchbox submit` writes a 16-line CONTRIBUTING.md into every submission\nbundle (hardcoded as `_CONTRIBUTING_TEXT` at\n`benchbox/cli/commands/submit.py:29-46`), and the CLI's \"Next step\"\nmessage tells contributors to follow it. That file does NOT include\nthree steps the canonical `docs/contributing-results.md` requires:\n\n 1. Regenerate `corpus-inventory.json` before commit.\n 2. Target the `published-results` branch (not `main`).\n 3. Optionally validate the bundle locally with\n `scripts/validate_submission.py`.\n\nA contributor who reads only the in-bundle file will open a PR against\n`main` with a stale inventory. CI catches it, but that's wasted\nmaintainer cycles and a worse first-time UX. Surfaced by the\nagent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).\n\nFix options (pick one):\n A. Generate the in-bundle file from `docs/contributing-results.md`\n at package-build time so the two cannot drift.\n B. Replace the in-bundle file with a one-page checklist that points\n to the canonical doc URL and lists the three required commands\n inline.", "id": "dry-run-followup-package-canonical-contributing", "preserves": ["The CLI's existing 'Next step' message still points contributors at the in-bundle file", "`benchbox submit --output` continues to package a self-contained bundle directory"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/commands/test_submit.py"}], "state": "done", "title": "Make benchbox submit's packaged CONTRIBUTING.md match docs/contributing-results.md", "verifications": [{"command": "uv run -- benchbox submit --last --output /tmp/dry-run-canonical && grep -cE 'published-results|generate_corpus_inventory|validate_submission|docs\\.benchbox\\.dev' /tmp/dry-run-canonical/CONTRIBUTING.md", "description": "Packaged CONTRIBUTING.md mentions all four required items", "expected": ">= 4", "item_id": "dry-run-followup-package-canonical-contributing", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/commands/test_submit.py -q", "description": "Submit unit tests still pass", "expected": "All tests pass", "item_id": "dry-run-followup-package-canonical-contributing", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-package-canonical-contributing", "needs": [], "notes": "A vs B is a real architectural call:\n - A: harder-to-drift but introduces a build-time read of\n `docs/contributing-results.md` from `submit.py`. Requires\n packaging the doc into the wheel or reading it relative to\n the repo root.\n - B: simpler; ships a checklist that names the four required\n steps and points contributors to docs.benchbox.dev for the\n full doc. Requires updating the checklist whenever the\n canonical doc changes -- but the checklist is small and\n stable.\nDefault recommendation: option B unless wheel-packaging the doc\nis already cheap. Document the choice + rationale in a one-line\ncomment above `_CONTRIBUTING_TEXT` so future maintainers don't\nre-litigate it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pick option A (regenerate from canonical doc at build time) or option B (static checklist + canonical URL)", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-package-canonical-contributing", "needs": ["w1"], "notes": "Touch `benchbox/cli/commands/submit.py` (`_CONTRIBUTING_TEXT`).\nThe new file content MUST include all four:\n - `published-results` as the PR base branch (not `main`)\n - `generate_corpus_inventory.py --write` before commit\n - `validate_submission.py` for local pre-push check\n - canonical doc URL (https://docs.benchbox.dev/contributing-results\n or wherever the canonical doc lives at land time)\nCoordinate with `dry-run-followup-submitted-by-flag` and\n`dry-run-followup-manifest-filename-convention` since both\nwill edit the same template after this lands.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the chosen option in benchbox/cli/commands/submit.py", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-package-canonical-contributing", "needs": ["w2"], "notes": "Add to `tests/unit/cli/commands/test_submit.py`: run\n`benchbox submit --last --output ` against a fixture result,\nread the resulting CONTRIBUTING.md, assert presence of\n`published-results`, `generate_corpus_inventory`,\n`validate_submission`, and the canonical doc URL. The test\npins the contract independent of which option (A or B) was\nchosen.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test asserting the in-bundle file mentions all four required items", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the default fallback source without updating the doc table in the same PR", "instead": "keep code and doc consistent", "why": "contributors verifying their manifest will look for whichever field the doc names"}, {"dont": "make empty submitted_by a hard error", "instead": "warn and continue", "why": "anonymous submissions are valid"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": ["dry-run-followup-package-canonical-contributing"], "description": "`docs/contributing-results.md` describes the submission manifest as\ncontaining a \"contributor\" field. The CLI does fall back to\n`git config user.name` via `_get_git_username()` at\n`benchbox/cli/commands/submit.py:49-60`, but when that returns empty\n(no git user.name configured \u2014 e.g. fresh sandbox), the manifest's\n`submitted_by` is silently set to `\"\"` with no warning.\n\nTwo improvements:\n 1. Warn loudly when the fallback returns empty, telling the\n contributor exactly how to set it.\n 2. Add `--submitted-by TEXT` for explicit override (CI, automation,\n contributors who prefer not to set git config).\n\nSurfaced by the agent-proxy dry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).\nThe Cowork sandbox had no `git config user.name` set, which is what\ntriggered the silent-empty path that this TODO addresses.", "id": "dry-run-followup-submitted-by-flag", "preserves": ["Existing `_get_git_username()` fallback behavior remains the default when no flag is passed", "The empty-submitted_by case stays a warning, not a hard failure -- some contributors may legitimately submit anonymously"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/cli/commands/submit.py"}, {"kind": "only_modify", "path_glob": "docs/contributing-results.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/commands/test_submit.py"}], "state": "done", "title": "benchbox submit: warn on empty submitted_by, add --submitted-by override", "verifications": [{"command": "uv run -- benchbox submit --help | grep -c -- '--submitted-by'", "description": "--submitted-by flag is in the help output", "expected": "1", "item_id": "dry-run-followup-submitted-by-flag", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/cli/commands/test_submit.py -q", "description": "Submit unit tests pass", "expected": "All tests pass", "item_id": "dry-run-followup-submitted-by-flag", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-submitted-by-flag", "needs": [], "notes": "In the manifest-write path, if `submitted_by` would be empty,\nprint a warning to stderr: \"submitted_by is empty \u2014 set\n`git config user.name ` or pass --submitted-by NAME\".\nDo not fail the command; the empty field is a soft issue, not\na hard error.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Warn when _get_git_username returns empty and no --submitted-by was passed", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-submitted-by-flag", "needs": ["w1"], "notes": "Add `--submitted-by TEXT` option to `benchbox submit`. Precedence:\nexplicit flag > `git config user.name` > empty (with warning).\nDocument in `docs/contributing-results.md` and in the in-bundle\nCONTRIBUTING.md (which is being rewritten in\n`dry-run-followup-package-canonical-contributing` \u2014 coordinate\nthe doc change there).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add --submitted-by TEXT flag and plumb into manifest write", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-submitted-by-flag", "needs": ["w2"], "notes": "Test cases: (a) --submitted-by overrides git config, (b) git\nconfig user.name is used when no flag, (c) warning is emitted\nwhen both are empty. Mock `_get_git_username` for determinism.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add unit tests for the three precedence paths", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "name a different JSON key than the bundle actually emits", "instead": "check a real bundle's top-level keys before naming one", "why": "a contributor grepping their bundle should find what the doc says is there"}, {"dont": "remove the `uv run` examples in favor of plain `python`", "instead": "add the fallback as an alternative line", "why": "most contributors do have uv and removing it would break the canonical Python tooling pattern"}], "approach": null, "blocked_reason": null, "category": "Documentation", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-28T00:00:00Z", "completed_pr": null, "created_at": "2026-04-28T00:00:00Z", "deferrals": [], "deps": [], "description": "Two small contributor-facing clarifications in\n`docs/contributing-results.md`, both surfaced by the agent-proxy\ndry-run on 2026-04-28\n(see `_project/handoffs/external-dry-run-retrospective-2026-04-28.md`).\n\n1. The \"Running Validation Locally\" block uses `uv run -- python \u2026`\n for every example. In environments where `uv` is not installed or\n is slow to bootstrap a venv, plain `python scripts/validate_submission.py \u2026`\n works identically. Add a one-line \"no uv? use python directly\"\n note. (Cowork's sandbox specifically hit a 45s `uv run` stall\n and had to fall back to `python3` \u2014 a real-world contributor on\n a constrained machine could hit the same.)\n\n2. The \"Quality Expectations\" section says \"Schema v2 format - only\n the current schema version is accepted\". The actual JSON key is\n `version` (string, currently `\"2.1\"`), not `schema_version`. Name\n the key explicitly so contributors can grep their bundle to\n verify compliance.", "id": "dry-run-followup-uv-fallback-and-schema-key", "preserves": ["The Quality Expectations section still names schema v2 as the accepted format", "The existing `uv run -- python` examples remain in the doc as the canonical invocation -- the fallback is added as an alternative, not a replacement"], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "docs/contributing-results.md"}], "state": "done", "title": "Local-validation block: document non-uv fallback and schema-version key name", "verifications": [{"command": "grep -cE 'top-level `version`|JSON key `version`|`version` field' docs/contributing-results.md", "description": "Doc names the top-level version JSON key explicitly", "expected": ">= 1", "item_id": "dry-run-followup-uv-fallback-and-schema-key", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -ciE 'without uv|no uv|don.t have uv' docs/contributing-results.md", "description": "Doc mentions a non-uv fallback for the validation block", "expected": ">= 1", "item_id": "dry-run-followup-uv-fallback-and-schema-key", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-uv-fallback-and-schema-key", "needs": [], "notes": "Add a sentence: \"If you don't have `uv`, replace `uv run -- python`\nwith `python` \u2014 the scripts have no uv-specific dependencies.\"\nPlace it once at the top of the local-validation block.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a non-uv fallback note to Running Validation Locally", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-uv-fallback-and-schema-key", "needs": [], "notes": "Change \"Schema v2 format - only the current schema version is\naccepted\" to \"Schema v2 format (top-level `version` field,\ncurrently `\\\"2.1\\\"`)\". Optionally add a one-line how-to:\n`python3 -c \"import json; print(json.load(open('bundle.json'))['version'])\"`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Name the version JSON key explicitly under Quality Expectations", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "revert develop's writer to per-directory hash", "instead": "fix the validator instead", "why": "per-file is the right contract for individually-addressable bundles"}, {"dont": "skip the cross-branch contract test (w3)", "instead": "the test is the regression-prevention story for this whole class of bug", "why": "the same drift will happen again the next time either side changes"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-30T00:00:00Z", "completed_pr": null, "created_at": "2026-04-29T00:00:00Z", "deferrals": [], "deps": [], "description": "The `scripts/validate_submission.py` on the `published-results` branch\n(pinned at v0.2.1, commit `0182c9556`) uses a per-DIRECTORY hash\nalgorithm (`hashlib.sha256()` over `path.encode() + file.read_bytes()`\nfor every file in `bundle_dir` excluding the manifest). The same\nscript on `develop` (post-PR #33) uses a per-FILE hash (just\n`sha256(bundle_file)` plus a `companion_hashes` map). The writer in\n`develop`'s `submit.py` also uses per-file. Result: a bundle produced\nby current `develop`'s `benchbox submit` will fail CI on\n`published-results` with `Bundle hash mismatch: manifest says XXXX,\ncomputed YYYY`.\n\nThis was confirmed empirically during the 2026-04-29 Codex\nagent-proxy dry-run:\n - Codex's manifest: `bundle_hash: 799c80a2...` (per-file SHA-256)\n - Validator on published-results computed: `8a89ab6f...` (per-dir)\n - `shasum -a 256` on the actual bundle file: `799c80a2...` (matches\n the manifest exactly)\n\nForensic and full reproduction at\n`_project/handoffs/external-dry-run-retrospective-2026-04-29.md`\n(Appendix B).\n\nThis is a release-blocker for the community submission flow: until\nthe validator on `published-results` matches the writer's contract,\nno contributor can successfully submit a result regardless of how\nclosely they follow the docs. The bug is invisible to maintainers\nwho only run the validator inside `develop`, because both sides\nagree there.", "id": "dry-run-followup-validator-hash-contract-mismatch", "preserves": ["Backwards-compat with already-merged community submissions in results-data/bundles/ (zero exist today, so this is theoretical)", "develop's per-file hash contract (per-file is the right design -- bundles are individually addressable in results-data/bundles/)"], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": "scripts/validate_submission.py (on published-results branch)"}, {"kind": "only_modify", "path_glob": "tests/fixtures/ (only if w3 vendors the validator instead of pulling live)"}, {"kind": "only_modify", "path_glob": "tests/integration/test_cross_branch_validator_contract.py (new file, w3)"}], "state": "done", "title": "Sync published-results validator with develop's per-file hash contract (release-blocker)", "verifications": [{"command": "git fetch --quiet origin published-results && \\\nTMP=$(mktemp) && \\\ngit show origin/published-results:scripts/validate_submission.py > \"$TMP\" && \\\npython3 \"$TMP\" /Users/joe/Developer/BenchBox/submission/bundle/tpch_sf001_duckdb_sql_20260428_205018_e4f24ee2.json 2>&1 | tail -3; \\\nrc=$?; rm -f \"$TMP\"; exit $rc\n", "description": "Validator on published-results passes against a develop-submitted bundle", "expected": "0 error(s), 0 warning(s)", "item_id": "dry-run-followup-validator-hash-contract-mismatch", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/integration/test_cross_branch_validator_contract.py -q", "description": "Cross-branch contract test (added by w3) passes", "expected": "All tests pass (test file is created by w3; this verification only meaningful after w3 lands)", "item_id": "dry-run-followup-validator-hash-contract-mismatch", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "dry-run-followup-validator-hash-contract-mismatch", "needs": [], "notes": "DECISION: choose path A -- update the `published-results`\nvalidator directly.\n\nRationale:\n - This is a production submission-flow blocker independent of\n the Python package release cadence: GitHub Actions executes\n `scripts/validate_submission.py` from the PR checkout of the\n `published-results` branch.\n - `develop` already has the correct per-file writer and\n validator contract. Waiting for the release-flow dry run would\n delay contributor submissions without improving the contract.\n - The branch-specific validator fix can be verified immediately\n by running the `published-results` validator against the\n Codex dry-run bundle cited in the retrospective.\n\n`/code review W1`: no required issues. The rejected path B remains\nthe longer-term release hygiene path, but it is not the fastest\nunblock for external contributors.\n\nTwo paths:\nA. Cherry-pick the post-PR-#33 `scripts/validate_submission.py`\n (and any related changes) onto `published-results`. Quickest\n unblock; doesn't depend on a release.\nB. Cut the next benchbox release (per\n `release-flow-dry-run-rc1`) which propagates the validator\n to v0.x.y, then sync `published-results` from the release tag.\nRecommend A for speed (the validator is a script, not a wheel\nartefact -- it runs from the PR's checkout of published-results\nregardless of whether benchbox is installed). B is cleaner long\nterm but blocks on the release-flow dry-run already in flight.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Decide path: cherry-pick develop's validator to published-results, or land a release first", "wid": "w1"}, {"evidence": null, "item_id": "dry-run-followup-validator-hash-contract-mismatch", "needs": ["w1"], "notes": "Applied path A in branch\n`fix/published-results-validator-manifest` and opened PR #87\nagainst `published-results`:\nhttps://github.com/joeharris76/BenchBox/pull/87\n\nScope of the branch-specific patch:\n - `scripts/validate_submission.py` now skips `*.manifest.json`\n during bundle discovery.\n - The validator prefers `.manifest.json`, while\n preserving the legacy `submission-manifest.json` fallback.\n\nVerification run against the Codex dry-run bundle cited in the\nretrospective:\n - legacy `submission-manifest.json` layout: 0 errors,\n 0 warnings.\n - per-bundle `.manifest.json` layout: 0 errors,\n 0 warnings.\n\n`/code review W2`: no required issues. The patch is branch-local,\npreserves legacy manifests, and does not touch the writer.\n\nAfter sync, re-run the validator on `published-results` against\na freshly-submitted bundle:\n cd \n python3 scripts/validate_submission.py /path/to/bundle.json\nExpected: 0 errors, 0 warnings.\nUse the artefacts Codex left at\n`/Users/joe/Developer/BenchBox/submission/` (or regenerate via\n`benchbox submit --last`) for the round-trip.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply chosen path; verify with a round-trip test", "wid": "w2"}, {"evidence": null, "item_id": "dry-run-followup-validator-hash-contract-mismatch", "needs": ["w2"], "notes": "Already landed before this TODO cleanup:\n - `tests/integration/test_cross_branch_validator_contract.py`\n - `tests/fixtures/published_results_validator.py`\n\nEvidence:\n - `git log -- tests/integration/test_cross_branch_validator_contract.py tests/fixtures/published_results_validator.py`\n shows the contract test and vendored validator landed in\n prior dry-run follow-up commits (#49, #51, #45).\n - `cmp -s scripts/validate_submission.py tests/fixtures/published_results_validator.py`\n returns 0, so the vendored validator matches the develop\n validator byte-for-byte.\n - The test covers writer output, companion hashes, symlink\n rejection, missing-manifest-field UX, and validator drift.\n\n`/code review W3`: no required issues. The original TODO\nsuggested fetching `origin/published-results` live; the landed\ntest intentionally vendors the validator to keep the fast lane\ndeterministic and avoid network/branch-state flake.\n\nAdd `tests/integration/test_cross_branch_validator_contract.py`\nthat validates the writer's emitted hashes match what the\nvalidator on `published-results` computes. Implementation shape:\n - Run `benchbox submit` to produce a bundle + manifest in a\n temp dir.\n - Fetch `scripts/validate_submission.py` from\n `origin/published-results` (via `git show\n origin/published-results:scripts/validate_submission.py`)\n and exec it against the freshly-emitted bundle.\n - Assert no hash mismatch.\nVendor the published-results validator into\n`tests/fixtures/` if pulling it live makes the test flaky\n(network, branch state). Prevents the same drift from\nhappening again the next time either side of the contract\nchanges.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a CI cross-branch contract test to prevent recurrence", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Blog Revision", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2025-12-15T00:00:00Z", "deferrals": [], "deps": [], "description": "Verify the claim in duckdb-architecture.md that DuckDB processes TPC-H SF10\n\"in under 10 seconds\" on a modern laptop.\n\nCurrent text (line 17-19):\n> \"DuckDB regularly matches or beats traditional data warehouses on TPC-H\n> benchmarks. This isn't marketing - on a modern laptop, DuckDB processes\n> TPC-H SF10 (60 million lineitem rows) in under 10 seconds.\"\n\nThis claim has no citation and should be verified with our own benchmark.\nQuick validation run, update the draft with actual measured time.\n\nReference draft: _blog/platform-deep-dives/drafts/duckdb-architecture.md", "id": "duckdb-architecture-verification", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "[DuckDB Architecture] Verify 'under 10 seconds' TPC-H claim", "verifications": [], "work": [{"evidence": null, "item_id": "duckdb-architecture-verification", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm local benchmark environment and exact DuckDB TPC-H SF10 run configuration", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-architecture-verification", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Execute baseline DuckDB TPC-H SF10 benchmark command and capture raw timing output", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-architecture-verification", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run two additional trials and record per-run timings plus median runtime", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-architecture-verification", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Record hardware context used for the benchmark (CPU, RAM, storage, OS)", "wid": "w4"}, {"evidence": null, "item_id": "duckdb-architecture-verification", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update draft claim with measured runtime and variability context from captured runs", "wid": "w5"}, {"evidence": null, "item_id": "duckdb-architecture-verification", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add reproducibility command and run notes for readers", "wid": "w6"}, {"evidence": null, "item_id": "duckdb-architecture-verification", "needs": ["w4", "w5", "w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cross-check all updated performance statements against captured benchmark evidence", "wid": "w7"}, {"evidence": null, "item_id": "duckdb-architecture-verification", "needs": ["w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Final editorial pass for consistency, citation clarity, and reproducibility wording", "wid": "w8"}], "worktree": "blog-content"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "inline the smoke into `tests/`", "why": "it is a CI-time check, not a unit test, and pulls a network resource (the community extension) on every run."}, {"INSTEAD": "(not specified)", "dont": "pin the community-extension commit by editing `pyproject.toml`", "why": "the extension is loaded by DuckDB itself via `INSTALL ... FROM community`, not by uv. The ADR may choose vendoring as a separate mechanism."}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-06T00:00:00Z", "completed_pr": null, "created_at": "2026-05-05T00:00:00Z", "deferrals": [], "deps": [], "description": "Blind-spot `2026-05-02-155524` documented that the DuckDB community\ndatasketches extension (commit 2e38607, downloaded on-demand by\n`INSTALL datasketches FROM community`) silently dropped the `theta`\nand `frequent_items` families between PR #114 morning and the audit\nlater that day. Four sketch ops referencing `datasketch_theta` /\n`datasketch_frequent_items` are still in\n`benchbox/core/write_primitives/catalog/operations.yaml` and fail\nend-to-end. CI does not currently exercise these aggregates beyond\nloader unit tests.\n\nTwo work threads close the gap:\n\n1. Fast smoke that fails the build if any planned-on family (`theta`,\n `frequent_items`, `cpc`, `req`, `kll`, `hll`) disappears from a\n freshly-installed extension.\n2. Decision artifact (ADR) on vendoring vs HLL substitution. Current\n state of evidence: PR #182 already added CPC + REQ family ops, so\n HLL substitution is not the only path. Vendoring in\n `_binaries/datasketches/` gives version control but adds artifact\n tree weight.", "id": "duckdb-datasketches-family-ci-smoke", "preserves": ["Existing op definitions in `operations.yaml` are not removed -- the ADR may keep them and let the smoke gate the build instead.", "PR #182 CPC + REQ family ops continue to load and run on DuckDB.", "The smoke does not block the build for transient network failures fetching the extension -- network errors are reported but not fatal."], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "_binaries/datasketches/"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/write-primitives-sketch-functions.md"}, {"kind": "only_modify", "path_glob": "docs/development/adr/"}, {"kind": "only_modify", "path_glob": "scripts/duckdb_datasketches_smoke.py"}], "state": "done", "title": "Add CI smoke for required DuckDB datasketches families and decide vendoring vs HLL fallback", "verifications": [{"command": "test -f scripts/duckdb_datasketches_smoke.py", "description": "Smoke script exists", "expected": "exit 0", "item_id": "duckdb-datasketches-family-ci-smoke", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/duckdb_datasketches_smoke.py 2>&1 | grep -cE '^(theta|frequent_items|cpc|req|kll|hll):'", "description": "Smoke runs locally and reports per-family status", "expected": ">= 6", "item_id": "duckdb-datasketches-family-ci-smoke", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -rE 'duckdb_datasketches_smoke|datasketches.*smoke' .github/workflows/", "description": "CI job present", "expected": "matches >= 1", "item_id": "duckdb-datasketches-family-ci-smoke", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": [], "notes": "Connect to a fresh DuckDB, `INSTALL datasketches FROM community`,\nthen for each required family (`theta`, `frequent_items`, `cpc`,\n`req`, `kll`, `hll`) call a representative function (e.g.\n`SELECT datasketch_theta(1)`). Exit non-zero if any family raises\na Catalog Error. Print the resolved extension version (commit\nhash) on every run so failures are diagnosable from CI logs alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write `scripts/duckdb_datasketches_smoke.py` enumerating required families", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w1"], "notes": "Add a job to `.github/workflows/pr.yml` (or a new\n`extension-smoke.yml`) that runs the script. Cache: do not cache\nthe extension download -- the whole point is to detect upstream\nrebuilds. Job timeout: <2 min wall.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the smoke into a fast CI lane", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w1"], "notes": "Capture trade-offs: vendoring gives version pinning but adds\n~5 MB to the artifact tree per platform; HLL fallback narrows\ncross-engine sketch coverage but matches the Redshift ceiling\nalready documented. Decision criterion: how often does the\nupstream community-extensions build actually drop a family? If\nsmoke catches drift in <1 month, fallback path is enough; if\ndrift is monthly, vendoring earns its weight.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Author ADR: vendor `_binaries/datasketches/` vs accept HLL fallback", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w1"], "notes": "One-shot. Link the BenchBox smoke output and ask why the theta +\nfrequent-items families were dropped from the v1.3.2 build\nartifact. No ongoing engagement; tracking the response can stay\nin the ADR.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "File an upstream issue against duckdb-community-extensions if smoke confirms drift", "wid": "w4"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w3"], "notes": "Until the ADR's chosen path lands, mark the four affected ops\n(`sketch_insert_theta_per_partition`, `sketch_insert_topk_per_shard`,\n`sketch_query_theta_union_merge`, `sketch_query_topk_combine`) as\n\"blocked on upstream extension stability\" with a link to the ADR.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document affected ops in `docs/benchmarks/write-primitives-sketch-functions.md`", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "inline the smoke into `tests/`", "instead": "(not specified)", "why": "it is a CI-time check, not a unit test, and pulls a network resource (the community extension) on every run."}, {"dont": "pin the community-extension commit by editing `pyproject.toml`", "instead": "(not specified)", "why": "the extension is loaded by DuckDB itself via `INSTALL ... FROM community`, not by uv. The ADR may choose vendoring as a separate mechanism."}], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-06T00:00:00Z", "completed_pr": null, "created_at": "2026-05-05T00:00:00Z", "deferrals": [], "deps": [], "description": "Blind-spot `2026-05-02-155524` documented that the DuckDB community\ndatasketches extension (commit 2e38607, downloaded on-demand by\n`INSTALL datasketches FROM community`) silently dropped the `theta`\nand `frequent_items` families between PR #114 morning and the audit\nlater that day. Four sketch ops referencing `datasketch_theta` /\n`datasketch_frequent_items` are still in\n`benchbox/core/write_primitives/catalog/operations.yaml` and fail\nend-to-end. CI does not currently exercise these aggregates beyond\nloader unit tests.\n\nTwo work threads close the gap:\n\n1. Fast smoke that fails the build if any planned-on family (`theta`,\n `frequent_items`, `cpc`, `req`, `kll`, `hll`) disappears from a\n freshly-installed extension.\n2. Decision artifact (ADR) on vendoring vs HLL substitution. Current\n state of evidence: PR #182 already added CPC + REQ family ops, so\n HLL substitution is not the only path. Vendoring in\n `_binaries/datasketches/` gives version control but adds artifact\n tree weight.", "id": "duckdb-datasketches-family-ci-smoke", "preserves": ["Existing op definitions in `operations.yaml` are not removed -- the ADR may keep them and let the smoke gate the build instead.", "PR #182 CPC + REQ family ops continue to load and run on DuckDB.", "The smoke does not block the build for transient network failures fetching the extension -- network errors are reported but not fatal."], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/"}, {"kind": "only_modify", "path_glob": "_binaries/datasketches/"}, {"kind": "only_modify", "path_glob": "docs/benchmarks/write-primitives-sketch-functions.md"}, {"kind": "only_modify", "path_glob": "docs/development/adr/"}, {"kind": "only_modify", "path_glob": "scripts/duckdb_datasketches_smoke.py"}], "state": "done", "title": "Add CI smoke for required DuckDB datasketches families and decide vendoring vs HLL fallback", "verifications": [{"command": "test -f scripts/duckdb_datasketches_smoke.py", "description": "Smoke script exists", "expected": "exit 0", "item_id": "duckdb-datasketches-family-ci-smoke", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python scripts/duckdb_datasketches_smoke.py 2>&1 | grep -cE '^(theta|frequent_items|cpc|req|kll|hll):'", "description": "Smoke runs locally and reports per-family status", "expected": ">= 6", "item_id": "duckdb-datasketches-family-ci-smoke", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -rE 'duckdb_datasketches_smoke|datasketches.*smoke' .github/workflows/", "description": "CI job present", "expected": "matches >= 1", "item_id": "duckdb-datasketches-family-ci-smoke", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": [], "notes": "Connect to a fresh DuckDB, `INSTALL datasketches FROM community`,\nthen for each required family (`theta`, `frequent_items`, `cpc`,\n`req`, `kll`, `hll`) call a representative function (e.g.\n`SELECT datasketch_theta(1)`). Exit non-zero if any family raises\na Catalog Error. Print the resolved extension version (commit\nhash) on every run so failures are diagnosable from CI logs alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write `scripts/duckdb_datasketches_smoke.py` enumerating required families", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w1"], "notes": "Add a job to `.github/workflows/pr.yml` (or a new\n`extension-smoke.yml`) that runs the script. Cache: do not cache\nthe extension download -- the whole point is to detect upstream\nrebuilds. Job timeout: <2 min wall.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire the smoke into a fast CI lane", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w1"], "notes": "Capture trade-offs: vendoring gives version pinning but adds\n~5 MB to the artifact tree per platform; HLL fallback narrows\ncross-engine sketch coverage but matches the Redshift ceiling\nalready documented. Decision criterion: how often does the\nupstream community-extensions build actually drop a family? If\nsmoke catches drift in <1 month, fallback path is enough; if\ndrift is monthly, vendoring earns its weight.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Author ADR: vendor `_binaries/datasketches/` vs accept HLL fallback", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w1"], "notes": "One-shot. Link the BenchBox smoke output and ask why the theta +\nfrequent-items families were dropped from the v1.3.2 build\nartifact. No ongoing engagement; tracking the response can stay\nin the ADR.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "File an upstream issue against duckdb-community-extensions if smoke confirms drift", "wid": "w4"}, {"evidence": null, "item_id": "duckdb-datasketches-family-ci-smoke", "needs": ["w3"], "notes": "Until the ADR's chosen path lands, mark the four affected ops\n(`sketch_insert_theta_per_partition`, `sketch_insert_topk_per_shard`,\n`sketch_query_theta_union_merge`, `sketch_query_topk_combine`) as\n\"blocked on upstream extension stability\" with a link to the ADR.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document affected ops in `docs/benchmarks/write-primitives-sketch-functions.md`", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Support and Deployment", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Update all remaining DuckDB examples to use modern 3-line adapter pattern with tuning flags like TPC-H and TPC-DS examples\"\n- \"**Tasks**:\"\n- \"**Files Affected**: All `examples/duckdb_*.py` files except TPC-H, TPC-DS, TPC-DI, and all_benchmarks\"\n- \"**Impact**: \u2705 **COMPLETED** - Eliminated outdated anti-patterns across all DuckDB examples, reduced maintenance overhead by 40%+, demonstrated correct modern framework usage consistently, and provided uniform user experience with tuning capabilities across all benchmarks. All 8 modernized examples tested and working with unified tuning configuration support.\"", "id": "duckdb-example-modernization-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "DuckDB Example Modernization \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "keep timing out of fingerprints", "dont": "change the fingerprinting algorithm to include timing", "why": "fingerprints are used for plan-change detection across runs, and timing varies between identical plans"}, {"INSTEAD": "always keep the escape hatch", "dont": "remove the analyze_plans opt-out", "why": "concurrent throughput benchmarks and CI speed runs need lightweight structural-only plan capture"}, {"INSTEAD": "EXPLAIN ANALYZE is sufficient and schema-compatible", "dont": "switch to PRAGMA enable_profiling", "why": "it uses a different JSON schema (operator_type vs name) requiring full parser rewrite"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-27T00:00:00Z", "completed_pr": null, "created_at": "2026-02-27T00:00:00Z", "deferrals": [], "deps": [], "description": "Code review (2026-02-27) found that the current get_query_plan() uses EXPLAIN (FORMAT JSON)\nwithout ANALYZE, capturing ESTIMATED plans only - timing and cardinality fields are absent\nin stored plans. Switching to EXPLAIN ANALYZE (FORMAT JSON) by default will capture actual\nper-operator execution statistics while keeping the existing JSON schema intact.\n\nDecision: EXPLAIN ANALYZE is the new default for capture_plans=True.\nPRAGMA enable_profiling was rejected (different JSON schema, version-dependent inline\nreading, not thread-safe in file mode) - see context_sections for full rationale.\n\nSchema compatibility: EXPLAIN ANALYZE (FORMAT JSON) produces the same field names as\nEXPLAIN (FORMAT JSON): name / timing / cardinality / extra_info / children.\nThe DuckDBQueryPlanParser already handles timing+cardinality when present - proven by\nthe DUCKDB_1_0_JSON_WITH_TIMING fixture (tests/fixtures/duckdb_plans_by_version.py:177).\nThe core get_query_plan() change is one word: EXPLAIN \u2192 EXPLAIN ANALYZE.\n\nTrade-off acknowledged: EXPLAIN ANALYZE re-executes each query (2\u00d7 cost per captured\nquery). This is acceptable for plan capture runs where rich operator data is the goal.\nAn opt-out config key (analyze_plans: false) must be available for users who want\nstructural-only plan capture without the execution overhead.\n\nNote: plan fingerprinting is unaffected - it already excludes timing/cardinality by\ndesign (structural comparison only).\n\nReference: /Users/joe/.claude/plans/temporal-kindling-tarjan.md", "id": "duckdb-explain-analyze-default", "preserves": ["Opt-out path MUST remain available: analyze_plans: false in config (or equivalent) restores EXPLAIN without ANALYZE for users who need minimal overhead", "Parser behavior for plans without timing fields MUST continue working unchanged (backwards compatibility with stored plans from before this change)", "Plan fingerprinting MUST NOT include timing/cardinality (QueryPlanDAG.compute_plan_fingerprint excludes these by design - structural comparison only)"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Switch DuckDB plan capture to use EXPLAIN ANALYZE by default for actual per-operator timing", "verifications": [{"command": "uv run -- python -c \"import duckdb, json; con = duckdb.connect(); con.execute('CREATE TABLE t AS SELECT range AS id FROM range(1000)'); rows = con.execute('EXPLAIN ANALYZE (FORMAT JSON) SELECT count(*) FROM t').fetchall(); plan = json.loads(rows[0][1]); print('timing present:', 'timing' in json.dumps(plan))\"", "description": "EXPLAIN ANALYZE (FORMAT JSON) populates timing fields - confirms schema compatibility", "expected": "timing present: True", "item_id": "duckdb-explain-analyze-default", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/query_plans/parsers/test_duckdb_parser.py -v", "description": "All parser tests pass (fingerprint stability, WITH_TIMING fixture handling)", "expected": "all tests pass, 0 failures", "item_id": "duckdb-explain-analyze-default", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_duckdb_plan_capture.py tests/unit/core/query_plans/parsers/test_duckdb_parser.py -v", "description": "All plan capture tests pass", "expected": "all tests pass, 0 failures", "item_id": "duckdb-explain-analyze-default", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update get_query_plan() to use EXPLAIN ANALYZE (FORMAT JSON) by default; update docstring to reflect ANALYZE-by-default and the opt-out", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add analyze_plans: bool config field to PlatformAdapter.__init__ (default True, read from config dict)", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate EXPLAIN ANALYZE in get_query_plan() behind self.analyze_plans; fall back to EXPLAIN when False", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests in test_duckdb_plan_capture.py: assert timing fields present in default capture", "wid": "w4"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add test: with analyze_plans=False, captured plan has no timing/cardinality fields", "wid": "w5"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update capture_query_plan() docstring and CLI --capture-plans help text to mention EXPLAIN ANALYZE default and analyze_plans opt-out", "wid": "w6"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "collapse to one source of truth.", "dont": "leave both a decorative registry and direct dict imports", "why": "they can silently disagree (1 vs 21)"}, {"INSTEAD": "use a targeted comment/literal-stripped regex.", "dont": "pull in a SQL parser for the gate", "why": "it is on the per-op resolution path"}, {"INSTEAD": "gate on the MERGE INTO token only.", "dont": "widen the gate to the whole merge category", "why": "portable SCD2 ops are merge-category and must keep running on DuckDB"}], "approach": "Harden the single gate function, then reconcile its metadata and registry.\nPrefer a small comment/literal stripper + regex over a full SQL parser\n(no parser dependency on this hot path). For the registry, pick one source of\ntruth: the runtime already reads the dicts directly, so either wire the\nregistry to the same derivation or drop the register loop entirely - do not\nleave two mechanisms that can diverge. Apply the same decision symmetrically\nto the postgres rule.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-11T00:00:00Z", "completed_pr": null, "created_at": "2026-07-10T00:00:00Z", "deferrals": [], "deps": ["write-primitives-sql-override-skip-order"], "description": "SCD2 adversarial audit findings N6 and N7.\n\nN6 - benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:\n40-42 tests \"MERGE INTO\" in write_sql.upper(). Verified (scratchpad repro3.py)\nfalse positives and false negatives, both real against DuckDB 1.3.2:\n - false positive: token only in a comment ('-- ... MERGE INTO ...') -> gate\n skips an op DuckDB would run fine.\n - false positive: token only in a string literal ('MERGE INTO staging')\n -> same.\n - false negatives: 'MERGE INTO' (two spaces), 'MERGE\\nINTO', and MERGE\n reached via a CTE -> gate does NOT skip, and DuckDB 1.3.2 raises\n ParserException on all three (confirmed).\nNo live instance today (all 21 catalog ops with the token use the exact\nsingle-space form), so this is a regression trap for the next MERGE op added:\nit would run and hard-error instead of skipping cleanly. The gate also reads\noperation.write_sql (benchmark.py:422) and runs BEFORE platform_overrides\nresolution (benchmark.py:428-432), so it does not see the effective SQL.\n\nN7 - the gate's own metadata is wrong and its REGISTRY is write-only:\n - duckdb_write_primitives.py:10-19 says \"20 legacy MERGE INTO ops\"; the\n actual count of catalog ops whose write_sql contains MERGE INTO is 21 (the\n 20 merge-category ops + bulk_load_upsert_mode).\n - :27 says DUCKDB_WRITE_PRIMITIVES_OPERATION_SKIPS holds \"gaps that cannot be\n detected from the write_sql token\", but its only entry\n (bulk_load_upsert_mode) IS token-detectable (verified), so that entry and\n the operation_id lookup at :43-45 are dead code.\n - the REGISTRY.register loop at :49-66 registers 1 duckdb execution-filter\n rule while the runtime skips 21. Nothing resolves Phase.EXECUTION_FILTER:\n benchmark.py:42-45 imports the dicts directly, so the registry is\n decorative today - but a future consumer (coverage matrix, compat report)\n would get a wrong DuckDB answer. postgres_write_primitives.py:97-114 has\n the same write-only-registry shape.\n - tests/integration/test_write_primitives_duckdb.py:515 repeats the\n \"112 - 20 failing MERGE INTO\" arithmetic.", "id": "duckdb-merge-token-gate-hardening", "preserves": ["All 21 genuine MERGE INTO ops continue to skip cleanly on DuckDB.", "All 3 portable SCD2 ops (UPDATE+INSERT) continue to RUN on DuckDB (not skipped).", "bulk_load_upsert_mode still skips on DuckDB (whether via token or an explicit rule).", "postgres_write_primitives skip behavior is unchanged for the SCD2 and merge ops."], "prior_art": [{"concept": "benchbox/core/write_primitives/benchmark.py:421-424 the token-gate call site (before platform_overrides resolution)", "decision": "reuse", "path": "benchbox/core/write_primitives/benchmark.py:421-424"}, {"concept": "benchbox/sql_compat/context.py:17 Phase.EXECUTION_FILTER - resolved by nothing today", "decision": "reuse", "path": "benchbox/sql_compat/context.py:17"}, {"concept": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:33-46 the token gate function - harden", "decision": "reuse", "path": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:33-46"}, {"concept": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:49-66 write-only REGISTRY.register loop - decide populate vs delete", "decision": "reuse", "path": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:49-66"}, {"concept": "benchbox/sql_compat/rules/execution_filter/postgres_write_primitives.py:97-114 same write-only-registry shape - reconcile consistently", "decision": "reuse", "path": "benchbox/sql_compat/rules/execution_filter/postgres_write_primitives.py:97-114"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/write_primitives/catalog/operations.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/execution_filter/postgres_write_primitives.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_write_primitives_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/unit/sql_compat"}], "state": "done", "title": "Harden DuckDB MERGE INTO token gate and reconcile its dead REGISTRY/comments", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_write_primitives_duckdb.py -k scd2 -n 0 -q", "description": "SCD2 ops still run (not skipped) on DuckDB; all MERGE INTO ops still skip.", "expected": "all pass", "item_id": "duckdb-merge-token-gate-hardening", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/sql_compat -n 0 -q -k merge_token_gate", "description": "New gate unit test covers comment/literal false positives and whitespace/CTE false negatives.", "expected": "new test passes", "item_id": "duckdb-merge-token-gate-hardening", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": [], "notes": "Strip line comments (--...), block comments (/*...*/), and single-quoted\nstring literals before matching, then apply re.search(r'\\bMERGE\\s+INTO\\b').\nCover the exact cases in scratchpad repro3.py ATTACK 6.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace substring test with a whitespace-tolerant regex over comment- and string-literal-stripped SQL (MERGE\\s+INTO).", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w1"], "notes": "Requires moving the gate call after override resolution in\nbenchmark.py:428-432, or passing the resolved SQL into the gate. Keep the\nchange minimal and covered by a test with a duckdb override. This edits the\nSAME _get_effective_write_sql control flow as\nwrite-primitives-sql-override-skip-order; land AFTER that TODO (see\ndeps.needs) and rebase on its reordering to avoid a benchmark.py conflict.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Evaluate the gate against the effective SQL (after platform_overrides), not operation.write_sql, so an override that swaps MERGE in/out is gated correctly.", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix stale comments: :10-19 '20' -> 21 (incl bulk_load_upsert_mode); remove/justify the dead OPERATION_SKIPS entry + :43-45 lookup.", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w3"], "notes": "Pick ONE source of truth. The runtime reads the dicts directly, so either\nwire the registry to the same derivation or drop the register loop entirely\nin both duckdb_write_primitives.py and postgres_write_primitives.py.\nDocument the choice so no future consumer re-diverges (1 rule vs 21 skips).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile the write-only REGISTRY: populate token-derived skips or delete the decorative register loop (duckdb + postgres).", "wid": "w4"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct the arithmetic comment at tests/integration/test_write_primitives_duckdb.py:515.", "wid": "w5"}], "worktree": "scd2-audit-followup"} -{"anti_patterns": [{"INSTEAD": "only the no_acl set needs updating", "dont": "change supports_acl_introspection/role_hierarchy/column_grants for DuckDB", "why": "those already return False"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [], "deps": [], "description": "`supports_acl('duckdb')` currently returns `True` (or a non-False value)\nbecause 'duckdb' is missing from the `no_acl` frozenset in\n`benchbox/core/metadata_primitives/ddl.py:583`. DuckDB is a Tier 4\nplatform that does not support ACL / GRANT/REVOKE semantics, consistent\nwith its classification in `catalog/queries.yaml:1435-1438`.\n\nSurfaced as a pending work item (w2) in the DONE file for\n`reconcile-docs-review-code-followups` (completed 2026-04-16). Items w1\n(HashAlgorithm Literal narrowing) and w3 (stream cursor call sites) were\ncompleted; w2 was never executed.\n\nThe fix is a one-line change. The anti_pattern note from the DONE file\nspecifies: do NOT change supports_acl_introspection / role_hierarchy /\ncolumn_grants for DuckDB \u2014 those already correctly return False \u2014 only the\n`no_acl` set needs 'duckdb' added.", "id": "duckdb-supports-acl-no-acl-set-fix", "preserves": ["All other platform ACL classifications are unaffected", "supports_acl_introspection('duckdb'), role_hierarchy('duckdb'), and column_grants('duckdb') remain unchanged (already return False correctly)"], "prior_art": [{"concept": "benchbox/core/metadata_primitives/ddl.py:583 \u2014 add 'duckdb' to the no_acl set here", "decision": "reuse", "path": "benchbox/core/metadata_primitives/ddl.py:583"}, {"concept": "catalog/queries.yaml:1435-1438 \u2014 DuckDB Tier 4 classification; no ACL support", "decision": "reuse", "path": "catalog/queries.yaml:1435-1438"}, {"concept": "reconcile-docs-review-code-followups DONE file \u2014 w2 was the missed work item", "decision": "reuse", "path": "reconcile-docs-review-code-followups"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/ddl.py"}, {"kind": "only_modify", "path_glob": "tests/ (only if a test encodes the buggy True return value)"}], "state": "done", "title": "Add 'duckdb' to the no_acl set in ddl.py so supports_acl('duckdb') returns False", "verifications": [{"command": "python -c \"from benchbox.core.metadata_primitives.ddl import supports_acl; assert supports_acl('duckdb') is False, f'got {supports_acl(\\\"duckdb\\\")}'\" && echo OK", "description": "supports_acl('duckdb') returns False after the fix", "expected": "OK", "item_id": "duckdb-supports-acl-no-acl-set-fix", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k metadata_primitives -m fast -q", "description": "metadata_primitives tests pass", "expected": "all tests pass", "item_id": "duckdb-supports-acl-no-acl-set-fix", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "duckdb-supports-acl-no-acl-set-fix", "needs": [], "notes": "grep -n 'no_acl' benchbox/core/metadata_primitives/ddl.py\nConfirm 'duckdb' is absent from the set. Also verify:\n python -c \"from benchbox.core.metadata_primitives.ddl import supports_acl; print(supports_acl('duckdb'))\"\nShould print True or raise; target is False after the fix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm supports_acl('duckdb') still returns a non-False value at the cited line", "wid": "w0"}, {"evidence": null, "item_id": "duckdb-supports-acl-no-acl-set-fix", "needs": ["w0"], "notes": "One-line change: add 'duckdb' to the no_acl frozenset.\nDo NOT touch supports_acl_introspection, role_hierarchy, or column_grants\n\u2014 those already return False for DuckDB and are correct.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'duckdb' to the no_acl set in ddl.py:583", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-supports-acl-no-acl-set-fix", "needs": ["w1"], "notes": "uv run -- python -m pytest tests/ -k metadata_primitives -m fast -q\nIf any test asserts supports_acl('duckdb') is True, that test was\nencoding the bug \u2014 update it to assert False.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run metadata_primitives tests; update any test that encodes the old True return value", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the fingerprinting algorithm to include timing", "instead": "keep timing out of fingerprints", "why": "fingerprints are used for plan-change detection across runs, and timing varies between identical plans"}, {"dont": "remove the analyze_plans opt-out", "instead": "always keep the escape hatch", "why": "concurrent throughput benchmarks and CI speed runs need lightweight structural-only plan capture"}, {"dont": "switch to PRAGMA enable_profiling", "instead": "EXPLAIN ANALYZE is sufficient and schema-compatible", "why": "it uses a different JSON schema (operator_type vs name) requiring full parser rewrite"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-27T00:00:00Z", "completed_pr": null, "created_at": "2026-02-27T00:00:00Z", "deferrals": [], "deps": [], "description": "Code review (2026-02-27) found that the current get_query_plan() uses EXPLAIN (FORMAT JSON)\nwithout ANALYZE, capturing ESTIMATED plans only - timing and cardinality fields are absent\nin stored plans. Switching to EXPLAIN ANALYZE (FORMAT JSON) by default will capture actual\nper-operator execution statistics while keeping the existing JSON schema intact.\n\nDecision: EXPLAIN ANALYZE is the new default for capture_plans=True.\nPRAGMA enable_profiling was rejected (different JSON schema, version-dependent inline\nreading, not thread-safe in file mode) - see context_sections for full rationale.\n\nSchema compatibility: EXPLAIN ANALYZE (FORMAT JSON) produces the same field names as\nEXPLAIN (FORMAT JSON): name / timing / cardinality / extra_info / children.\nThe DuckDBQueryPlanParser already handles timing+cardinality when present - proven by\nthe DUCKDB_1_0_JSON_WITH_TIMING fixture (tests/fixtures/duckdb_plans_by_version.py:177).\nThe core get_query_plan() change is one word: EXPLAIN \u2192 EXPLAIN ANALYZE.\n\nTrade-off acknowledged: EXPLAIN ANALYZE re-executes each query (2\u00d7 cost per captured\nquery). This is acceptable for plan capture runs where rich operator data is the goal.\nAn opt-out config key (analyze_plans: false) must be available for users who want\nstructural-only plan capture without the execution overhead.\n\nNote: plan fingerprinting is unaffected - it already excludes timing/cardinality by\ndesign (structural comparison only).\n\nReference: /Users/joe/.claude/plans/temporal-kindling-tarjan.md", "id": "duckdb-explain-analyze-default", "preserves": ["Opt-out path MUST remain available: analyze_plans: false in config (or equivalent) restores EXPLAIN without ANALYZE for users who need minimal overhead", "Parser behavior for plans without timing fields MUST continue working unchanged (backwards compatibility with stored plans from before this change)", "Plan fingerprinting MUST NOT include timing/cardinality (QueryPlanDAG.compute_plan_fingerprint excludes these by design - structural comparison only)"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Switch DuckDB plan capture to use EXPLAIN ANALYZE by default for actual per-operator timing", "verifications": [{"command": "uv run -- python -c \"import duckdb, json; con = duckdb.connect(); con.execute('CREATE TABLE t AS SELECT range AS id FROM range(1000)'); rows = con.execute('EXPLAIN ANALYZE (FORMAT JSON) SELECT count(*) FROM t').fetchall(); plan = json.loads(rows[0][1]); print('timing present:', 'timing' in json.dumps(plan))\"", "description": "EXPLAIN ANALYZE (FORMAT JSON) populates timing fields - confirms schema compatibility", "expected": "timing present: True", "item_id": "duckdb-explain-analyze-default", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/query_plans/parsers/test_duckdb_parser.py -v", "description": "All parser tests pass (fingerprint stability, WITH_TIMING fixture handling)", "expected": "all tests pass, 0 failures", "item_id": "duckdb-explain-analyze-default", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_duckdb_plan_capture.py tests/unit/core/query_plans/parsers/test_duckdb_parser.py -v", "description": "All plan capture tests pass", "expected": "all tests pass, 0 failures", "item_id": "duckdb-explain-analyze-default", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update get_query_plan() to use EXPLAIN ANALYZE (FORMAT JSON) by default; update docstring to reflect ANALYZE-by-default and the opt-out", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add analyze_plans: bool config field to PlatformAdapter.__init__ (default True, read from config dict)", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate EXPLAIN ANALYZE in get_query_plan() behind self.analyze_plans; fall back to EXPLAIN when False", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests in test_duckdb_plan_capture.py: assert timing fields present in default capture", "wid": "w4"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add test: with analyze_plans=False, captured plan has no timing/cardinality fields", "wid": "w5"}, {"evidence": null, "item_id": "duckdb-explain-analyze-default", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update capture_query_plan() docstring and CLI --capture-plans help text to mention EXPLAIN ANALYZE default and analyze_plans opt-out", "wid": "w6"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "leave both a decorative registry and direct dict imports", "instead": "collapse to one source of truth.", "why": "they can silently disagree (1 vs 21)"}, {"dont": "pull in a SQL parser for the gate", "instead": "use a targeted comment/literal-stripped regex.", "why": "it is on the per-op resolution path"}, {"dont": "widen the gate to the whole merge category", "instead": "gate on the MERGE INTO token only.", "why": "portable SCD2 ops are merge-category and must keep running on DuckDB"}], "approach": "Harden the single gate function, then reconcile its metadata and registry.\nPrefer a small comment/literal stripper + regex over a full SQL parser\n(no parser dependency on this hot path). For the registry, pick one source of\ntruth: the runtime already reads the dicts directly, so either wire the\nregistry to the same derivation or drop the register loop entirely - do not\nleave two mechanisms that can diverge. Apply the same decision symmetrically\nto the postgres rule.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-11T00:00:00Z", "completed_pr": null, "created_at": "2026-07-10T00:00:00Z", "deferrals": [], "deps": ["write-primitives-sql-override-skip-order"], "description": "SCD2 adversarial audit findings N6 and N7.\n\nN6 - benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:\n40-42 tests \"MERGE INTO\" in write_sql.upper(). Verified (scratchpad repro3.py)\nfalse positives and false negatives, both real against DuckDB 1.3.2:\n - false positive: token only in a comment ('-- ... MERGE INTO ...') -> gate\n skips an op DuckDB would run fine.\n - false positive: token only in a string literal ('MERGE INTO staging')\n -> same.\n - false negatives: 'MERGE INTO' (two spaces), 'MERGE\\nINTO', and MERGE\n reached via a CTE -> gate does NOT skip, and DuckDB 1.3.2 raises\n ParserException on all three (confirmed).\nNo live instance today (all 21 catalog ops with the token use the exact\nsingle-space form), so this is a regression trap for the next MERGE op added:\nit would run and hard-error instead of skipping cleanly. The gate also reads\noperation.write_sql (benchmark.py:422) and runs BEFORE platform_overrides\nresolution (benchmark.py:428-432), so it does not see the effective SQL.\n\nN7 - the gate's own metadata is wrong and its REGISTRY is write-only:\n - duckdb_write_primitives.py:10-19 says \"20 legacy MERGE INTO ops\"; the\n actual count of catalog ops whose write_sql contains MERGE INTO is 21 (the\n 20 merge-category ops + bulk_load_upsert_mode).\n - :27 says DUCKDB_WRITE_PRIMITIVES_OPERATION_SKIPS holds \"gaps that cannot be\n detected from the write_sql token\", but its only entry\n (bulk_load_upsert_mode) IS token-detectable (verified), so that entry and\n the operation_id lookup at :43-45 are dead code.\n - the REGISTRY.register loop at :49-66 registers 1 duckdb execution-filter\n rule while the runtime skips 21. Nothing resolves Phase.EXECUTION_FILTER:\n benchmark.py:42-45 imports the dicts directly, so the registry is\n decorative today - but a future consumer (coverage matrix, compat report)\n would get a wrong DuckDB answer. postgres_write_primitives.py:97-114 has\n the same write-only-registry shape.\n - tests/integration/test_write_primitives_duckdb.py:515 repeats the\n \"112 - 20 failing MERGE INTO\" arithmetic.", "id": "duckdb-merge-token-gate-hardening", "preserves": ["All 21 genuine MERGE INTO ops continue to skip cleanly on DuckDB.", "All 3 portable SCD2 ops (UPDATE+INSERT) continue to RUN on DuckDB (not skipped).", "bulk_load_upsert_mode still skips on DuckDB (whether via token or an explicit rule).", "postgres_write_primitives skip behavior is unchanged for the SCD2 and merge ops."], "prior_art": [{"concept": "benchbox/core/write_primitives/benchmark.py:421-424 the token-gate call site (before platform_overrides resolution)", "decision": "reuse", "path": "benchbox/core/write_primitives/benchmark.py:421-424"}, {"concept": "benchbox/sql_compat/context.py:17 Phase.EXECUTION_FILTER - resolved by nothing today", "decision": "reuse", "path": "benchbox/sql_compat/context.py:17"}, {"concept": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:33-46 the token gate function - harden", "decision": "reuse", "path": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:33-46"}, {"concept": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:49-66 write-only REGISTRY.register loop - decide populate vs delete", "decision": "reuse", "path": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py:49-66"}, {"concept": "benchbox/sql_compat/rules/execution_filter/postgres_write_primitives.py:97-114 same write-only-registry shape - reconcile consistently", "decision": "reuse", "path": "benchbox/sql_compat/rules/execution_filter/postgres_write_primitives.py:97-114"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/write_primitives/catalog/operations.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/write_primitives/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/execution_filter/duckdb_write_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/sql_compat/rules/execution_filter/postgres_write_primitives.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_write_primitives_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/unit/sql_compat"}], "state": "done", "title": "Harden DuckDB MERGE INTO token gate and reconcile its dead REGISTRY/comments", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_write_primitives_duckdb.py -k scd2 -n 0 -q", "description": "SCD2 ops still run (not skipped) on DuckDB; all MERGE INTO ops still skip.", "expected": "all pass", "item_id": "duckdb-merge-token-gate-hardening", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/sql_compat -n 0 -q -k merge_token_gate", "description": "New gate unit test covers comment/literal false positives and whitespace/CTE false negatives.", "expected": "new test passes", "item_id": "duckdb-merge-token-gate-hardening", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": [], "notes": "Strip line comments (--...), block comments (/*...*/), and single-quoted\nstring literals before matching, then apply re.search(r'\\bMERGE\\s+INTO\\b').\nCover the exact cases in scratchpad repro3.py ATTACK 6.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace substring test with a whitespace-tolerant regex over comment- and string-literal-stripped SQL (MERGE\\s+INTO).", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w1"], "notes": "Requires moving the gate call after override resolution in\nbenchmark.py:428-432, or passing the resolved SQL into the gate. Keep the\nchange minimal and covered by a test with a duckdb override. This edits the\nSAME _get_effective_write_sql control flow as\nwrite-primitives-sql-override-skip-order; land AFTER that TODO (see\ndeps.needs) and rebase on its reordering to avoid a benchmark.py conflict.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Evaluate the gate against the effective SQL (after platform_overrides), not operation.write_sql, so an override that swaps MERGE in/out is gated correctly.", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix stale comments: :10-19 '20' -> 21 (incl bulk_load_upsert_mode); remove/justify the dead OPERATION_SKIPS entry + :43-45 lookup.", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w3"], "notes": "Pick ONE source of truth. The runtime reads the dicts directly, so either\nwire the registry to the same derivation or drop the register loop entirely\nin both duckdb_write_primitives.py and postgres_write_primitives.py.\nDocument the choice so no future consumer re-diverges (1 rule vs 21 skips).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile the write-only REGISTRY: populate token-derived skips or delete the decorative register loop (duckdb + postgres).", "wid": "w4"}, {"evidence": null, "item_id": "duckdb-merge-token-gate-hardening", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct the arithmetic comment at tests/integration/test_write_primitives_duckdb.py:515.", "wid": "w5"}], "worktree": "scd2-audit-followup"} +{"anti_patterns": [{"dont": "change supports_acl_introspection/role_hierarchy/column_grants for DuckDB", "instead": "only the no_acl set needs updating", "why": "those already return False"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-16T00:00:00Z", "completed_pr": null, "created_at": "2026-07-16T00:00:00Z", "deferrals": [], "deps": [], "description": "`supports_acl('duckdb')` currently returns `True` (or a non-False value)\nbecause 'duckdb' is missing from the `no_acl` frozenset in\n`benchbox/core/metadata_primitives/ddl.py:583`. DuckDB is a Tier 4\nplatform that does not support ACL / GRANT/REVOKE semantics, consistent\nwith its classification in `catalog/queries.yaml:1435-1438`.\n\nSurfaced as a pending work item (w2) in the DONE file for\n`reconcile-docs-review-code-followups` (completed 2026-04-16). Items w1\n(HashAlgorithm Literal narrowing) and w3 (stream cursor call sites) were\ncompleted; w2 was never executed.\n\nThe fix is a one-line change. The anti_pattern note from the DONE file\nspecifies: do NOT change supports_acl_introspection / role_hierarchy /\ncolumn_grants for DuckDB \u2014 those already correctly return False \u2014 only the\n`no_acl` set needs 'duckdb' added.", "id": "duckdb-supports-acl-no-acl-set-fix", "preserves": ["All other platform ACL classifications are unaffected", "supports_acl_introspection('duckdb'), role_hierarchy('duckdb'), and column_grants('duckdb') remain unchanged (already return False correctly)"], "prior_art": [{"concept": "benchbox/core/metadata_primitives/ddl.py:583 \u2014 add 'duckdb' to the no_acl set here", "decision": "reuse", "path": "benchbox/core/metadata_primitives/ddl.py:583"}, {"concept": "catalog/queries.yaml:1435-1438 \u2014 DuckDB Tier 4 classification; no ACL support", "decision": "reuse", "path": "catalog/queries.yaml:1435-1438"}, {"concept": "reconcile-docs-review-code-followups DONE file \u2014 w2 was the missed work item", "decision": "reuse", "path": "reconcile-docs-review-code-followups"}], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/metadata_primitives/ddl.py"}, {"kind": "only_modify", "path_glob": "tests/ (only if a test encodes the buggy True return value)"}], "state": "done", "title": "Add 'duckdb' to the no_acl set in ddl.py so supports_acl('duckdb') returns False", "verifications": [{"command": "python -c \"from benchbox.core.metadata_primitives.ddl import supports_acl; assert supports_acl('duckdb') is False, f'got {supports_acl(\\\"duckdb\\\")}'\" && echo OK", "description": "supports_acl('duckdb') returns False after the fix", "expected": "OK", "item_id": "duckdb-supports-acl-no-acl-set-fix", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k metadata_primitives -m fast -q", "description": "metadata_primitives tests pass", "expected": "all tests pass", "item_id": "duckdb-supports-acl-no-acl-set-fix", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "duckdb-supports-acl-no-acl-set-fix", "needs": [], "notes": "grep -n 'no_acl' benchbox/core/metadata_primitives/ddl.py\nConfirm 'duckdb' is absent from the set. Also verify:\n python -c \"from benchbox.core.metadata_primitives.ddl import supports_acl; print(supports_acl('duckdb'))\"\nShould print True or raise; target is False after the fix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm supports_acl('duckdb') still returns a non-False value at the cited line", "wid": "w0"}, {"evidence": null, "item_id": "duckdb-supports-acl-no-acl-set-fix", "needs": ["w0"], "notes": "One-line change: add 'duckdb' to the no_acl frozenset.\nDo NOT touch supports_acl_introspection, role_hierarchy, or column_grants\n\u2014 those already return False for DuckDB and are correct.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add 'duckdb' to the no_acl set in ddl.py:583", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-supports-acl-no-acl-set-fix", "needs": ["w1"], "notes": "uv run -- python -m pytest tests/ -k metadata_primitives -m fast -q\nIf any test asserts supports_acl('duckdb') is True, that test was\nencoding the bug \u2014 update it to assert False.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run metadata_primitives tests; update any test that encodes the old True return value", "wid": "w2"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-11-26T00:00:00Z", "deferrals": [], "deps": [], "description": "The DuckDB text parser flattens box-drawing plans into a linear chain, losing\njoins/branching and producing misleading fingerprints/insights whenever JSON\nparsing fails. Implement real hierarchy extraction from box art (or explicitly\ndisable fallback with clear errors) so captured plans remain trustworthy. Add\ndetection to prefer JSON and surface warnings when text parsing is incomplete.", "id": "duckdb-text-plan-reliability", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Make DuckDB text plan parsing reliable or fail fast", "verifications": [], "work": [], "worktree": "query-plan-remediation"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Support and Deployment", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Update the modern DuckDB TPC-DS and TPC-H examples to integrate with updated tuning configurations and improved TPC specification-compliant query ordering\"\n- \"**Tasks**:\"\n- \"**Files Affected**: `examples/duckdb_tpch.py`, `examples/duckdb_tpcds.py`, `examples/tuning_demonstration.py`, `benchbox/platforms/duckdb.py`, related documentation and test files\"\n- \"**Impact**: \u2705 Modernized DuckDB examples now showcase the full capabilities of the unified tuning configuration system with platform-aware validation, comprehensive constraint management, and performance comparison capabilities. Examples demonstrate production-ready benchmarking practices with clean command-line interfaces, statistical analysis, and complete integration with the modern adapter pattern. Created a powerful demonstration script that enables users to explore all tuning features and compare performance across different optimization scenarios.\"", "id": "duckdb-tpc-ds-and-tpc-h-example-updates-for-enhanced-tuning-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "DuckDB TPC-DS and TPC-H Example Updates for Enhanced Tuning \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "sample the first shard only and assume consistency within a table", "dont": "check trailing delimiter on every shard", "why": "has_trailing_delimiter() opens and reads the file, and TPC-DS has hundreds of shards"}, {"INSTEAD": "keep it as a safety net", "dont": "remove null_padding=true", "why": "some older TPC-DS data files may have inconsistent trailing delimiters across shards"}, {"INSTEAD": "use ctx.exit(1) for non-zero exit", "dont": "use sys.exit() in CLI handlers", "why": "Click manages exit codes via ctx.exit()"}], "approach": "Use the existing has_trailing_delimiter() and get_column_names_with_trailing()\nfrom benchbox/utils/file_format.py - they already handle compressed files and\nedge cases. For the loader fix:\n1. Sample the first shard to detect trailing delimiter\n2. Build column name list via get_column_names_with_trailing()\n3. Replace SELECT * EXCLUDE with explicit column projection: SELECT {schema_cols} FROM read_csv(...)\n4. Keep null_padding=true as a safety net\n\nFor the exit code fix, add ctx.exit(1) after the FAILED status branch in run.py.\nCheck both the orchestrated and non-orchestrated execution paths.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "duckdb-tpcds-trailing-delimiter-regression", "id": 68, "reason": "Only DuckDB native handler uses this pattern; other platforms use different load paths. Defer until after regression is fixed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason states only the DuckDB native handler uses this delimiter pattern; other platforms use different load paths, so the audit target doesn't exist.", "summary": "Audit other loaders (SQLite, ClickHouse, etc.) for same trailing delimiter assumption"}], "deps": [], "description": "DuckDB + TPC-DS loading fails after TPC-DS datagen was changed to remove trailing\ndelimiters from generated .dat files. The pipe-delimited loader in\nDuckDBNativeHandler.load_table() unconditionally appends a _trailing_delimiter_\ndummy column and uses SELECT * EXCLUDE (_trailing_delimiter_), which causes a\nBinder Error when the file has no trailing delimiter.\n\nObserved in a 72-run DuckDB version matrix (1.0.0-1.5.0rc1) across tpch/tpcds/\nclickbench/ssb - TPC-H succeeds but TPC-DS consistently fails with:\n - \"Binder Error: Column '_trailing_delimiter_' in EXCLUDE list not found in FROM clause\"\n - \"Data validation failed - benchmark execution halted\"\n - \"Empty tables detected: ...\"\n\nAdditionally, `benchbox run` exits with code 0 even when the benchmark status is\nFAILED, masking failures in automation pipelines. No result JSON is written for\nfailed runs.\n\nUtility functions `has_trailing_delimiter()` and `get_column_names_with_trailing()`\nalready exist in benchbox/utils/file_format.py but are not used by the loader.", "id": "duckdb-tpcds-trailing-delimiter-regression", "preserves": ["DuckDBNativeHandler.load_table() continues to handle CSV and Parquet inputs via their existing code paths (only pipe-delimited path is affected)", "DuckDBNativeHandler.load_table() continues to load TPC-H .tbl files (which still have trailing delimiters) correctly", "Successful benchmark runs still exit with code 0 and produce result JSON"], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/utils/file_format.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/test_trailing_delimiter_loading.py"}], "state": "done", "title": "Fix DuckDB TPC-DS loader regression: trailing delimiter detection and CLI exit code", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/base/ -v -k 'load or data_loading'", "description": "Existing data loading tests still pass", "expected": "all tests pass, 0 failures", "item_id": "duckdb-tpcds-trailing-delimiter-regression", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_trailing_delimiter_loading.py -v", "description": "New trailing delimiter regression tests pass", "expected": "all tests pass including with-trailing and without-trailing variants", "item_id": "duckdb-tpcds-trailing-delimiter-regression", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke test for TPC-H (must not regress)", "expected": "all fast tests pass", "item_id": "duckdb-tpcds-trailing-delimiter-regression", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": [], "notes": "In DuckDBNativeHandler.load_table() (data_loading.py ~line 640-666):\n1. Import and call has_trailing_delimiter(file_path, delimiter, col_names) on the first shard\n2. Use get_column_names_with_trailing() to conditionally add dummy column\n3. Replace SELECT * EXCLUDE (_trailing_delimiter_) with explicit column projection\n (SELECT col1, col2, ... FROM read_csv(...)) for robustness\n4. Keep null_padding=true for safety with both file variants\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix DuckDB pipe-delimited loader to detect trailing delimiter per file and project explicit schema columns instead of SELECT * EXCLUDE", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": [], "notes": "In benchbox/cli/commands/run.py (~lines 2243-2296):\n- After benchmark execution, if result.validation_status == \"FAILED\", call ctx.exit(1)\n- Also check the orchestrated run path (~lines 1512-1554)\n- Ensure consistent behavior: either write a failed result artifact OR exit non-zero with clear message\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix CLI exit code: benchbox run must return non-zero when benchmark status is FAILED", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": ["w1", "w2"], "notes": "Unit tests:\n- Pipe-delimited load WITH trailing delimiter (existing behavior preserved)\n- Pipe-delimited load WITHOUT trailing delimiter (new behavior works)\n- TPC-DS load path with .dat.zst shards and no trailing delimiter\nIntegration/CLI test:\n- benchbox run exit code is non-zero on benchmark failure\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for trailing delimiter handling and exit code propagation", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": ["w1", "w2", "w3"], "notes": "Run: benchbox run --platform duckdb --benchmark tpcds --scale 10 --phases load,power \\\n --platform-option driver_version=1.0.0 --platform-option driver_auto_install=true --non-interactive\nConfirm:\n- No _trailing_delimiter_ binder errors in output\n- Tables load successfully, result JSON is produced\n- Non-zero exit on intentionally induced failure (e.g., bad data file)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify fix with live DuckDB TPC-DS run at scale 10", "wid": "w4"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "use the ATTACH 'ducklake:postgres:...' connection string", "dont": "connect to the PostgreSQL catalog via psycopg", "why": "DuckLake reads the catalog through DuckDB's own `postgres` extension"}, {"INSTEAD": "reuse them", "dont": "duplicate PG connection-arg parsing", "why": "POSTGRES_FAMILY_* helpers already exist"}, {"INSTEAD": "only create the S3 secret when data_path is a cloud URI", "dont": "require S3 credentials for the default path", "why": "most benchmark runs are local"}], "approach": "Build on the base adapter. create_connection already ATTACHes a catalog after\nINSTALL/LOAD ducklake; generalize it to switch the ATTACH string on a `catalog`\noption and to prepend INSTALL sqlite / INSTALL postgres / INSTALL httpfs as\nneeded. Reuse POSTGRES_FAMILY_* option helpers for PG params and cloud_storage\nhelpers to detect s3:// DATA_PATH. Keep S3 secret creation optional and\ncredential-chain-first so no creds are needed for local runs. Gate all live\nextended-backend tests behind availability/credential skips.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-10T00:00:00Z", "completed_pr": null, "created_at": "2026-07-07T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-catalog-storage-backends", "id": 93, "reason": "DuckLake docs explicitly flag MySQL as not recommended (compatibility issues)", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason cites DuckLake docs explicitly flagging MySQL catalog as not recommended due to compatibility issues; upstream-blocked by design.", "summary": "MySQL catalog backend"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-catalog-storage-backends", "id": 94, "reason": "S3 first; other object stores are a follow-on once the secret/httpfs path is proven", "resolution": "promoted", "resolved_item": "gcs-azure-object-storage-backends-for-data-path", "resolved_reason": null, "summary": "GCS / Azure object storage backends for DATA_PATH"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-catalog-storage-backends", "id": 95, "reason": "Single-process benchmarking parity first; concurrency benchmarks are a separate research item", "resolution": "promoted", "resolved_item": "multi-client-concurrent-writer-benchmark-scenarios-the-multi", "resolved_reason": null, "summary": "Multi-client / concurrent-writer benchmark scenarios (the 'multiplayer DuckDB' story)"}], "deps": ["ducklake-platform-adapter"], "description": "Extend the DuckLake platform adapter (see ducklake-platform-adapter) beyond the\ndefault DuckDB-file catalog + local storage to the full v1 scope:\n\n - Catalog metadata backends: SQLite and PostgreSQL (in addition to DuckDB).\n - Data storage: S3 / cloud object storage for DATA_PATH (in addition to local).\n\nDuckLake's catalog backend is chosen by the ATTACH connection string, each\nneeding a DuckDB extension:\n duckdb : ATTACH 'ducklake:meta.ducklake' (default; done in base item)\n sqlite : INSTALL sqlite; ATTACH 'ducklake:sqlite:meta.sqlite' (DATA_PATH ...)\n postgres: INSTALL postgres; ATTACH 'ducklake:postgres:dbname=... host=...' (DATA_PATH ...)\n(MySQL is explicitly excluded \u2014 DuckLake docs flag it as not recommended.)\n\nCloud storage is orthogonal to the catalog: DATA_PATH can be 's3://bucket/prefix/'\nwith httpfs + a DuckDB secret:\n INSTALL httpfs;\n CREATE SECRET (TYPE s3, PROVIDER credential_chain); -- or KEY_ID/SECRET/REGION\n ATTACH 'ducklake:...' AS lake (DATA_PATH 's3://bucket/bench/');\n\nSelection is surfaced via --platform-option catalog=duckdb|sqlite|postgres and\n--platform-option data_path=.\n\nSources: https://ducklake.select/docs/stable/duckdb/usage/choosing_a_catalog_database ,\nhttps://duckdb.org/docs/lts/core_extensions/ducklake", "id": "ducklake-catalog-storage-backends", "preserves": ["Existing PostgreSQL-family platforms (postgresql, pg_duckdb, timescaledb) and their option plumbing are unaffected", "No new hard dependency: sqlite/postgres/httpfs are DuckDB runtime extensions, not Python packages; the global pyproject `duckdb` pin is untouched", "The default DuckDB-file catalog + local storage path from ducklake-platform-adapter keeps working unchanged when no catalog/data_path option is given"], "prior_art": [{"concept": "benchbox/core/platform_registry.py DeploymentCapability:210 (requires_cloud_storage, requires_network, auth_methods) \u2014 EXTEND the ducklake metadata deployment_modes for self-hosted-PG and cloud-S3", "decision": "extend", "path": "benchbox/core/platform_registry.py"}, {"concept": "benchbox/platforms/__init__.py _OPTION_SPEC_ROWS:604 + register_config_builder:826 \u2014 EXTEND: add catalog/data_path/PG/S3 option specs and (if creds persistence needed) a config builder", "decision": "extend", "path": "benchbox/platforms/__init__.py"}, {"concept": "benchbox/platforms/base/config_utils.py POSTGRES_FAMILY_PLATFORM_FIELDS/POSTGRES_FAMILY_BASE_OPTIONS + make_platform_config_builder \u2014 REUSE for the postgres-catalog connection params (pg_duckdb.py:419 shows usage)", "decision": "reuse", "path": "benchbox/platforms/base/config_utils.py"}, {"concept": "benchbox/platforms/duckdb.py extension-install idiom (_try_delta_scan:136, _try_iceberg_scan:153) \u2014 REUSE for INSTALL/LOAD sqlite|postgres|httpfs", "decision": "reuse", "path": "benchbox/platforms/duckdb.py"}, {"concept": "benchbox/utils/cloud_storage.py is_cloud_path/get_cloud_path_info (used in duckdb.py:844-848) \u2014 REUSE to detect and route s3:// DATA_PATH", "decision": "reuse", "path": "benchbox/utils/cloud_storage.py"}, {"concept": "tests/integration/test_open_table_formats.py:715-733 TestDuckLakeFormatSmoke extension-availability guard \u2014 REUSE the skip-guard pattern for extension/credential gating", "decision": "reuse", "path": "tests/integration/test_open_table_formats.py:715-733"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/pg_duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/utils/cloud_storage.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/ducklake.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/comparison-matrix.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/ducklake.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/support-status.md"}, {"kind": "only_modify", "path_glob": "tests/integration/test_ducklake_integration.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_ducklake_adapter.py"}], "state": "done", "title": "DuckLake platform: SQLite + PostgreSQL catalogs and S3/cloud data storage", "verifications": [{"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01 --platform-option catalog=sqlite", "description": "SQLite catalog runs TPC-H", "expected": "queries succeed; metadata.sqlite + Parquet written", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01 --platform-option catalog=postgres --platform-option pg_host=localhost", "description": "PostgreSQL catalog runs TPC-H (with a reachable PG server + pre-created catalog DB)", "expected": "queries succeed; catalog rows in the Postgres ducklake_catalog DB", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01 --platform-option data_path=s3:///bench/", "description": "S3 DATA_PATH (with creds present) writes Parquet to the bucket", "expected": "queries succeed; Parquet objects appear under s3:///bench/", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/integration/test_ducklake_integration.py -q", "description": "Extended-backend integration tests pass or skip cleanly", "expected": "sqlite case passes; postgres/s3 cases pass or skip when unavailable", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": [], "notes": "Generalize create_connection to build the ATTACH string per catalog; validate\ncatalog in {duckdb,sqlite,postgres}. SQLite: INSTALL sqlite; ATTACH\n'ducklake:sqlite:.sqlite' AS lake (DATA_PATH '').\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add catalog-backend selection (--platform-option catalog=duckdb|sqlite|postgres) + SQLite backend", "wid": "w1"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": ["w1"], "notes": "INSTALL postgres; ATTACH 'ducklake:postgres:dbname= host= user=... password=...'\nAS lake (DATA_PATH ''). Pull PG connection params from platform options/env via\nthe POSTGRES_FAMILY_* helpers. Catalog DB must be pre-created (document this).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add PostgreSQL catalog backend, reusing PG-family option/credential machinery", "wid": "w2"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": [], "notes": "When data_path is a cloud URI (is_cloud_path): INSTALL httpfs; create an S3 secret\n(credential_chain by default, or explicit KEY_ID/SECRET/REGION from options) before\nATTACH with DATA_PATH 's3://...'. Extend ducklake metadata deployment_modes with a\ncloud mode (requires_cloud_storage/requires_network).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add S3/cloud DATA_PATH support (httpfs + DuckDB secret + cloud_storage routing)", "wid": "w3"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": ["w1", "w2", "w3"], "notes": "tests/integration/test_ducklake_integration.py: add a SQLite-catalog TPC-H SF 0.01\ncase; a PostgreSQL-catalog case behind the existing PG fixtures/marker; an S3 case\nskipped when creds absent. Guard on ducklake/sqlite/postgres/httpfs extension\navailability like TestDuckLakeFormatSmoke.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration tests for sqlite/postgres catalogs and s3 storage (extension/creds-gated)", "wid": "w4"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": ["w1", "w2", "w3"], "notes": "Document the three catalog backends, local vs S3 DATA_PATH, required extensions,\nand the pre-created-PG-database prerequisite. Update deployment-mode columns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs: catalog-backend + storage options in ducklake.md, comparison-matrix, support-status", "wid": "w5"}], "worktree": "ducklake-platform"} +{"anti_patterns": [{"dont": "check trailing delimiter on every shard", "instead": "sample the first shard only and assume consistency within a table", "why": "has_trailing_delimiter() opens and reads the file, and TPC-DS has hundreds of shards"}, {"dont": "remove null_padding=true", "instead": "keep it as a safety net", "why": "some older TPC-DS data files may have inconsistent trailing delimiters across shards"}, {"dont": "use sys.exit() in CLI handlers", "instead": "use ctx.exit(1) for non-zero exit", "why": "Click manages exit codes via ctx.exit()"}], "approach": "Use the existing has_trailing_delimiter() and get_column_names_with_trailing()\nfrom benchbox/utils/file_format.py - they already handle compressed files and\nedge cases. For the loader fix:\n1. Sample the first shard to detect trailing delimiter\n2. Build column name list via get_column_names_with_trailing()\n3. Replace SELECT * EXCLUDE with explicit column projection: SELECT {schema_cols} FROM read_csv(...)\n4. Keep null_padding=true as a safety net\n\nFor the exit code fix, add ctx.exit(1) after the FAILED status branch in run.py.\nCheck both the orchestrated and non-orchestrated execution paths.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "duckdb-tpcds-trailing-delimiter-regression", "id": 68, "reason": "Only DuckDB native handler uses this pattern; other platforms use different load paths. Defer until after regression is fixed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason states only the DuckDB native handler uses this delimiter pattern; other platforms use different load paths, so the audit target doesn't exist.", "summary": "Audit other loaders (SQLite, ClickHouse, etc.) for same trailing delimiter assumption"}], "deps": [], "description": "DuckDB + TPC-DS loading fails after TPC-DS datagen was changed to remove trailing\ndelimiters from generated .dat files. The pipe-delimited loader in\nDuckDBNativeHandler.load_table() unconditionally appends a _trailing_delimiter_\ndummy column and uses SELECT * EXCLUDE (_trailing_delimiter_), which causes a\nBinder Error when the file has no trailing delimiter.\n\nObserved in a 72-run DuckDB version matrix (1.0.0-1.5.0rc1) across tpch/tpcds/\nclickbench/ssb - TPC-H succeeds but TPC-DS consistently fails with:\n - \"Binder Error: Column '_trailing_delimiter_' in EXCLUDE list not found in FROM clause\"\n - \"Data validation failed - benchmark execution halted\"\n - \"Empty tables detected: ...\"\n\nAdditionally, `benchbox run` exits with code 0 even when the benchmark status is\nFAILED, masking failures in automation pipelines. No result JSON is written for\nfailed runs.\n\nUtility functions `has_trailing_delimiter()` and `get_column_names_with_trailing()`\nalready exist in benchbox/utils/file_format.py but are not used by the loader.", "id": "duckdb-tpcds-trailing-delimiter-regression", "preserves": ["DuckDBNativeHandler.load_table() continues to handle CSV and Parquet inputs via their existing code paths (only pipe-delimited path is affected)", "DuckDBNativeHandler.load_table() continues to load TPC-H .tbl files (which still have trailing delimiters) correctly", "Successful benchmark runs still exit with code 0 and produce result JSON"], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/utils/file_format.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/base/test_trailing_delimiter_loading.py"}], "state": "done", "title": "Fix DuckDB TPC-DS loader regression: trailing delimiter detection and CLI exit code", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/base/ -v -k 'load or data_loading'", "description": "Existing data loading tests still pass", "expected": "all tests pass, 0 failures", "item_id": "duckdb-tpcds-trailing-delimiter-regression", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/base/test_trailing_delimiter_loading.py -v", "description": "New trailing delimiter regression tests pass", "expected": "all tests pass including with-trailing and without-trailing variants", "item_id": "duckdb-tpcds-trailing-delimiter-regression", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Smoke test for TPC-H (must not regress)", "expected": "all fast tests pass", "item_id": "duckdb-tpcds-trailing-delimiter-regression", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": [], "notes": "In DuckDBNativeHandler.load_table() (data_loading.py ~line 640-666):\n1. Import and call has_trailing_delimiter(file_path, delimiter, col_names) on the first shard\n2. Use get_column_names_with_trailing() to conditionally add dummy column\n3. Replace SELECT * EXCLUDE (_trailing_delimiter_) with explicit column projection\n (SELECT col1, col2, ... FROM read_csv(...)) for robustness\n4. Keep null_padding=true for safety with both file variants\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix DuckDB pipe-delimited loader to detect trailing delimiter per file and project explicit schema columns instead of SELECT * EXCLUDE", "wid": "w1"}, {"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": [], "notes": "In benchbox/cli/commands/run.py (~lines 2243-2296):\n- After benchmark execution, if result.validation_status == \"FAILED\", call ctx.exit(1)\n- Also check the orchestrated run path (~lines 1512-1554)\n- Ensure consistent behavior: either write a failed result artifact OR exit non-zero with clear message\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix CLI exit code: benchbox run must return non-zero when benchmark status is FAILED", "wid": "w2"}, {"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": ["w1", "w2"], "notes": "Unit tests:\n- Pipe-delimited load WITH trailing delimiter (existing behavior preserved)\n- Pipe-delimited load WITHOUT trailing delimiter (new behavior works)\n- TPC-DS load path with .dat.zst shards and no trailing delimiter\nIntegration/CLI test:\n- benchbox run exit code is non-zero on benchmark failure\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for trailing delimiter handling and exit code propagation", "wid": "w3"}, {"evidence": null, "item_id": "duckdb-tpcds-trailing-delimiter-regression", "needs": ["w1", "w2", "w3"], "notes": "Run: benchbox run --platform duckdb --benchmark tpcds --scale 10 --phases load,power \\\n --platform-option driver_version=1.0.0 --platform-option driver_auto_install=true --non-interactive\nConfirm:\n- No _trailing_delimiter_ binder errors in output\n- Tables load successfully, result JSON is produced\n- Non-zero exit on intentionally induced failure (e.g., bad data file)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify fix with live DuckDB TPC-DS run at scale 10", "wid": "w4"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "connect to the PostgreSQL catalog via psycopg", "instead": "use the ATTACH 'ducklake:postgres:...' connection string", "why": "DuckLake reads the catalog through DuckDB's own `postgres` extension"}, {"dont": "duplicate PG connection-arg parsing", "instead": "reuse them", "why": "POSTGRES_FAMILY_* helpers already exist"}, {"dont": "require S3 credentials for the default path", "instead": "only create the S3 secret when data_path is a cloud URI", "why": "most benchmark runs are local"}], "approach": "Build on the base adapter. create_connection already ATTACHes a catalog after\nINSTALL/LOAD ducklake; generalize it to switch the ATTACH string on a `catalog`\noption and to prepend INSTALL sqlite / INSTALL postgres / INSTALL httpfs as\nneeded. Reuse POSTGRES_FAMILY_* option helpers for PG params and cloud_storage\nhelpers to detect s3:// DATA_PATH. Keep S3 secret creation optional and\ncredential-chain-first so no creds are needed for local runs. Gate all live\nextended-backend tests behind availability/credential skips.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-10T00:00:00Z", "completed_pr": null, "created_at": "2026-07-07T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-catalog-storage-backends", "id": 93, "reason": "DuckLake docs explicitly flag MySQL as not recommended (compatibility issues)", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Reason cites DuckLake docs explicitly flagging MySQL catalog as not recommended due to compatibility issues; upstream-blocked by design.", "summary": "MySQL catalog backend"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-catalog-storage-backends", "id": 94, "reason": "S3 first; other object stores are a follow-on once the secret/httpfs path is proven", "resolution": "promoted", "resolved_item": "gcs-azure-object-storage-backends-for-data-path", "resolved_reason": null, "summary": "GCS / Azure object storage backends for DATA_PATH"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-catalog-storage-backends", "id": 95, "reason": "Single-process benchmarking parity first; concurrency benchmarks are a separate research item", "resolution": "promoted", "resolved_item": "multi-client-concurrent-writer-benchmark-scenarios-the-multi", "resolved_reason": null, "summary": "Multi-client / concurrent-writer benchmark scenarios (the 'multiplayer DuckDB' story)"}], "deps": ["ducklake-platform-adapter"], "description": "Extend the DuckLake platform adapter (see ducklake-platform-adapter) beyond the\ndefault DuckDB-file catalog + local storage to the full v1 scope:\n\n - Catalog metadata backends: SQLite and PostgreSQL (in addition to DuckDB).\n - Data storage: S3 / cloud object storage for DATA_PATH (in addition to local).\n\nDuckLake's catalog backend is chosen by the ATTACH connection string, each\nneeding a DuckDB extension:\n duckdb : ATTACH 'ducklake:meta.ducklake' (default; done in base item)\n sqlite : INSTALL sqlite; ATTACH 'ducklake:sqlite:meta.sqlite' (DATA_PATH ...)\n postgres: INSTALL postgres; ATTACH 'ducklake:postgres:dbname=... host=...' (DATA_PATH ...)\n(MySQL is explicitly excluded \u2014 DuckLake docs flag it as not recommended.)\n\nCloud storage is orthogonal to the catalog: DATA_PATH can be 's3://bucket/prefix/'\nwith httpfs + a DuckDB secret:\n INSTALL httpfs;\n CREATE SECRET (TYPE s3, PROVIDER credential_chain); -- or KEY_ID/SECRET/REGION\n ATTACH 'ducklake:...' AS lake (DATA_PATH 's3://bucket/bench/');\n\nSelection is surfaced via --platform-option catalog=duckdb|sqlite|postgres and\n--platform-option data_path=.\n\nSources: https://ducklake.select/docs/stable/duckdb/usage/choosing_a_catalog_database ,\nhttps://duckdb.org/docs/lts/core_extensions/ducklake", "id": "ducklake-catalog-storage-backends", "preserves": ["Existing PostgreSQL-family platforms (postgresql, pg_duckdb, timescaledb) and their option plumbing are unaffected", "No new hard dependency: sqlite/postgres/httpfs are DuckDB runtime extensions, not Python packages; the global pyproject `duckdb` pin is untouched", "The default DuckDB-file catalog + local storage path from ducklake-platform-adapter keeps working unchanged when no catalog/data_path option is given"], "prior_art": [{"concept": "benchbox/core/platform_registry.py DeploymentCapability:210 (requires_cloud_storage, requires_network, auth_methods) \u2014 EXTEND the ducklake metadata deployment_modes for self-hosted-PG and cloud-S3", "decision": "extend", "path": "benchbox/core/platform_registry.py"}, {"concept": "benchbox/platforms/__init__.py _OPTION_SPEC_ROWS:604 + register_config_builder:826 \u2014 EXTEND: add catalog/data_path/PG/S3 option specs and (if creds persistence needed) a config builder", "decision": "extend", "path": "benchbox/platforms/__init__.py"}, {"concept": "benchbox/platforms/base/config_utils.py POSTGRES_FAMILY_PLATFORM_FIELDS/POSTGRES_FAMILY_BASE_OPTIONS + make_platform_config_builder \u2014 REUSE for the postgres-catalog connection params (pg_duckdb.py:419 shows usage)", "decision": "reuse", "path": "benchbox/platforms/base/config_utils.py"}, {"concept": "benchbox/platforms/duckdb.py extension-install idiom (_try_delta_scan:136, _try_iceberg_scan:153) \u2014 REUSE for INSTALL/LOAD sqlite|postgres|httpfs", "decision": "reuse", "path": "benchbox/platforms/duckdb.py"}, {"concept": "benchbox/utils/cloud_storage.py is_cloud_path/get_cloud_path_info (used in duckdb.py:844-848) \u2014 REUSE to detect and route s3:// DATA_PATH", "decision": "reuse", "path": "benchbox/utils/cloud_storage.py"}, {"concept": "tests/integration/test_open_table_formats.py:715-733 TestDuckLakeFormatSmoke extension-availability guard \u2014 REUSE the skip-guard pattern for extension/credential gating", "decision": "reuse", "path": "tests/integration/test_open_table_formats.py:715-733"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/pg_duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/postgresql.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/utils/cloud_storage.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/ducklake.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/comparison-matrix.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/ducklake.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/support-status.md"}, {"kind": "only_modify", "path_glob": "tests/integration/test_ducklake_integration.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_ducklake_adapter.py"}], "state": "done", "title": "DuckLake platform: SQLite + PostgreSQL catalogs and S3/cloud data storage", "verifications": [{"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01 --platform-option catalog=sqlite", "description": "SQLite catalog runs TPC-H", "expected": "queries succeed; metadata.sqlite + Parquet written", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01 --platform-option catalog=postgres --platform-option pg_host=localhost", "description": "PostgreSQL catalog runs TPC-H (with a reachable PG server + pre-created catalog DB)", "expected": "queries succeed; catalog rows in the Postgres ducklake_catalog DB", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01 --platform-option data_path=s3:///bench/", "description": "S3 DATA_PATH (with creds present) writes Parquet to the bucket", "expected": "queries succeed; Parquet objects appear under s3:///bench/", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/integration/test_ducklake_integration.py -q", "description": "Extended-backend integration tests pass or skip cleanly", "expected": "sqlite case passes; postgres/s3 cases pass or skip when unavailable", "item_id": "ducklake-catalog-storage-backends", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": [], "notes": "Generalize create_connection to build the ATTACH string per catalog; validate\ncatalog in {duckdb,sqlite,postgres}. SQLite: INSTALL sqlite; ATTACH\n'ducklake:sqlite:.sqlite' AS lake (DATA_PATH '').\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add catalog-backend selection (--platform-option catalog=duckdb|sqlite|postgres) + SQLite backend", "wid": "w1"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": ["w1"], "notes": "INSTALL postgres; ATTACH 'ducklake:postgres:dbname= host= user=... password=...'\nAS lake (DATA_PATH ''). Pull PG connection params from platform options/env via\nthe POSTGRES_FAMILY_* helpers. Catalog DB must be pre-created (document this).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add PostgreSQL catalog backend, reusing PG-family option/credential machinery", "wid": "w2"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": [], "notes": "When data_path is a cloud URI (is_cloud_path): INSTALL httpfs; create an S3 secret\n(credential_chain by default, or explicit KEY_ID/SECRET/REGION from options) before\nATTACH with DATA_PATH 's3://...'. Extend ducklake metadata deployment_modes with a\ncloud mode (requires_cloud_storage/requires_network).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add S3/cloud DATA_PATH support (httpfs + DuckDB secret + cloud_storage routing)", "wid": "w3"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": ["w1", "w2", "w3"], "notes": "tests/integration/test_ducklake_integration.py: add a SQLite-catalog TPC-H SF 0.01\ncase; a PostgreSQL-catalog case behind the existing PG fixtures/marker; an S3 case\nskipped when creds absent. Guard on ducklake/sqlite/postgres/httpfs extension\navailability like TestDuckLakeFormatSmoke.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integration tests for sqlite/postgres catalogs and s3 storage (extension/creds-gated)", "wid": "w4"}, {"evidence": null, "item_id": "ducklake-catalog-storage-backends", "needs": ["w1", "w2", "w3"], "notes": "Document the three catalog backends, local vs S3 DATA_PATH, required extensions,\nand the pre-created-PG-database prerequisite. Update deployment-mode columns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update docs: catalog-backend + storage options in ducklake.md, comparison-matrix, support-status", "wid": "w5"}], "worktree": "ducklake-platform"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-03T00:00:00Z", "completed_pr": null, "created_at": "2026-01-31T00:00:00Z", "deferrals": [], "deps": [], "description": "Implement DuckLake open table format support on par with existing Delta Lake\nand Iceberg implementations. DuckLake is DuckDB's native open table format\nthat provides ACID transactions, time travel, and schema evolution.\n\nThis integration follows the established pattern for open table formats in\nBenchBox, including:\n- Format converter for TBL to DuckLake conversion\n- Format capability registration with platform support matrix\n- Maintenance operations for ACID workloads\n- CLI integration for the convert command\n- Comprehensive test coverage", "id": "ducklake-open-table-format", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Add DuckLake Open Table Format Support", "verifications": [], "work": [], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "return 'duckdb' from get_target_dialect and inherit execution", "dont": "add a new SQL dialect or query driver", "why": "DuckLake IS DuckDB"}, {"INSTEAD": "registration in _OPTIONAL_ADAPTERS is sufficient", "dont": "hardcode argparse platform choices", "why": "CLI choices come from PlatformRegistry.get_available_platforms()"}, {"INSTEAD": "enforce >=1.3 at runtime in create_connection with a clear error instead", "dont": "raise the global pyproject `duckdb` floor to >=1.3", "why": "the 1.0 floor is deliberate for duckdb-wasm on-disk-format compat (pyproject.toml:90-94)"}, {"INSTEAD": "let inherited DuckDB load_data write into `lake` after USE", "dont": "route load through the ducklake_converter format pipeline", "why": "native platform load should CREATE/COPY into the attached catalog directly"}], "approach": "Subclass, don't rebuild. DuckLake is DuckDB + a catalog ATTACH, so\nDuckLakeAdapter(DuckDBAdapter) mirrors PgDuckDBAdapter(PostgreSQLAdapter):\nsuper().create_connection() yields a DuckDB connection; INSTALL/LOAD ducklake;\nATTACH the DuckLake catalog; USE it so it becomes the default catalog. Inherited\ncreate_schema + load_data (via DataLoader) then write into DuckLake with no\noverride \u2014 verify this in w3 before assuming it. Declare platform_family/\ninherits_from='duckdb' so dialect translation and benchmark compatibility are\ninherited (motherduck pattern). Reuse the exact ATTACH SQL from\nducklake_maintenance.py:186.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-07T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-platform-adapter", "id": 96, "reason": "Tracked in ducklake-catalog-storage-backends (depends on this item)", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Tracked in ducklake-catalog-storage-backends, which is now under _project/DONE/ducklake-platform/ (already completed).", "summary": "SQLite and PostgreSQL catalog backends; S3/cloud object storage for DATA_PATH"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-platform-adapter", "id": 97, "reason": "Adapter parity first; lakehouse-feature benchmarks are a separate research item", "resolution": "promoted", "resolved_item": "ducklake-specific-benchmarks-time-travel-schema-evolution-sn", "resolved_reason": null, "summary": "DuckLake-specific benchmarks (time travel, schema evolution, snapshot/maintenance workloads)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-platform-adapter", "id": 98, "reason": "Inherits DuckDB's unsupported_benchmarks set unchanged", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason is a statement that DuckLake inherits DuckDB's existing unsupported-benchmark set unchanged, not an action item.", "summary": "Vector search / any benchmark DuckDB itself does not support"}], "deps": [], "description": "Add DuckLake as a runnable BenchBox platform so it can be benchmarked on\nTPC-H/TPC-DS/etc. alongside the other 45 SQL engines.\n\nDuckLake (GA v1.0, April 2026 \u2014 https://ducklake.select) is an open lakehouse\nformat: table DATA is Parquet on local disk or object storage, and all catalog\nMETADATA (schemas, snapshots, stats, file lists) lives in a standard SQL\ndatabase (DuckDB / SQLite / PostgreSQL) instead of Iceberg manifests or Delta\nJSON logs. It ships as a DuckDB extension and uses DuckDB's engine + SQL\ndialect unchanged:\n INSTALL ducklake;\n ATTACH 'ducklake:' AS lake (DATA_PATH '');\n USE lake; -- then normal CREATE TABLE / COPY / SELECT\n\nCurrent state: DuckLake already exists in BenchBox as a table *format*\n(converter benchbox/utils/format_converters/ducklake_converter.py,\nDUCKLAKE_CAPABILITY in benchbox/platforms/base/format_capabilities.py:180,\nDataFrame maintenance ops in\nbenchbox/platforms/dataframe/ducklake_maintenance.py) but NOT as a *platform*.\nThere is no DuckLakeAdapter and no `ducklake` entry in PlatformRegistry. This\nTODO adds the adapter and wires it in.\n\nThis item = the MVP: DuckLakeAdapter subclassing DuckDBAdapter, with a\nDuckDB-file catalog + local Parquet storage, full registry wiring, docs, and\nlive TPC-H/TPC-DS validation. SQLite/PostgreSQL catalogs and S3 storage are a\nfollow-on (see ducklake-catalog-storage-backends).\n\nEvidence (pinned): the worktree venv resolves duckdb==1.3.2; the `ducklake`\nextension requires DuckDB >= 1.3 (spec 0.1/0.2; 1.4 for 0.3). The global\n`duckdb` pin in pyproject.toml is intentionally `>=1.0.0,<2.0.0` (kept at 1.0\nfloor for duckdb-wasm on-disk-format compatibility \u2014 see the comment at\npyproject.toml:90-94). Do NOT raise that floor; enforce the >=1.3 requirement\nat runtime in the adapter instead.", "id": "ducklake-platform-adapter", "preserves": ["DuckDBAdapter behavior is unchanged \u2014 DuckLakeAdapter only overrides create_connection (+ catalog targeting if w3 requires it); all existing tests/integration/*duckdb* keep passing", "PlatformRegistry metadata/support-status validation continues to pass for all platforms", "The existing DuckLake table-FORMAT path (ducklake_converter.py, DUCKLAKE_CAPABILITY) is untouched", "The global pyproject.toml `duckdb` pin stays >=1.0.0,<2.0.0 (duckdb-wasm on-disk-format compat)"], "prior_art": [{"concept": "benchbox/platforms/base/format_capabilities.py:180 DUCKLAKE_CAPABILITY + utils/format_converters/ducklake_converter.py \u2014 LEAVE AS-IS (orthogonal format-conversion pipeline; native adapter load writes DuckLake via the attached catalog, not the converter)", "decision": "reuse", "path": "benchbox/platforms/base/format_capabilities.py:180"}, {"concept": "benchbox/platforms/dataframe/ducklake_maintenance.py:168-189: proven INSTALL/LOAD ducklake + ATTACH 'ducklake:...' (DATA_PATH ...) SQL \u2014 REUSE the exact statements", "decision": "reuse", "path": "benchbox/platforms/dataframe/ducklake_maintenance.py:168-189"}, {"concept": "benchbox/platforms/duckdb.py: create_connection:709, create_schema:775, load_data:838, execute_query:917, plan capture:1038, extension-install idiom _try_delta_scan:136 \u2014 REUSE/INHERIT unchanged; override only create_connection + catalog targeting", "decision": "reuse", "path": "benchbox/platforms/duckdb.py"}, {"concept": "benchbox/platforms/motherduck.py: platform_family/inherits_from='duckdb' metadata + get_target_dialect via resolve_dialect_for_query_translation \u2014 REUSE the family-inheritance pattern for dialect + benchmark compatibility", "decision": "reuse", "path": "benchbox/platforms/motherduck.py"}, {"concept": "benchbox/platforms/pg_duckdb.py: PgDuckDBAdapter(PostgreSQLAdapter) overrides create_connection to install/configure an extension \u2014 EXTEND: primary structural template for DuckLakeAdapter(DuckDBAdapter)", "decision": "extend", "path": "benchbox/platforms/pg_duckdb.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/ducklake_maintenance.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/utils/format_converters/ducklake_converter.py"}, {"kind": "only_modify", "path_glob": "CHANGELOG.md"}, {"kind": "only_modify", "path_glob": "README.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/ducklake.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/comparison-matrix.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/ducklake.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/index.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/support-status.md"}, {"kind": "only_modify", "path_glob": "tests/integration/test_ducklake_integration.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_ducklake_adapter.py"}], "state": "done", "title": "Add DuckLake as a first-class benchmark platform (core adapter)", "verifications": [{"command": "uv run -- benchbox platforms list", "description": "Platform is discoverable", "expected": "output includes 'ducklake'", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_abc_conformance.py tests/unit/platforms/test_adapter_validation_consistency.py -q", "description": "Registry metadata/support-status validation + ABC conformance pass with ducklake registered", "expected": "all pass; PlatformRegistry validates ducklake metadata against its support_status without error", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_ducklake_adapter.py -q", "description": "Adapter unit tests pass", "expected": "all pass, 0 failures", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01", "description": "Live TPC-H at SF 0.01 (duckdb catalog, local storage)", "expected": "all 22 queries succeed; Parquet data + .ducklake catalog written under benchmark_runs/", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpcds --scale 0.01", "description": "Live TPC-DS at SF 0.01", "expected": "queries succeed; results validate", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "ducklake-platform-adapter", "needs": [], "notes": "uv run -- python -c \"import duckdb; c=duckdb.connect(); c.execute('INSTALL ducklake'); c.execute('LOAD ducklake'); print(duckdb.__version__)\"\nRecord duckdb version + PASS in the PR body / commit message. Confirms the\npinned-evidence version and that the extension is fetchable in CI.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate DuckLake extension loads on the pinned DuckDB (>=1.3.2) and commit a compact summary", "wid": "w1"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w1"], "notes": "Model on pg_duckdb.py. Implement: platform_name -> 'DuckLake';\nget_target_dialect -> 'duckdb'; add_cli_arguments (--ducklake-metadata-path,\n--ducklake-data-path); from_config (resolve metadata_path default under\nbenchmark_runs/databases, data_path default sibling ducklake_data/);\n__init__; create_connection = super().create_connection() then INSTALL/LOAD\nducklake, ATTACH 'ducklake:' AS lake (DATA_PATH ''), USE lake,\nwith an actionable RuntimeError if the extension fails (point to DuckDB>=1.3);\nget_platform_info adds catalog_backend/data_path/ducklake extension version.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create benchbox/platforms/ducklake.py \u2014 DuckLakeAdapter(DuckDBAdapter), DuckDB-file catalog + local DATA_PATH", "wid": "w2"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w2"], "notes": "Read benchbox/platforms/base/data_loading.py DataLoader/handler SQL. If it\nissues unqualified DDL/COPY it flows into `lake` after USE \u2014 no override.\nIf it hard-qualifies `main.` or opens its own connection, override\ncreate_schema/load_data to qualify with the catalog name (lake.main.).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm inherited create_schema/load_data write into the attached catalog; override to qualify if not", "wid": "w3"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w2"], "notes": "_OPTIONAL_ADAPTERS (:1234) add ('ducklake','benchbox.platforms.ducklake','DuckLakeAdapter');\n_PLATFORM_SUPPORT_STATUS (:44) add 'ducklake':'experimental';\n_PLATFORM_METADATA_JSON (:121) add a block modeled on motherduck with\ncapabilities supports_sql:true, supports_dataframe:false, default_mode:'sql',\nplatform_family:'duckdb', inherits_from:'duckdb', empty unsupported_benchmarks,\nand requirements noting DuckDB>=1.3. Registry validation cross-checks all three.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register in benchbox/core/platform_registry.py (3 places, family=duckdb)", "wid": "w4"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w4"], "notes": "_LAZY_ADAPTER_ROWS (:68) add 'DuckLakeAdapter|.ducklake';\n_EXPORT_NAMES (:276) add 'DuckLakeAdapter';\n_OPTION_SPEC_ROWS (:604) add ducklake|metadata_path|... and ducklake|data_path|...\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire lazy import/export + option specs in benchbox/platforms/__init__.py", "wid": "w5"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w3", "w5"], "notes": "from_config resolution, catalog-string construction, get_target_dialect=='duckdb',\nplatform appears in registry. ABC conformance (test_abc_conformance.py) and\nadapter-validation suites parametrize over the registry automatically. Add a\nmodule-level pytestmark marker (CI enforces via test_marker_strategy.py).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests: tests/unit/platforms/test_ducklake_adapter.py + run ABC-conformance suite", "wid": "w6"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w4"], "notes": "New docs/platforms/ducklake.md (model on motherduck.md); add to\ndocs/platforms/index.md list + {toctree} (:183-225); add rows to\ncomparison-matrix.md and support-status.md; add to README.md SQL Platforms\nlist (:60); add a CHANGELOG.md entry.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Docs: ducklake.md + index toctree, comparison-matrix, support-status, README, CHANGELOG", "wid": "w7"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w6"], "notes": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01\n(then tpcds, then SF 1). Confirm all queries pass, Parquet + .ducklake catalog\nappear on disk, results validate, and TPC-H power results/row-counts match the\n`duckdb` platform at the same SF (same engine).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Live validation: TPC-H + TPC-DS at SF 0.01 then SF 1 (duckdb catalog, local); cross-check vs duckdb platform", "wid": "w8"}], "worktree": "ducklake-platform"} +{"anti_patterns": [{"dont": "add a new SQL dialect or query driver", "instead": "return 'duckdb' from get_target_dialect and inherit execution", "why": "DuckLake IS DuckDB"}, {"dont": "hardcode argparse platform choices", "instead": "registration in _OPTIONAL_ADAPTERS is sufficient", "why": "CLI choices come from PlatformRegistry.get_available_platforms()"}, {"dont": "raise the global pyproject `duckdb` floor to >=1.3", "instead": "enforce >=1.3 at runtime in create_connection with a clear error instead", "why": "the 1.0 floor is deliberate for duckdb-wasm on-disk-format compat (pyproject.toml:90-94)"}, {"dont": "route load through the ducklake_converter format pipeline", "instead": "let inherited DuckDB load_data write into `lake` after USE", "why": "native platform load should CREATE/COPY into the attached catalog directly"}], "approach": "Subclass, don't rebuild. DuckLake is DuckDB + a catalog ATTACH, so\nDuckLakeAdapter(DuckDBAdapter) mirrors PgDuckDBAdapter(PostgreSQLAdapter):\nsuper().create_connection() yields a DuckDB connection; INSTALL/LOAD ducklake;\nATTACH the DuckLake catalog; USE it so it becomes the default catalog. Inherited\ncreate_schema + load_data (via DataLoader) then write into DuckLake with no\noverride \u2014 verify this in w3 before assuming it. Declare platform_family/\ninherits_from='duckdb' so dialect translation and benchmark compatibility are\ninherited (motherduck pattern). Reuse the exact ATTACH SQL from\nducklake_maintenance.py:186.\n", "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-09T00:00:00Z", "completed_pr": null, "created_at": "2026-07-07T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-platform-adapter", "id": 96, "reason": "Tracked in ducklake-catalog-storage-backends (depends on this item)", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Tracked in ducklake-catalog-storage-backends, which is now under _project/DONE/ducklake-platform/ (already completed).", "summary": "SQLite and PostgreSQL catalog backends; S3/cloud object storage for DATA_PATH"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-platform-adapter", "id": 97, "reason": "Adapter parity first; lakehouse-feature benchmarks are a separate research item", "resolution": "promoted", "resolved_item": "ducklake-specific-benchmarks-time-travel-schema-evolution-sn", "resolved_reason": null, "summary": "DuckLake-specific benchmarks (time travel, schema evolution, snapshot/maintenance workloads)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-platform-adapter", "id": 98, "reason": "Inherits DuckDB's unsupported_benchmarks set unchanged", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason is a statement that DuckLake inherits DuckDB's existing unsupported-benchmark set unchanged, not an action item.", "summary": "Vector search / any benchmark DuckDB itself does not support"}], "deps": [], "description": "Add DuckLake as a runnable BenchBox platform so it can be benchmarked on\nTPC-H/TPC-DS/etc. alongside the other 45 SQL engines.\n\nDuckLake (GA v1.0, April 2026 \u2014 https://ducklake.select) is an open lakehouse\nformat: table DATA is Parquet on local disk or object storage, and all catalog\nMETADATA (schemas, snapshots, stats, file lists) lives in a standard SQL\ndatabase (DuckDB / SQLite / PostgreSQL) instead of Iceberg manifests or Delta\nJSON logs. It ships as a DuckDB extension and uses DuckDB's engine + SQL\ndialect unchanged:\n INSTALL ducklake;\n ATTACH 'ducklake:' AS lake (DATA_PATH '');\n USE lake; -- then normal CREATE TABLE / COPY / SELECT\n\nCurrent state: DuckLake already exists in BenchBox as a table *format*\n(converter benchbox/utils/format_converters/ducklake_converter.py,\nDUCKLAKE_CAPABILITY in benchbox/platforms/base/format_capabilities.py:180,\nDataFrame maintenance ops in\nbenchbox/platforms/dataframe/ducklake_maintenance.py) but NOT as a *platform*.\nThere is no DuckLakeAdapter and no `ducklake` entry in PlatformRegistry. This\nTODO adds the adapter and wires it in.\n\nThis item = the MVP: DuckLakeAdapter subclassing DuckDBAdapter, with a\nDuckDB-file catalog + local Parquet storage, full registry wiring, docs, and\nlive TPC-H/TPC-DS validation. SQLite/PostgreSQL catalogs and S3 storage are a\nfollow-on (see ducklake-catalog-storage-backends).\n\nEvidence (pinned): the worktree venv resolves duckdb==1.3.2; the `ducklake`\nextension requires DuckDB >= 1.3 (spec 0.1/0.2; 1.4 for 0.3). The global\n`duckdb` pin in pyproject.toml is intentionally `>=1.0.0,<2.0.0` (kept at 1.0\nfloor for duckdb-wasm on-disk-format compatibility \u2014 see the comment at\npyproject.toml:90-94). Do NOT raise that floor; enforce the >=1.3 requirement\nat runtime in the adapter instead.", "id": "ducklake-platform-adapter", "preserves": ["DuckDBAdapter behavior is unchanged \u2014 DuckLakeAdapter only overrides create_connection (+ catalog targeting if w3 requires it); all existing tests/integration/*duckdb* keep passing", "PlatformRegistry metadata/support-status validation continues to pass for all platforms", "The existing DuckLake table-FORMAT path (ducklake_converter.py, DUCKLAKE_CAPABILITY) is untouched", "The global pyproject.toml `duckdb` pin stays >=1.0.0,<2.0.0 (duckdb-wasm on-disk-format compat)"], "prior_art": [{"concept": "benchbox/platforms/base/format_capabilities.py:180 DUCKLAKE_CAPABILITY + utils/format_converters/ducklake_converter.py \u2014 LEAVE AS-IS (orthogonal format-conversion pipeline; native adapter load writes DuckLake via the attached catalog, not the converter)", "decision": "reuse", "path": "benchbox/platforms/base/format_capabilities.py:180"}, {"concept": "benchbox/platforms/dataframe/ducklake_maintenance.py:168-189: proven INSTALL/LOAD ducklake + ATTACH 'ducklake:...' (DATA_PATH ...) SQL \u2014 REUSE the exact statements", "decision": "reuse", "path": "benchbox/platforms/dataframe/ducklake_maintenance.py:168-189"}, {"concept": "benchbox/platforms/duckdb.py: create_connection:709, create_schema:775, load_data:838, execute_query:917, plan capture:1038, extension-install idiom _try_delta_scan:136 \u2014 REUSE/INHERIT unchanged; override only create_connection + catalog targeting", "decision": "reuse", "path": "benchbox/platforms/duckdb.py"}, {"concept": "benchbox/platforms/motherduck.py: platform_family/inherits_from='duckdb' metadata + get_target_dialect via resolve_dialect_for_query_translation \u2014 REUSE the family-inheritance pattern for dialect + benchmark compatibility", "decision": "reuse", "path": "benchbox/platforms/motherduck.py"}, {"concept": "benchbox/platforms/pg_duckdb.py: PgDuckDBAdapter(PostgreSQLAdapter) overrides create_connection to install/configure an extension \u2014 EXTEND: primary structural template for DuckLakeAdapter(DuckDBAdapter)", "decision": "extend", "path": "benchbox/platforms/pg_duckdb.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/ducklake_maintenance.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/duckdb.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/utils/format_converters/ducklake_converter.py"}, {"kind": "only_modify", "path_glob": "CHANGELOG.md"}, {"kind": "only_modify", "path_glob": "README.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/platform_registry.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/ducklake.py"}, {"kind": "only_modify", "path_glob": "docs/platforms/comparison-matrix.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/ducklake.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/index.md"}, {"kind": "only_modify", "path_glob": "docs/platforms/support-status.md"}, {"kind": "only_modify", "path_glob": "tests/integration/test_ducklake_integration.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_ducklake_adapter.py"}], "state": "done", "title": "Add DuckLake as a first-class benchmark platform (core adapter)", "verifications": [{"command": "uv run -- benchbox platforms list", "description": "Platform is discoverable", "expected": "output includes 'ducklake'", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_abc_conformance.py tests/unit/platforms/test_adapter_validation_consistency.py -q", "description": "Registry metadata/support-status validation + ABC conformance pass with ducklake registered", "expected": "all pass; PlatformRegistry validates ducklake metadata against its support_status without error", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_ducklake_adapter.py -q", "description": "Adapter unit tests pass", "expected": "all pass, 0 failures", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01", "description": "Live TPC-H at SF 0.01 (duckdb catalog, local storage)", "expected": "all 22 queries succeed; Parquet data + .ducklake catalog written under benchmark_runs/", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- benchbox run --platform ducklake --benchmark tpcds --scale 0.01", "description": "Live TPC-DS at SF 0.01", "expected": "queries succeed; results validate", "item_id": "ducklake-platform-adapter", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "ducklake-platform-adapter", "needs": [], "notes": "uv run -- python -c \"import duckdb; c=duckdb.connect(); c.execute('INSTALL ducklake'); c.execute('LOAD ducklake'); print(duckdb.__version__)\"\nRecord duckdb version + PASS in the PR body / commit message. Confirms the\npinned-evidence version and that the extension is fetchable in CI.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate DuckLake extension loads on the pinned DuckDB (>=1.3.2) and commit a compact summary", "wid": "w1"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w1"], "notes": "Model on pg_duckdb.py. Implement: platform_name -> 'DuckLake';\nget_target_dialect -> 'duckdb'; add_cli_arguments (--ducklake-metadata-path,\n--ducklake-data-path); from_config (resolve metadata_path default under\nbenchmark_runs/databases, data_path default sibling ducklake_data/);\n__init__; create_connection = super().create_connection() then INSTALL/LOAD\nducklake, ATTACH 'ducklake:' AS lake (DATA_PATH ''), USE lake,\nwith an actionable RuntimeError if the extension fails (point to DuckDB>=1.3);\nget_platform_info adds catalog_backend/data_path/ducklake extension version.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create benchbox/platforms/ducklake.py \u2014 DuckLakeAdapter(DuckDBAdapter), DuckDB-file catalog + local DATA_PATH", "wid": "w2"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w2"], "notes": "Read benchbox/platforms/base/data_loading.py DataLoader/handler SQL. If it\nissues unqualified DDL/COPY it flows into `lake` after USE \u2014 no override.\nIf it hard-qualifies `main.` or opens its own connection, override\ncreate_schema/load_data to qualify with the catalog name (lake.main.
).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Confirm inherited create_schema/load_data write into the attached catalog; override to qualify if not", "wid": "w3"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w2"], "notes": "_OPTIONAL_ADAPTERS (:1234) add ('ducklake','benchbox.platforms.ducklake','DuckLakeAdapter');\n_PLATFORM_SUPPORT_STATUS (:44) add 'ducklake':'experimental';\n_PLATFORM_METADATA_JSON (:121) add a block modeled on motherduck with\ncapabilities supports_sql:true, supports_dataframe:false, default_mode:'sql',\nplatform_family:'duckdb', inherits_from:'duckdb', empty unsupported_benchmarks,\nand requirements noting DuckDB>=1.3. Registry validation cross-checks all three.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Register in benchbox/core/platform_registry.py (3 places, family=duckdb)", "wid": "w4"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w4"], "notes": "_LAZY_ADAPTER_ROWS (:68) add 'DuckLakeAdapter|.ducklake';\n_EXPORT_NAMES (:276) add 'DuckLakeAdapter';\n_OPTION_SPEC_ROWS (:604) add ducklake|metadata_path|... and ducklake|data_path|...\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire lazy import/export + option specs in benchbox/platforms/__init__.py", "wid": "w5"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w3", "w5"], "notes": "from_config resolution, catalog-string construction, get_target_dialect=='duckdb',\nplatform appears in registry. ABC conformance (test_abc_conformance.py) and\nadapter-validation suites parametrize over the registry automatically. Add a\nmodule-level pytestmark marker (CI enforces via test_marker_strategy.py).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unit tests: tests/unit/platforms/test_ducklake_adapter.py + run ABC-conformance suite", "wid": "w6"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w4"], "notes": "New docs/platforms/ducklake.md (model on motherduck.md); add to\ndocs/platforms/index.md list + {toctree} (:183-225); add rows to\ncomparison-matrix.md and support-status.md; add to README.md SQL Platforms\nlist (:60); add a CHANGELOG.md entry.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Docs: ducklake.md + index toctree, comparison-matrix, support-status, README, CHANGELOG", "wid": "w7"}, {"evidence": null, "item_id": "ducklake-platform-adapter", "needs": ["w6"], "notes": "uv run -- benchbox run --platform ducklake --benchmark tpch --scale 0.01\n(then tpcds, then SF 1). Confirm all queries pass, Parquet + .ducklake catalog\nappear on disk, results validate, and TPC-H power results/row-counts match the\n`duckdb` platform at the same SF (same engine).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Live validation: TPC-H + TPC-DS at SF 0.01 then SF 1 (duckdb catalog, local); cross-check vs duckdb platform", "wid": "w8"}], "worktree": "ducklake-platform"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-10T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "ducklake-post-merge-review-followups", "id": 5, "reason": "Requires new phase instrumentation; the coarse ducklake-vs-duckdb comparison is available today. Track only if w11/w12 conclude attribution is required for beta.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Sub-task of the still-open parent item; belongs inside its existing work items rather than a new TODO. (tracked by ducklake-post-merge-review-followups)", "summary": "Fine-grained attribution of catalog-metadata ops vs Parquet-scan vs engine time"}], "deps": ["ducklake-postgres-credential-leak-redaction"], "description": "Follow-ups from a deep post-merge review of the shipped DuckLake adapter\n(#1082 core adapter, #1096 catalog backends + S3; both in _project/DONE/).\nThe confirmed credential leak is tracked separately and gates this item\n(see deps.needs: ducklake-postgres-credential-leak-redaction).\n\nHeadline risk is COVERAGE, not code: the adapter's core live behavior runs in\nZERO CI lanes. tests/integration/test_ducklake_integration.py is module-marked\n`slow`; the core in-process ATTACH class (TestDuckLakeLiveConnection) and the\nPostgres/S3 classes additionally carry `live_integration`. Every default lane\nselects `not (slow or ... or live_integration)`; the `-m slow` PR jobs are\nfile-scoped to OTHER files; release-canary/validate-main-pr exclude\nlive_integration. Net: classes 1/3/4 run in no lane; the SQLite class runs only\nat the main-release gate -- never on the develop PR that merged it. Both feature\nPRs squash-auto-merged with no human review (platforms/ is not a CODEOWNERS\npath). The Postgres and S3 classes have never executed anywhere.\n\nTwo related process blind spots are recorded under _project/blind-spots/\n(auto-revert blame heuristic; slow+live_integration coverage theater).\n", "id": "ducklake-post-merge-review-followups", "preserves": [], "prior_art": [{"concept": "benchbox/core/platform_registry.py:128 ducklake deployment_modes (two orthogonal axes encoded as three pseudo-modes) \u2014 SUPERSEDE/annotate (w10)", "decision": "supersede", "path": "benchbox/core/platform_registry.py:128"}, {"concept": "benchbox/platforms/ducklake.py:527-544 handle_existing_database local metadata_path reuse detection \u2014 SUPERSEDE for postgres with a server-side probe (w6)", "decision": "supersede", "path": "benchbox/platforms/ducklake.py:527-544"}, {"concept": "tests/integration/platforms/test_postgresql_live.py skip_unless_docker_service + the nightly live_postgresql job \u2014 REUSE the docker-service bring-up pattern for the PG class (w1)", "decision": "reuse", "path": "tests/integration/platforms/test_postgresql_live.py"}, {"concept": "tests/unit/test_marker_strategy.py + `make lint-markers` (CI-only marker gate) \u2014 EXTEND with a lane-coverage meta-check (w4)", "decision": "extend", "path": "tests/unit/test_marker_strategy.py"}], "priority": "medium-high", "scope": [], "state": "planning", "title": "DuckLake platform: post-merge review follow-ups (CI coverage, live-test execution, reuse/force, docs, governance)", "verifications": [{"command": "uv run -- python -m pytest tests/integration/test_ducklake_integration.py -m live_integration -n 0", "description": "PG and S3 live classes execute (not skip) against real infra and pass", "expected": "PG + S3 classes run and pass (or reveal fixable failures), not silently skipped", "item_id": "ducklake-post-merge-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "", "description": "Lane-coverage meta-check flags any zero-lane integration module", "expected": "test_ducklake_integration.py is selected by at least one lane after w3", "item_id": "ducklake-post-merge-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "Find the make docker-postgres target (e.g. make test-docker-up-postgresql).\nPre-create the ducklake_catalog database (adapter does NOT CREATE DATABASE).\nRun: uv run -- python -m pytest tests/integration/test_ducklake_integration.py -m live_integration -n 0 -k Postgres\nReport and fix every failure. This class has never executed against a real server.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Bring up live PostgreSQL, pre-create the catalog DB, and run the DuckLake PG live_integration class; fix what breaks", "wid": "w1"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "platform_registry.py:128 encodes catalog(duckdb/sqlite/postgres) x storage(local/s3)\nas local/postgres_catalog/cloud_storage. Consequence: `ducklake:postgres_catalog`\nvalidates (adapter_factory.py:190) but does nothing (adapter reads only\ncatalog=), and docs/platforms/comparison-matrix.md:81 lists them beside real\n:mode platforms as mutually exclusive. Collapse to one mode or mark the entries\ndescriptive-only; correct the matrix row.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Fix the deployment_modes modeling: two orthogonal axes (catalog backend x storage location) presented as three exclusive pseudo-modes", "wid": "w10"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "None exists (registry only records support_status: experimental). Write measurable criteria.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Define the concrete experimental -> beta exit criterion for the ducklake platform", "wid": "w11"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "Network variance sits inside every timed phase (per-query S3 GET in\nexecute_query; PG catalog round-trips in ATTACH/commit). No annotation flags\nremote-backed runs. Decide: quarantine/annotate vs allow to published-results.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Decide the publishability policy for remote-backed (PG catalog / S3 storage) DuckLake results", "wid": "w12"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "#1082/#1096 merged via squash auto-merge, no human review; platforms/ is not a\nsoundness path. Decide whether to add benchbox/platforms/** (or new adapters)\nto .github/CODEOWNERS + auto_merge_soundness_paths.py.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Decide whether platform adapters that produce publishable numbers should be a CODEOWNERS owner-review path", "wid": "w13"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "The adapter runs no compaction/merge/inlining between load and query; a\nmulti-batch load leaves many small Parquet files. If a competitor auto-compacts\nand DuckLake doesn't, scan-time comparisons are unfair. Settle with a\nload->compact->query vs load->query timing delta, then instrument or annotate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Decide whether absence of DuckLake compaction/inlining biases cross-engine comparisons; instrument or document", "wid": "w14"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "Set BENCHBOX_S3_TEST_BUCKET + AWS creds and run the S3 class\n(-m live_integration -k S3). Never executed. Exercise both credential_chain\nand explicit KEY_ID/SECRET paths, and force-recreate leaving orphaned Parquet (see w7).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Run the DuckLake S3 data_path live_integration class against a real bucket; fix what breaks", "wid": "w2"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "TestDuckLakeLiveConnection needs only the ducklake extension (no external\nservice) yet is live_integration -> runs nowhere. Re-marker it (or add a\nlane) so core ATTACH/cursor-scoping/force-recreate/platform_info is exercised\non develop PRs. Keep PG/S3 classes live_integration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Give the extension-only in-process ATTACH class a marker a real lane runs, so core adapter behavior is gated in CI", "wid": "w3"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "Resolve each test module's module+class markers against every workflow/Makefile\n`-m` expression; fail if any module matches no lane. Prevents future\ncoverage-theater. Model on test_marker_strategy.py / lint-markers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add a CI meta-check that fails when an integration test module is selected by zero configured lanes", "wid": "w4"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "tests/integration/test_ducklake_integration.py: DUCKLAKE_AVAILABLE =\n@cache _probe_extension('ducklake') runs at import/collection; a transient\nextension-repo blip caches False and silently skips even the SQLite class that\ngates release-canary. Emit a warning/xfail (not a silent skip) at the release gate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make _probe_extension's cached-False outcome loud instead of a silent session-wide skip", "wid": "w5"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "handle_existing_database keys off local metadata_path.exists(), never true for\na server-side PG catalog -> database_was_reused stays False -> second run hits\ninherited plain CREATE TABLE -> 'table already exists' crash; --force is a no-op.\nProbe the PG catalog server-side (ducklake metadata tables) for reuse; honor\n--force by dropping/recreating the catalog schema. NOTE: current behavior\ncrashes (safe) rather than silently misreporting cold-load timings -- confirmed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Postgres catalog reuse/force: server-side detection so reruns don't crash and --force works", "wid": "w6"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "_reset_ducklake_catalog correctly guards rmtree with `not _data_path_is_cloud`\n(no dangerous deletion -- verified), but cloud data is never reset; new catalog\nonly references new files (results stay correct) yet storage leaks. Clear the\nprefix via the cloud client on force, or document the leak.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "--force with cloud (s3) DATA_PATH leaves orphaned Parquet; clear the prefix or document", "wid": "w7"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "pyproject pins duckdb>=1.0.0,<2.0.0; uv sync can resolve 1.2 which cannot load\nducklake. Mitigated by a clean runtime guard (ducklake.py:606) so it's not a\nconfusing failure -- but an optional-dependency extra `ducklake=[\"duckdb>=1.3,<2.0\"]`\nor a documented requirement would prevent a broken environment.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Express the duckdb>=1.3 floor installably (ducklake extra) or document the requirement", "wid": "w8"}, {"evidence": null, "item_id": "ducklake-post-merge-review-followups", "needs": [], "notes": "#1096's scope_limit forbade the CHANGELOG edit and no follow-up corrected it;\nthe unreleased notes still say 'DuckDB-file catalog + local Parquet storage'.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Correct CHANGELOG.md to reflect the shipped feature (sqlite/postgres catalogs + S3), not just the #1082 MVP", "wid": "w9"}], "worktree": "ducklake-review-followups"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Security and Correctness", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-10T00:00:00Z", "deferrals": [], "deps": [], "description": "CONFIRMED credential leak found in a post-merge review of the shipped DuckLake\nadapter (#1082, #1096; both in _project/DONE/).\n\nWhen catalog=postgres and the ATTACH fails (server unreachable OR wrong\npassword -- the *normal* first-run case), DuckDB raises an IOException whose\nmessage echoes the full libpq connection string, INCLUDING the plaintext\npassword, twice. benchbox/platforms/ducklake.py create_connection() catches\nthis and re-raises it verbatim via `... Underlying error: {e}` (around\nducklake.py:684-695), so the password propagates to:\n - stderr / console / --verbose output\n - the runner's serialized error list (benchbox/core/runner/runner.py:209,253\n stringify exceptions into errors=[...]), which for this tool can be\n exported to result JSON and potentially published.\n\nReproduced live (bogus PG target, connection refused):\n ATTACH 'ducklake:postgres:dbname=cat host=127.0.0.1 user=bob password=SuperSecretPw123 port=5432' AS lake (DATA_PATH ...)\n -> IOException: \"... password=SuperSecretPw123 port=5432 ... Unable to connect\n to Postgres at dbname=cat host=127.0.0.1 user=bob password=SuperSecretPw123 ...\"\n -> `SuperSecretPw123` present in str(e): True\n\nAsymmetry that makes this clearly a miss (not a design choice): the S3 secret\npath in the SAME function already redacts (ducklake.py:669-678 catches the\nCREATE SECRET failure and suppresses key/secret material). The postgres ATTACH\npath was not given the same treatment.\n", "id": "ducklake-postgres-credential-leak-redaction", "preserves": ["Non-postgres (duckdb/sqlite) ATTACH failures keep their informative metadata_path/data_path context", "The S3 CREATE SECRET redaction at ducklake.py:669-678 stays intact", "get_platform_info() continues to omit pg_user/pg_password"], "prior_art": [{"concept": "benchbox/platforms/ducklake.py:669-678 S3 CREATE SECRET redaction (suppresses key/secret in the raised RuntimeError, raise ... from None) \u2014 REUSE the same shape for the postgres ATTACH failure", "decision": "reuse", "path": "benchbox/platforms/ducklake.py:669-678"}, {"concept": "benchbox/platforms/ducklake.py:717-735 get_platform_info already excludes pg_user/pg_password from serialized platform config \u2014 EXTEND the same no-secret discipline to the exception path", "decision": "extend", "path": "benchbox/platforms/ducklake.py:717-735"}], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": "benchbox/platforms/ducklake.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_ducklake_adapter.py"}], "state": "planning", "title": "DuckLake postgres catalog: redact DB password from ATTACH-failure exception (confirmed leak)", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/test_ducklake_adapter.py -q", "description": "New redaction regression test passes and is mutation-sensitive", "expected": "all pass; reverting w1 flips the new test to red", "item_id": "ducklake-postgres-credential-leak-redaction", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_ducklake_adapter.py -k credential -q", "description": "No plaintext password reaches the raised message for a failed postgres ATTACH", "expected": "sentinel password absent from exception text", "item_id": "ducklake-postgres-credential-leak-redaction", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "ducklake-postgres-credential-leak-redaction", "needs": [], "notes": "In ducklake.py create_connection()'s `except Exception as e:` around\nlines 684-695: when self.catalog == 'postgres', do NOT interpolate raw {e}\nand do NOT echo metadata_path/attach_target that embed the connstring.\nPrefer emitting `type(e).__name__` plus host/port/database (already\nnon-secret), OR run the message through a scrubber that strips\n`password=\\S+` / `password='...'`. Keep the non-postgres branches informative.\nAlso confirm the version-guard reject path (ducklake.py:605-618) and the\nouter wrap do not re-expand a redacted message.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Redact the postgres connection string (password) from the ATTACH-failure RuntimeError in create_connection()", "wid": "w1"}, {"evidence": null, "item_id": "ducklake-postgres-credential-leak-redaction", "needs": ["w1"], "notes": "Add a hermetic unit test (tests/unit/platforms/test_ducklake_adapter.py) that\ndrives create_connection() with catalog=postgres and a stubbed setup_conn\nwhose .execute('ATTACH ...') raises an IOException carrying a sentinel\npassword (e.g. 'PW_SENTINEL_do_not_leak'); assert the sentinel is absent from\nstr(raised) and from any log record. Mutation-check: reverting w1 must make\nthe test fail. This is hermetic (no live PG) so it runs in the fast lane.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Regression test: assert no password/secret ever appears in a DuckLake postgres ATTACH-failure message", "wid": "w2"}, {"evidence": null, "item_id": "ducklake-postgres-credential-leak-redaction", "needs": ["w1"], "notes": "grep ducklake.py (and any log.* / raise with {e}/{self.metadata_path}) for\npaths that could embed pg_password or the connstring: verbose logging\n(654-658 logs host/port/db only -- confirm), the _reset/handle paths, and the\nouter create_connection wrap. Extract a small `_redact_pg_connstring()` helper\nif reused in more than one place. Confirm s3_secret has no un-redacted egress.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Sweep every other secret egress (pg_password, s3_secret) for the same raw-interpolation pattern", "wid": "w3"}], "worktree": "ducklake-review-followups"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:33:42Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #97 of ducklake-platform-adapter.\nDeferred: DuckLake-specific benchmarks (time travel, schema evolution, snapshot/maintenance workloads)\nReason deferred: Adapter parity first; lakehouse-feature benchmarks are a separate research item", "id": "ducklake-specific-benchmarks-time-travel-schema-evolution-sn", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "DuckLake-specific benchmarks (time travel, schema evolution, snapshot/maintenance workloads)", "verifications": [], "work": [], "worktree": "ducklake-platform"} -{"anti_patterns": [{"INSTEAD": "centralize internals only.", "dont": "change public benchmark facade signatures during refactor", "why": "downstream modules and tests rely on them"}, {"INSTEAD": "require demonstrated behavioral equivalence first.", "dont": "consolidate structurally similar but domain-specific SQL/generator logic", "why": "AST shape can be misleading"}, {"INSTEAD": "use narrow function/path-level ignores tied to classified groups.", "dont": "ignore whole directories in duplicate config", "why": "this hides future regressions"}, {"INSTEAD": "prefer focused mixins/helpers.", "dont": "introduce deep inheritance chains for consolidation", "why": "adapter hierarchy is already complex"}], "approach": "Start with evidence, not edits: build a full clone-group matrix before changing code.\nFor each group classified as parallel_implementation, identify the narrowest shared abstraction\npoint (utility function, mixin, or core helper module) and migrate incrementally.\nKeep each consolidation track independently testable and committable.\nApply ignore rules only after explicit group-level classification and only with narrow scope.\nRe-run duplicate scan after each consolidation track to confirm the expected impact.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "duplicate-code-followup-triage-centralization", "id": 69, "reason": "Some adapter duplicates may look parallel but depend on platform-specific cursor/session behavior best validated in live integrations.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague: no concrete list of which adapter duplicates need live-platform validation; not actionable as stated, not tracked elsewhere.", "summary": "Consolidation requiring live cloud platform behavior validation"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "duplicate-code-followup-triage-centralization", "id": 70, "reason": "High risk of over-abstraction; keep as intentional or defer unless strong equivalence evidence exists.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly recommends keeping SQL bodies as intentional absent strong equivalence evidence; not an actionable task.", "summary": "Refactoring large benchmark SQL/query bodies that are structurally similar but domain-different"}], "deps": ["consolidate-duplicate-code"], "description": "The latest `make duplicate-check-verbose` run reports 401 duplicate function groups and\n13,796 duplicated lines (threshold 15,180). The first consolidation pass removed high-value\nduplicates, but the report still contains many mixed groups where some are intentional\nboilerplate while others are true parallel implementations.\n\nThis follow-up TODO performs a full classification pass for all currently identified clone\ngroups, then applies the correct action for each class:\n- Intended duplication: keep implementation but mark with narrow ignore rules.\n- Parallel implementation: centralize shared logic into core modules/mixins/helpers and update call sites.\n\nThe key deliverable is a complete, auditable triage map of every reported clone group plus\nimplementation work that lowers duplicated lines further and reduces noise in future reports.", "id": "duplicate-code-followup-triage-centralization", "preserves": ["DataFrame adapter correctness for pandas-family and expression-family query execution paths.", "Duplicate checker remains useful (no broad ignores that hide real regressions).", "Firebolt/Presto/Trino execution semantics and error handling behavior in platform adapter tests.", "Spark-family load paths continue to support current file formats and loading options.", "Top-level benchmark facade APIs (`get_query`, `get_queries`, `get_create_tables_sql`) remain backwards compatible."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/cli/main.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/protocols.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/core-functionality/planning/duplicate-code-followup-triage-centralization.yaml"}, {"kind": "only_modify", "path_glob": "_project/quality/duplicate-code-followup/classification-matrix.yaml"}, {"kind": "only_modify", "path_glob": "_project/quality/duplicate-code-followup/latest-duplicates.json"}, {"kind": "only_modify", "path_glob": "_project/quality/duplicate-code-followup/latest-duplicates.txt"}, {"kind": "only_modify", "path_glob": "benchbox/amplab.py"}, {"kind": "only_modify", "path_glob": "benchbox/clickbench.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_mixins.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/query.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_analytics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_etl.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/azure_synapse.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/clickhouse.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/h2odb.py"}, {"kind": "only_modify", "path_glob": "benchbox/metadata_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/cudf_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/datafusion_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/expression_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/modin_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/polars_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/trino.py"}, {"kind": "only_modify", "path_glob": "benchbox/read_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/ssb.py"}, {"kind": "only_modify", "path_glob": "benchbox/tpcdi.py"}, {"kind": "only_modify", "path_glob": "benchbox/transaction_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/write_primitives.py"}, {"kind": "only_modify", "path_glob": "docs/development/adding-new-platforms.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/check_duplicate_code.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/amplab/test_amplab_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/clickbench/test_clickbench_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/test_query.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/h2odb/test_h2odb_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/read_primitives/test_catalog_loader.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/read_primitives/test_query_manager_variants.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/ssb/test_ssb_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_config_schema_queries_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_tpcdi_sql_dataframe_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/transaction_primitives/test_generator_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_cudf_df_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_dask_df_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_datafusion_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_expression_family.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_modin_df_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_pandas_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_pandas_family.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_polars_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_firebolt_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_lakesail_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_presto_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_spark_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_trino_adapter.py"}], "state": "done", "title": "Follow-up duplicate-code triage and consolidation from AST clone report", "verifications": [{"command": "uv run -- python -c \"import json,yaml,pathlib; r=pathlib.Path('_project/quality/duplicate-code-followup/latest-duplicates.json'); m=pathlib.Path('_project/quality/duplicate-code-followup/classification-matrix.yaml'); report=json.loads(r.read_text()); matrix=yaml.safe_load(m.read_text()); rows=matrix['groups'] if isinstance(matrix,dict) and 'groups' in matrix else matrix; report_ids={g['hash'] for g in report['groups']}; matrix_ids={row['group_hash'] for row in rows}; missing=report_ids-matrix_ids; extra=matrix_ids-report_ids; assert not missing and not extra, f'missing={len(missing)} extra={len(extra)}'; print(f'coverage_ok groups={len(report_ids)}')\"", "description": "All clone groups in current report have explicit classifications and actions", "expected": "coverage_ok groups=, with zero missing/extra group hashes.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose", "description": "Core duplicate checker passes at new threshold", "expected": "PASS with duplicated lines below updated threshold.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q -n 0", "description": "Fast regression suite remains green", "expected": "all fast tests pass, 0 failures.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/ tests/unit/core/dataframe/ -q -n 0", "description": "Targeted platform and dataframe tests remain green after consolidation", "expected": "all targeted tests pass, 0 failures.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": [], "notes": "Run `make duplicate-check-verbose` plus JSON mode from `scripts/check_duplicate_code.py`.\nPersist artifacts under `_project/quality/duplicate-code-followup/` with timestamped files and\nrefresh canonical pointers:\n- `_project/quality/duplicate-code-followup/latest-duplicates.txt`\n- `_project/quality/duplicate-code-followup/latest-duplicates.json`\nInclude top groups and full-group inventory for review traceability.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline duplicate report artifacts in machine-readable and reviewable formats", "wid": "w1"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Evaluate and centralize genuinely parallel adapter methods (for example cloud adapter\nplatform info blocks, cache/session validators, shared execute/load flows) while preserving\nplatform-specific error semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate platform adapter parallel implementations into scoped mixins", "wid": "w10"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Address repeated query metadata accessors/filtering helpers and generator scaffolding\nonly where behavioral equivalence is demonstrated.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate TPC-DI and generator helper duplicates that are true parallel scaffolding", "wid": "w11"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w10", "w11", "w6", "w7", "w8", "w9"], "notes": "Run targeted unit suites for every touched module family and resolve regressions before final verification.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run targeted regression tests per consolidation/ignore track and fix breakages", "wid": "w12"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w12"], "notes": "Verify behavior parity with targeted tests for each consolidation track plus fast suite.\nUpdate threshold in pyproject to observed duplicated-line count plus 10% headroom.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run regression suite, re-run duplicate check, and lower threshold to new baseline + safety margin", "wid": "w13"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w13"], "notes": "Record total line reduction, finalized threshold, and list of consolidated groups.\nMove item through active -> done workflow only after all verification passes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Finalize TODO completion state, capture before/after metrics, and regenerate indexes", "wid": "w14"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w10"], "notes": "Add or update project documentation with a clear taxonomy of SQL platform `execute_query`\nvariants discovered during w10 analysis, including:\n- DBAPI cursor + validation-helper family\n- Spark SQL collect family\n- cloud Spark submit/wait/fetch job family\n- Livy statement API family\n- engine-specific outliers (BigQuery/DataFusion/Databend/MotherDuck/InfluxDB)\nDocument which families are safe consolidation targets now vs intentionally distinct.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document execute_query variant families and consolidation boundaries", "wid": "w15"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w1"], "notes": "Create a checklist with objective decision criteria and examples.\nAdd explicit disqualifiers for false consolidations (protocol methods, interface contracts,\nand facade delegation wrappers that are intentionally repetitive).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define strict classification rubric for intended boilerplate vs parallel implementation", "wid": "w2"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w2"], "notes": "Classify top 30 groups first, then refine rubric language where disagreements/ambiguity appear.\nThis phase locks classification examples before full-inventory triage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pilot classification on top duplicate groups and calibrate rubric", "wid": "w3"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w3"], "notes": "Evaluate every currently identified group from the full inventory (beyond top-N).\nOutput one row per group with class, rationale, action, confidence, and owner module.\nWrite matrix to `_project/quality/duplicate-code-followup/classification-matrix.yaml`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify remaining duplicate groups and produce complete action matrix", "wid": "w4"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w4"], "notes": "Verify one-to-one coverage between latest duplicate report groups and matrix rows.\nNo implementation starts until coverage and action mapping are complete and internally consistent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Validate matrix completeness and freeze implementation/ignore backlog", "wid": "w5"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Update duplicate checker config/script with tightly scoped ignore rules\n(function names plus path constraints where needed). Avoid broad suppressions and broad path wildcards.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ignore rules batch A for intended adapter/protocol boilerplate groups", "wid": "w6"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Apply ignores for verified intentional duplication in thin facades and domain-specific structural clones.\nKeep ignore entries traceable to matrix rows and rationale.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ignore rules batch B for intended facade/structural non-parallel groups", "wid": "w7"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Target groups such as repeated benchmark wrapper helpers (`get_queries`, `get_query`,\n`get_create_tables_sql`) and repeated query metadata/filter helper patterns where behavior matches.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate benchmark facade/query-helper parallel implementations into shared core utilities", "wid": "w8"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Focus on true parallel methods (`merge`, `groupby_agg`, `col/lit/from_pandas`, scalar conversion helpers)\nwhere semantics are aligned across adapters.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate DataFrame family parallel implementations via pandas/expression shared helpers", "wid": "w9"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change public benchmark facade signatures during refactor", "instead": "centralize internals only.", "why": "downstream modules and tests rely on them"}, {"dont": "consolidate structurally similar but domain-specific SQL/generator logic", "instead": "require demonstrated behavioral equivalence first.", "why": "AST shape can be misleading"}, {"dont": "ignore whole directories in duplicate config", "instead": "use narrow function/path-level ignores tied to classified groups.", "why": "this hides future regressions"}, {"dont": "introduce deep inheritance chains for consolidation", "instead": "prefer focused mixins/helpers.", "why": "adapter hierarchy is already complex"}], "approach": "Start with evidence, not edits: build a full clone-group matrix before changing code.\nFor each group classified as parallel_implementation, identify the narrowest shared abstraction\npoint (utility function, mixin, or core helper module) and migrate incrementally.\nKeep each consolidation track independently testable and committable.\nApply ignore rules only after explicit group-level classification and only with narrow scope.\nRe-run duplicate scan after each consolidation track to confirm the expected impact.\n", "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-20T00:00:00Z", "completed_pr": null, "created_at": "2026-02-20T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "duplicate-code-followup-triage-centralization", "id": 69, "reason": "Some adapter duplicates may look parallel but depend on platform-specific cursor/session behavior best validated in live integrations.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague: no concrete list of which adapter duplicates need live-platform validation; not actionable as stated, not tracked elsewhere.", "summary": "Consolidation requiring live cloud platform behavior validation"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "duplicate-code-followup-triage-centralization", "id": 70, "reason": "High risk of over-abstraction; keep as intentional or defer unless strong equivalence evidence exists.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Reason explicitly recommends keeping SQL bodies as intentional absent strong equivalence evidence; not an actionable task.", "summary": "Refactoring large benchmark SQL/query bodies that are structurally similar but domain-different"}], "deps": ["consolidate-duplicate-code"], "description": "The latest `make duplicate-check-verbose` run reports 401 duplicate function groups and\n13,796 duplicated lines (threshold 15,180). The first consolidation pass removed high-value\nduplicates, but the report still contains many mixed groups where some are intentional\nboilerplate while others are true parallel implementations.\n\nThis follow-up TODO performs a full classification pass for all currently identified clone\ngroups, then applies the correct action for each class:\n- Intended duplication: keep implementation but mark with narrow ignore rules.\n- Parallel implementation: centralize shared logic into core modules/mixins/helpers and update call sites.\n\nThe key deliverable is a complete, auditable triage map of every reported clone group plus\nimplementation work that lowers duplicated lines further and reduces noise in future reports.", "id": "duplicate-code-followup-triage-centralization", "preserves": ["DataFrame adapter correctness for pandas-family and expression-family query execution paths.", "Duplicate checker remains useful (no broad ignores that hide real regressions).", "Firebolt/Presto/Trino execution semantics and error handling behavior in platform adapter tests.", "Spark-family load paths continue to support current file formats and loading options.", "Top-level benchmark facade APIs (`get_query`, `get_queries`, `get_create_tables_sql`) remain backwards compatible."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/run.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/cli/main.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/context.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/dataframe/protocols.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/core-functionality/planning/duplicate-code-followup-triage-centralization.yaml"}, {"kind": "only_modify", "path_glob": "_project/quality/duplicate-code-followup/classification-matrix.yaml"}, {"kind": "only_modify", "path_glob": "_project/quality/duplicate-code-followup/latest-duplicates.json"}, {"kind": "only_modify", "path_glob": "_project/quality/duplicate-code-followup/latest-duplicates.txt"}, {"kind": "only_modify", "path_glob": "benchbox/amplab.py"}, {"kind": "only_modify", "path_glob": "benchbox/clickbench.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/benchmark_mixins.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/query.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/query_manager.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/queries.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_analytics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_etl.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcdi/query_validation.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/azure_synapse.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/clickhouse.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tuning/generators/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/h2odb.py"}, {"kind": "only_modify", "path_glob": "benchbox/metadata_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/cudf_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/dask_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/datafusion_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/expression_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/modin_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/pandas_family.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/polars_df.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/firebolt.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/lakesail.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/presto.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/spark.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/trino.py"}, {"kind": "only_modify", "path_glob": "benchbox/read_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/ssb.py"}, {"kind": "only_modify", "path_glob": "benchbox/tpcdi.py"}, {"kind": "only_modify", "path_glob": "benchbox/transaction_primitives.py"}, {"kind": "only_modify", "path_glob": "benchbox/write_primitives.py"}, {"kind": "only_modify", "path_glob": "docs/development/adding-new-platforms.md"}, {"kind": "only_modify", "path_glob": "pyproject.toml"}, {"kind": "only_modify", "path_glob": "scripts/check_duplicate_code.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/amplab/test_amplab_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/clickbench/test_clickbench_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/dataframe/test_query.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/h2odb/test_h2odb_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/read_primitives/test_catalog_loader.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/read_primitives/test_query_manager_variants.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/ssb/test_ssb_dataframe_queries.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_config_schema_queries_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpcdi/test_tpcdi_sql_dataframe_parity.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/transaction_primitives/test_generator_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_cudf_df_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_dask_df_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_datafusion_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_expression_family.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_modin_df_coverage.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_pandas_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_pandas_family.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/test_polars_df.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_firebolt_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_lakesail_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_presto_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_spark_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_trino_adapter.py"}], "state": "done", "title": "Follow-up duplicate-code triage and consolidation from AST clone report", "verifications": [{"command": "uv run -- python -c \"import json,yaml,pathlib; r=pathlib.Path('_project/quality/duplicate-code-followup/latest-duplicates.json'); m=pathlib.Path('_project/quality/duplicate-code-followup/classification-matrix.yaml'); report=json.loads(r.read_text()); matrix=yaml.safe_load(m.read_text()); rows=matrix['groups'] if isinstance(matrix,dict) and 'groups' in matrix else matrix; report_ids={g['hash'] for g in report['groups']}; matrix_ids={row['group_hash'] for row in rows}; missing=report_ids-matrix_ids; extra=matrix_ids-report_ids; assert not missing and not extra, f'missing={len(missing)} extra={len(extra)}'; print(f'coverage_ok groups={len(report_ids)}')\"", "description": "All clone groups in current report have explicit classifications and actions", "expected": "coverage_ok groups=, with zero missing/extra group hashes.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 1}, {"command": "make duplicate-check-verbose", "description": "Core duplicate checker passes at new threshold", "expected": "PASS with duplicated lines below updated threshold.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q -n 0", "description": "Fast regression suite remains green", "expected": "all fast tests pass, 0 failures.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/ tests/unit/core/dataframe/ -q -n 0", "description": "Targeted platform and dataframe tests remain green after consolidation", "expected": "all targeted tests pass, 0 failures.", "item_id": "duplicate-code-followup-triage-centralization", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": [], "notes": "Run `make duplicate-check-verbose` plus JSON mode from `scripts/check_duplicate_code.py`.\nPersist artifacts under `_project/quality/duplicate-code-followup/` with timestamped files and\nrefresh canonical pointers:\n- `_project/quality/duplicate-code-followup/latest-duplicates.txt`\n- `_project/quality/duplicate-code-followup/latest-duplicates.json`\nInclude top groups and full-group inventory for review traceability.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Capture baseline duplicate report artifacts in machine-readable and reviewable formats", "wid": "w1"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Evaluate and centralize genuinely parallel adapter methods (for example cloud adapter\nplatform info blocks, cache/session validators, shared execute/load flows) while preserving\nplatform-specific error semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate platform adapter parallel implementations into scoped mixins", "wid": "w10"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Address repeated query metadata accessors/filtering helpers and generator scaffolding\nonly where behavioral equivalence is demonstrated.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate TPC-DI and generator helper duplicates that are true parallel scaffolding", "wid": "w11"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w10", "w11", "w6", "w7", "w8", "w9"], "notes": "Run targeted unit suites for every touched module family and resolve regressions before final verification.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run targeted regression tests per consolidation/ignore track and fix breakages", "wid": "w12"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w12"], "notes": "Verify behavior parity with targeted tests for each consolidation track plus fast suite.\nUpdate threshold in pyproject to observed duplicated-line count plus 10% headroom.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run regression suite, re-run duplicate check, and lower threshold to new baseline + safety margin", "wid": "w13"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w13"], "notes": "Record total line reduction, finalized threshold, and list of consolidated groups.\nMove item through active -> done workflow only after all verification passes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Finalize TODO completion state, capture before/after metrics, and regenerate indexes", "wid": "w14"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w10"], "notes": "Add or update project documentation with a clear taxonomy of SQL platform `execute_query`\nvariants discovered during w10 analysis, including:\n- DBAPI cursor + validation-helper family\n- Spark SQL collect family\n- cloud Spark submit/wait/fetch job family\n- Livy statement API family\n- engine-specific outliers (BigQuery/DataFusion/Databend/MotherDuck/InfluxDB)\nDocument which families are safe consolidation targets now vs intentionally distinct.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document execute_query variant families and consolidation boundaries", "wid": "w15"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w1"], "notes": "Create a checklist with objective decision criteria and examples.\nAdd explicit disqualifiers for false consolidations (protocol methods, interface contracts,\nand facade delegation wrappers that are intentionally repetitive).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define strict classification rubric for intended boilerplate vs parallel implementation", "wid": "w2"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w2"], "notes": "Classify top 30 groups first, then refine rubric language where disagreements/ambiguity appear.\nThis phase locks classification examples before full-inventory triage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pilot classification on top duplicate groups and calibrate rubric", "wid": "w3"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w3"], "notes": "Evaluate every currently identified group from the full inventory (beyond top-N).\nOutput one row per group with class, rationale, action, confidence, and owner module.\nWrite matrix to `_project/quality/duplicate-code-followup/classification-matrix.yaml`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Classify remaining duplicate groups and produce complete action matrix", "wid": "w4"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w4"], "notes": "Verify one-to-one coverage between latest duplicate report groups and matrix rows.\nNo implementation starts until coverage and action mapping are complete and internally consistent.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Validate matrix completeness and freeze implementation/ignore backlog", "wid": "w5"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Update duplicate checker config/script with tightly scoped ignore rules\n(function names plus path constraints where needed). Avoid broad suppressions and broad path wildcards.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ignore rules batch A for intended adapter/protocol boilerplate groups", "wid": "w6"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Apply ignores for verified intentional duplication in thin facades and domain-specific structural clones.\nKeep ignore entries traceable to matrix rows and rationale.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement ignore rules batch B for intended facade/structural non-parallel groups", "wid": "w7"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Target groups such as repeated benchmark wrapper helpers (`get_queries`, `get_query`,\n`get_create_tables_sql`) and repeated query metadata/filter helper patterns where behavior matches.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate benchmark facade/query-helper parallel implementations into shared core utilities", "wid": "w8"}, {"evidence": null, "item_id": "duplicate-code-followup-triage-centralization", "needs": ["w5"], "notes": "Focus on true parallel methods (`merge`, `groupby_agg`, `col/lit/from_pandas`, scalar conversion helpers)\nwhere semantics are aligned across adapters.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consolidate DataFrame family parallel implementations via pandas/expression shared helpers", "wid": "w9"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-16T00:00:00Z", "completed_pr": null, "created_at": "2026-01-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Create a comprehensive end-to-end CLI test suite for benchbox that executes\nthe CLI completely and evaluates result output files to confirm correct execution.\n\nTests are grouped by platform types:\n- Local platforms (DuckDB, SQLite, DataFusion, PostgreSQL)\n- Cloud platforms (Snowflake, BigQuery, Redshift, Athena, Databricks)\n- DataFrame platforms (Pandas, Polars, Dask, PySpark, cuDF)\n\nThe goal is to test all primary code paths users would touch to ensure product\nfunctionality before shipping. This includes:\n- Full benchmark execution with result validation\n- Output file structure verification\n- Phase execution completeness\n- Error handling and edge cases\n- CLI options and flags coverage", "id": "e2e-cli-test-suite", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "E2E CLI Test Suite: Complete Platform Coverage", "verifications": [], "work": [], "worktree": "e2e-cli-tests"} -{"anti_patterns": [{"INSTEAD": "remove only live-path dependence on them", "dont": "delete historical result compatibility fallbacks in schema/loader code", "why": "those are valid artifact-boundary shims"}, {"INSTEAD": "choose one owner per path", "dont": "let config builders and adapters both own env/default resolution for the same connection fields", "why": "that guarantees drift and split behavior across callers"}, {"INSTEAD": "make callers pass benchmark identity explicitly", "dont": "replace multiple sniffers with one centralized infer-benchmark helper", "why": "that preserves the wrong ownership boundary"}, {"INSTEAD": "propagate the canonical id instead", "dont": "use display names, wrapper class names, or benchmark._name as the authoritative routing input once the caller already has benchmark_config.name", "why": "those representations have different semantics"}], "approach": "Fix the missing context at the caller boundary first, then remove the\ndownstream compensation that existed only because the caller failed to pass\nthat context.\n\nImplement in thin vertical slices:\n1. Benchmark identity contract and SQL runner propagation\n2. Base adapter routing and validation cleanup\n3. Base adapter dialect and single-query cleanup\n4. Live result creation and export cleanup\n5. Dry-run contract cleanup\n6. Doris config ownership cleanup\n7. SingleStore config ownership cleanup\n8. Regression tests and final audit\n\nTreat compatibility code as an explicit exception boundary. Live execution\npaths should become simpler; compatibility import/export paths may retain\nfallback behavior where they truly own legacy artifact adaptation.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-07T00:00:00Z", "completed_pr": null, "created_at": "2026-04-07T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "eliminate-non-data-loading-wrong-layer-compensation", "id": 311, "reason": "The review found similar-looking patterns, but not enough caller-context proof to classify them as confirmed wrong-layer compensation in the current flows", "resolution": "promoted", "resolved_item": "re-audit-databend-firebolt-starburst-and-clickhouse-cloud-fo", "resolved_reason": null, "summary": "Re-audit Databend, Firebolt, Starburst, and ClickHouse Cloud for broader config-ownership standardization after the confirmed fixes land"}], "deps": [], "description": "A targeted architectural review found 5 confirmed non-data-loading instances\nof wrong-layer compensation.\n\nConfirmed findings:\n1. SQL lifecycle omits canonical benchmark identity even though the caller\n already has it, so the base adapter infers benchmark type from benchmark\n object internals for routing, dialect selection, and row-count validation.\n2. Live result creation and export paths re-derive benchmark_id from\n benchmark names instead of receiving a caller-supplied canonical id.\n3. Dry-run query extraction and resource estimation ignore\n BenchmarkConfig.name and re-infer benchmark identity from benchmark\n object attributes.\n4. Doris resolves host/user/password/database in the config builder, then\n re-resolves env/defaults again in from_config() and __init__().\n5. SingleStore has the same duplicated connection/database ownership split.\n\nThe fix is not to centralize more inference. The fix is to make the caller\npass the context it already owns, then delete downstream compensation in the\nlive execution path. Historical artifact compatibility fallbacks and\nlegitimate owner-layer discovery are explicitly out of scope.", "id": "eliminate-non-data-loading-wrong-layer-compensation", "preserves": ["Benchmark-derived Doris/SingleStore database naming still works when no explicit database is provided", "Data-only/result-object paths that intentionally set benchmark_id explicitly continue to work", "DataFrame execution already receives benchmark_config explicitly and must not regress while SQL is brought to the same contract quality", "Explicit CLI database/schema overrides still win over generated names for Doris and SingleStore", "Historical artifact compatibility fallbacks in benchbox/core/results/schema.py and result-loading code remain intact", "The original data-loading-only wrong-layer-compensation fixes remain untouched"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/loader.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/file_sources.py"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/exporter.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/result_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/schemas.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/doris.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/singlestore.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_builder.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_loader.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_normalizer.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_runner_lifecycle.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter_database_management.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_doris_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_enhanced_benchmark_results.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_singlestore_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_cli_dryrun_real_execution.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_cli_dryrun_test_modes.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_results_exporter.py"}], "state": "done", "title": "Eliminate non-data-loading wrong-layer compensation across runner, dry-run, results, and platform config", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_runner_lifecycle.py tests/unit/platforms/test_base_adapter.py tests/unit/platforms/test_base_adapter_database_management.py -q", "description": "SQL lifecycle, benchmark identity propagation, and base-adapter behavior remain correct", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_dryrun.py tests/unit/core/test_dryrun_behavioral.py tests/unit/test_cli_dryrun_real_execution.py tests/unit/test_cli_dryrun_test_modes.py -q", "description": "Dry-run behavior remains correct while using explicit benchmark context", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/results/test_builder.py tests/unit/core/results/test_loader.py tests/unit/core/results/test_normalizer.py tests/unit/test_results_exporter.py tests/unit/platforms/test_enhanced_benchmark_results.py -q", "description": "Result-building and export paths preserve explicit benchmark_id behavior", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_doris_adapter.py tests/unit/platforms/test_singlestore_adapter.py -q", "description": "Doris and SingleStore config ownership regression tests pass", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 4}, {"command": "rg -n \"benchmark\\._name|benchmark\\.__class__\\.__name__|type\\(benchmark\\)\\.__name__|display_name.*benchmark_id|benchmark_id.*display_name\" benchbox/core/runner/runner.py benchbox/platforms/base/adapter.py benchbox/core/dryrun.py benchbox/base.py benchbox/core/results/result_factory.py benchbox/core/results/models.py benchbox/core/results/exporter.py", "description": "Post-fix audit shows no live-path benchmark identity sniffing remains in the touched execution surfaces", "expected": "no matches", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 5}, {"command": "rg -n \"os\\.getenv\\(|getenv\\(\" benchbox/platforms/doris.py benchbox/platforms/singlestore.py", "description": "Post-fix audit shows Doris and SingleStore do not duplicate getenv-based standard connection fallback in adapter construction paths", "expected": "matches, if any, are confined to the single chosen owner layer rather than duplicated across builder, from_config(), and __init__()", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": [], "notes": "Contract documented in benchbox/core/schemas.py docstrings for both\nRunConfig and BenchmarkConfig.\n\nOwnership hierarchy (CLI -> persistence):\n- BenchmarkConfig.name -> authoritative slug from CLI/orchestrator (e.g. \"tpch\")\n- RunConfig.benchmark -> propagated copy; runner must populate from BenchmarkConfig.name\n- execution_metadata[\"benchmark_id\"] -> normalised form written to results/export\n- display_name / wrapper class names -> presentation only, never routing inputs\n\nFix sequence: runner populates RunConfig.benchmark (w2), adapter uses\nrun_config[\"benchmark\"] instead of sniffing (w3/w4), results/export consume\nexplicit benchmark_id from execution_metadata (w5/w6), dry-run uses\nbenchmark_config.name directly (w7).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the canonical benchmark identity contract for live execution paths", "wid": "w1"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w3", "w4", "w5", "w6", "w7"], "notes": "Add or extend tests for:\n- SQL runner propagates benchmark identity into adapter/results\n- dry run uses BenchmarkConfig.name for TPC branching and estimation\n- exporter/result paths use explicit benchmark_id without name-derived drift\n\nPrimary test files:\n- tests/unit/core/test_runner_lifecycle.py\n- tests/unit/platforms/test_base_adapter.py\n- tests/unit/platforms/test_base_adapter_database_management.py\n- tests/unit/core/test_dryrun.py\n- tests/unit/core/test_dryrun_behavioral.py\n- tests/unit/test_cli_dryrun_real_execution.py\n- tests/unit/test_cli_dryrun_test_modes.py\n- tests/unit/core/results/test_builder.py\n- tests/unit/core/results/test_loader.py\n- tests/unit/core/results/test_normalizer.py\n- tests/unit/test_results_exporter.py\n- tests/unit/platforms/test_enhanced_benchmark_results.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for benchmark identity propagation across runner, adapter, dry-run, and results", "wid": "w10"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w8", "w9"], "notes": "Add or extend tests for:\n- Doris consumes builder-resolved config without env fallback duplication\n- SingleStore consumes builder-resolved config without env fallback duplication\n\nPrimary test files:\n- tests/unit/platforms/test_doris_adapter.py\n- tests/unit/platforms/test_singlestore_adapter.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for Doris and SingleStore config-ownership cleanup", "wid": "w11"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w10", "w11"], "notes": "643 tests pass across all touched surfaces.\n\nSurviving fallback sites (all confirmed valid):\n\nBenchmark identity in adapter/dryrun:\n- _get_benchmark_type() (adapter.py ~1717): the encapsulated fallback method;\n only called when no explicit slug is available from run_config. Live path\n never reaches it: run_config[\"benchmark_name\"] is preferred in all\n _execute_* routing methods.\n- \"or getattr(benchmark, '_name', ...)\" patterns in _get_dialect_queries,\n _execute_all_queries, _execute_single_query, _estimate_resources,\n _estimate_memory_usage: all are documented as fallbacks for callers that\n pre-date explicit slug propagation. Live path from runner always provides\n the slug via run_config[\"benchmark_name\"].\n\nDoris/SingleStore env vars:\n- _build_doris_config() and _build_singlestore_config(): single canonical\n resolvers for the CLI path. All env vars are HERE.\n- from_config() in both adapters: only DORIS_DATABASE / SINGLESTORE_DATABASE\n env var remains, for benchmark-derived database naming. This is the\n legitimate owner layer (from_config has benchmark+scale_factor context).\n- __init__() in both adapters: no env vars, Python defaults only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run focused verification and post-fix audits to ensure only legitimate compatibility boundaries still use fallbacks", "wid": "w12"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "Update the SQL lifecycle so RunConfig carries benchmark=benchmark_config.name\nand the SQL path does not drop that field before calling the adapter.\n\nPrimary files:\n- benchbox/core/runner/runner.py\n- benchbox/core/schemas.py\n\nCaller contract to enforce:\n- SQL path matches the DataFrame path in explicit benchmark context quality\n- platform_config/get_platform_config remains the owner of platform config,\n not benchmark identity backfilling inside adapters\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Thread benchmark identity through the SQL runner instead of stripping it before adapter.run_benchmark()", "wid": "w2"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w2"], "notes": "Replace active-path uses of benchmark._name / display_name / class-name\ninference with explicit benchmark identity from run_config or a directly\npassed benchmark_id argument for execution routing and validation.\n\nPrimary files:\n- benchbox/platforms/base/adapter.py\n\nSpecific call sites to clean up:\n- _get_benchmark_type()\n- _execute_power_test()\n- _execute_throughput_test()\n- _execute_maintenance_test()\n- _execute_combined_test()\n\nPreserve benchmark object access for legitimate benchmark behavior\nexecution; only remove identity inference used to compensate for omitted\ncaller context.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove base-adapter benchmark sniffing from live routing and validation", "wid": "w3"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w2"], "notes": "Clean up the remaining benchmark-identity inference in query selection and\nsingle-query execution paths.\n\nPrimary file:\n- benchbox/platforms/base/adapter.py\n\nSpecific call sites to clean up:\n- _get_dialect_queries()\n- _execute_single_query()\n\nKeep dialect and query execution behavior intact; only remove the need to\nsniff benchmark identity from wrapper internals.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove base-adapter benchmark sniffing from dialect selection and single-query execution", "wid": "w4"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w2"], "notes": "Ensure live-created BenchmarkResults receive benchmark_id from\nexecution_metadata produced by the runner/adapter path.\n\nPrimary files:\n- benchbox/base.py\n- benchbox/core/results/result_factory.py\n- benchbox/core/results/models.py\n\nRequired boundary decision:\n- Live execution paths must not rely on benchmark-name normalization\n- Historical artifact compatibility in benchbox/core/results/schema.py and\n loader utilities remains intact and must stay a compatibility boundary\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make live result creation consume explicit benchmark_id instead of deriving it from names", "wid": "w5"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w5"], "notes": "Ensure export code consumes the explicit benchmark_id already present on\nlive-created results instead of re-normalizing from names.\n\nPrimary file:\n- benchbox/core/results/exporter.py\n\nPreserve compatibility behavior for historical artifacts that enter through\nloader/schema boundaries rather than the live execution path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make result export preserve explicit benchmark_id without name-derived drift", "wid": "w6"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "DryRunExecutor already receives BenchmarkConfig and already uses it for\nplatform-config injection. Extend that same explicit contract to query\nextraction, real-test dry-run execution, data-size estimation, and memory\nestimation.\n\nPrimary file:\n- benchbox/core/dryrun.py\n\nTarget behaviors:\n- TPC test-mode branching keys off benchmark_config.name / canonical id\n- dry-run estimates do not depend on wrapper class names or _name sniffing\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor dry-run query extraction and estimation to use BenchmarkConfig.name instead of benchmark-object sniffing", "wid": "w7"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "Keep benchmark-aware database naming where benchmark/scale context first\nexists, but stop re-resolving host/port/http_port/username/password/database\ninside from_config() and __init__() for the standard config-built path.\n\nPrimary files:\n- benchbox/platforms/doris.py\n- benchbox/cli/database.py\n- benchbox/cli/platform_hooks.py\n- benchbox/core/platform_config.py\n\nEnd state:\n- builder returns resolved DatabaseConfig\n- from_config() maps/resolves only benchmark-derived database naming that\n truly belongs there\n- __init__() consumes resolved values instead of repeating env/default logic\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Doris config builder the single owner of standard CLI connection/env/default resolution", "wid": "w8"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "Apply the same ownership cleanup as Doris to SingleStore.\n\nPrimary files:\n- benchbox/platforms/singlestore.py\n- benchbox/cli/database.py\n- benchbox/cli/platform_hooks.py\n- benchbox/core/platform_config.py\n\nEnd state:\n- no adapter-side env/database fallback in the standard config-aware path\n- explicit database overrides still win\n- generated benchmark-scoped database naming still works when no explicit\n database is provided\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make SingleStore config builder the single owner of standard CLI connection/env/default resolution", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "delete historical result compatibility fallbacks in schema/loader code", "instead": "remove only live-path dependence on them", "why": "those are valid artifact-boundary shims"}, {"dont": "let config builders and adapters both own env/default resolution for the same connection fields", "instead": "choose one owner per path", "why": "that guarantees drift and split behavior across callers"}, {"dont": "replace multiple sniffers with one centralized infer-benchmark helper", "instead": "make callers pass benchmark identity explicitly", "why": "that preserves the wrong ownership boundary"}, {"dont": "use display names, wrapper class names, or benchmark._name as the authoritative routing input once the caller already has benchmark_config.name", "instead": "propagate the canonical id instead", "why": "those representations have different semantics"}], "approach": "Fix the missing context at the caller boundary first, then remove the\ndownstream compensation that existed only because the caller failed to pass\nthat context.\n\nImplement in thin vertical slices:\n1. Benchmark identity contract and SQL runner propagation\n2. Base adapter routing and validation cleanup\n3. Base adapter dialect and single-query cleanup\n4. Live result creation and export cleanup\n5. Dry-run contract cleanup\n6. Doris config ownership cleanup\n7. SingleStore config ownership cleanup\n8. Regression tests and final audit\n\nTreat compatibility code as an explicit exception boundary. Live execution\npaths should become simpler; compatibility import/export paths may retain\nfallback behavior where they truly own legacy artifact adaptation.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-07T00:00:00Z", "completed_pr": null, "created_at": "2026-04-07T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "eliminate-non-data-loading-wrong-layer-compensation", "id": 311, "reason": "The review found similar-looking patterns, but not enough caller-context proof to classify them as confirmed wrong-layer compensation in the current flows", "resolution": "promoted", "resolved_item": "re-audit-databend-firebolt-starburst-and-clickhouse-cloud-fo", "resolved_reason": null, "summary": "Re-audit Databend, Firebolt, Starburst, and ClickHouse Cloud for broader config-ownership standardization after the confirmed fixes land"}], "deps": [], "description": "A targeted architectural review found 5 confirmed non-data-loading instances\nof wrong-layer compensation.\n\nConfirmed findings:\n1. SQL lifecycle omits canonical benchmark identity even though the caller\n already has it, so the base adapter infers benchmark type from benchmark\n object internals for routing, dialect selection, and row-count validation.\n2. Live result creation and export paths re-derive benchmark_id from\n benchmark names instead of receiving a caller-supplied canonical id.\n3. Dry-run query extraction and resource estimation ignore\n BenchmarkConfig.name and re-infer benchmark identity from benchmark\n object attributes.\n4. Doris resolves host/user/password/database in the config builder, then\n re-resolves env/defaults again in from_config() and __init__().\n5. SingleStore has the same duplicated connection/database ownership split.\n\nThe fix is not to centralize more inference. The fix is to make the caller\npass the context it already owns, then delete downstream compensation in the\nlive execution path. Historical artifact compatibility fallbacks and\nlegitimate owner-layer discovery are explicitly out of scope.", "id": "eliminate-non-data-loading-wrong-layer-compensation", "preserves": ["Benchmark-derived Doris/SingleStore database naming still works when no explicit database is provided", "Data-only/result-object paths that intentionally set benchmark_id explicitly continue to work", "DataFrame execution already receives benchmark_config explicitly and must not regress while SQL is brought to the same contract quality", "Explicit CLI database/schema overrides still win over generated names for Doris and SingleStore", "Historical artifact compatibility fallbacks in benchbox/core/results/schema.py and result-loading code remain intact", "The original data-loading-only wrong-layer-compensation fixes remain untouched"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/loader.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/data_loading.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/file_sources.py"}, {"kind": "only_modify", "path_glob": "benchbox/base.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/exporter.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/result_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/schemas.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/doris.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/singlestore.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_builder.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_loader.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_normalizer.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_runner_lifecycle.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter_database_management.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_doris_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_enhanced_benchmark_results.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_singlestore_adapter.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_cli_dryrun_real_execution.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_cli_dryrun_test_modes.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_results_exporter.py"}], "state": "done", "title": "Eliminate non-data-loading wrong-layer compensation across runner, dry-run, results, and platform config", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/test_runner_lifecycle.py tests/unit/platforms/test_base_adapter.py tests/unit/platforms/test_base_adapter_database_management.py -q", "description": "SQL lifecycle, benchmark identity propagation, and base-adapter behavior remain correct", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_dryrun.py tests/unit/core/test_dryrun_behavioral.py tests/unit/test_cli_dryrun_real_execution.py tests/unit/test_cli_dryrun_test_modes.py -q", "description": "Dry-run behavior remains correct while using explicit benchmark context", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/results/test_builder.py tests/unit/core/results/test_loader.py tests/unit/core/results/test_normalizer.py tests/unit/test_results_exporter.py tests/unit/platforms/test_enhanced_benchmark_results.py -q", "description": "Result-building and export paths preserve explicit benchmark_id behavior", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/platforms/test_doris_adapter.py tests/unit/platforms/test_singlestore_adapter.py -q", "description": "Doris and SingleStore config ownership regression tests pass", "expected": "all selected tests pass", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 4}, {"command": "rg -n \"benchmark\\._name|benchmark\\.__class__\\.__name__|type\\(benchmark\\)\\.__name__|display_name.*benchmark_id|benchmark_id.*display_name\" benchbox/core/runner/runner.py benchbox/platforms/base/adapter.py benchbox/core/dryrun.py benchbox/base.py benchbox/core/results/result_factory.py benchbox/core/results/models.py benchbox/core/results/exporter.py", "description": "Post-fix audit shows no live-path benchmark identity sniffing remains in the touched execution surfaces", "expected": "no matches", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 5}, {"command": "rg -n \"os\\.getenv\\(|getenv\\(\" benchbox/platforms/doris.py benchbox/platforms/singlestore.py", "description": "Post-fix audit shows Doris and SingleStore do not duplicate getenv-based standard connection fallback in adapter construction paths", "expected": "matches, if any, are confined to the single chosen owner layer rather than duplicated across builder, from_config(), and __init__()", "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": [], "notes": "Contract documented in benchbox/core/schemas.py docstrings for both\nRunConfig and BenchmarkConfig.\n\nOwnership hierarchy (CLI -> persistence):\n- BenchmarkConfig.name -> authoritative slug from CLI/orchestrator (e.g. \"tpch\")\n- RunConfig.benchmark -> propagated copy; runner must populate from BenchmarkConfig.name\n- execution_metadata[\"benchmark_id\"] -> normalised form written to results/export\n- display_name / wrapper class names -> presentation only, never routing inputs\n\nFix sequence: runner populates RunConfig.benchmark (w2), adapter uses\nrun_config[\"benchmark\"] instead of sniffing (w3/w4), results/export consume\nexplicit benchmark_id from execution_metadata (w5/w6), dry-run uses\nbenchmark_config.name directly (w7).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the canonical benchmark identity contract for live execution paths", "wid": "w1"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w3", "w4", "w5", "w6", "w7"], "notes": "Add or extend tests for:\n- SQL runner propagates benchmark identity into adapter/results\n- dry run uses BenchmarkConfig.name for TPC branching and estimation\n- exporter/result paths use explicit benchmark_id without name-derived drift\n\nPrimary test files:\n- tests/unit/core/test_runner_lifecycle.py\n- tests/unit/platforms/test_base_adapter.py\n- tests/unit/platforms/test_base_adapter_database_management.py\n- tests/unit/core/test_dryrun.py\n- tests/unit/core/test_dryrun_behavioral.py\n- tests/unit/test_cli_dryrun_real_execution.py\n- tests/unit/test_cli_dryrun_test_modes.py\n- tests/unit/core/results/test_builder.py\n- tests/unit/core/results/test_loader.py\n- tests/unit/core/results/test_normalizer.py\n- tests/unit/test_results_exporter.py\n- tests/unit/platforms/test_enhanced_benchmark_results.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for benchmark identity propagation across runner, adapter, dry-run, and results", "wid": "w10"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w8", "w9"], "notes": "Add or extend tests for:\n- Doris consumes builder-resolved config without env fallback duplication\n- SingleStore consumes builder-resolved config without env fallback duplication\n\nPrimary test files:\n- tests/unit/platforms/test_doris_adapter.py\n- tests/unit/platforms/test_singlestore_adapter.py\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add regression tests for Doris and SingleStore config-ownership cleanup", "wid": "w11"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w10", "w11"], "notes": "643 tests pass across all touched surfaces.\n\nSurviving fallback sites (all confirmed valid):\n\nBenchmark identity in adapter/dryrun:\n- _get_benchmark_type() (adapter.py ~1717): the encapsulated fallback method;\n only called when no explicit slug is available from run_config. Live path\n never reaches it: run_config[\"benchmark_name\"] is preferred in all\n _execute_* routing methods.\n- \"or getattr(benchmark, '_name', ...)\" patterns in _get_dialect_queries,\n _execute_all_queries, _execute_single_query, _estimate_resources,\n _estimate_memory_usage: all are documented as fallbacks for callers that\n pre-date explicit slug propagation. Live path from runner always provides\n the slug via run_config[\"benchmark_name\"].\n\nDoris/SingleStore env vars:\n- _build_doris_config() and _build_singlestore_config(): single canonical\n resolvers for the CLI path. All env vars are HERE.\n- from_config() in both adapters: only DORIS_DATABASE / SINGLESTORE_DATABASE\n env var remains, for benchmark-derived database naming. This is the\n legitimate owner layer (from_config has benchmark+scale_factor context).\n- __init__() in both adapters: no env vars, Python defaults only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run focused verification and post-fix audits to ensure only legitimate compatibility boundaries still use fallbacks", "wid": "w12"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "Update the SQL lifecycle so RunConfig carries benchmark=benchmark_config.name\nand the SQL path does not drop that field before calling the adapter.\n\nPrimary files:\n- benchbox/core/runner/runner.py\n- benchbox/core/schemas.py\n\nCaller contract to enforce:\n- SQL path matches the DataFrame path in explicit benchmark context quality\n- platform_config/get_platform_config remains the owner of platform config,\n not benchmark identity backfilling inside adapters\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Thread benchmark identity through the SQL runner instead of stripping it before adapter.run_benchmark()", "wid": "w2"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w2"], "notes": "Replace active-path uses of benchmark._name / display_name / class-name\ninference with explicit benchmark identity from run_config or a directly\npassed benchmark_id argument for execution routing and validation.\n\nPrimary files:\n- benchbox/platforms/base/adapter.py\n\nSpecific call sites to clean up:\n- _get_benchmark_type()\n- _execute_power_test()\n- _execute_throughput_test()\n- _execute_maintenance_test()\n- _execute_combined_test()\n\nPreserve benchmark object access for legitimate benchmark behavior\nexecution; only remove identity inference used to compensate for omitted\ncaller context.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove base-adapter benchmark sniffing from live routing and validation", "wid": "w3"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w2"], "notes": "Clean up the remaining benchmark-identity inference in query selection and\nsingle-query execution paths.\n\nPrimary file:\n- benchbox/platforms/base/adapter.py\n\nSpecific call sites to clean up:\n- _get_dialect_queries()\n- _execute_single_query()\n\nKeep dialect and query execution behavior intact; only remove the need to\nsniff benchmark identity from wrapper internals.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove base-adapter benchmark sniffing from dialect selection and single-query execution", "wid": "w4"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w2"], "notes": "Ensure live-created BenchmarkResults receive benchmark_id from\nexecution_metadata produced by the runner/adapter path.\n\nPrimary files:\n- benchbox/base.py\n- benchbox/core/results/result_factory.py\n- benchbox/core/results/models.py\n\nRequired boundary decision:\n- Live execution paths must not rely on benchmark-name normalization\n- Historical artifact compatibility in benchbox/core/results/schema.py and\n loader utilities remains intact and must stay a compatibility boundary\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make live result creation consume explicit benchmark_id instead of deriving it from names", "wid": "w5"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w5"], "notes": "Ensure export code consumes the explicit benchmark_id already present on\nlive-created results instead of re-normalizing from names.\n\nPrimary file:\n- benchbox/core/results/exporter.py\n\nPreserve compatibility behavior for historical artifacts that enter through\nloader/schema boundaries rather than the live execution path.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make result export preserve explicit benchmark_id without name-derived drift", "wid": "w6"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "DryRunExecutor already receives BenchmarkConfig and already uses it for\nplatform-config injection. Extend that same explicit contract to query\nextraction, real-test dry-run execution, data-size estimation, and memory\nestimation.\n\nPrimary file:\n- benchbox/core/dryrun.py\n\nTarget behaviors:\n- TPC test-mode branching keys off benchmark_config.name / canonical id\n- dry-run estimates do not depend on wrapper class names or _name sniffing\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor dry-run query extraction and estimation to use BenchmarkConfig.name instead of benchmark-object sniffing", "wid": "w7"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "Keep benchmark-aware database naming where benchmark/scale context first\nexists, but stop re-resolving host/port/http_port/username/password/database\ninside from_config() and __init__() for the standard config-built path.\n\nPrimary files:\n- benchbox/platforms/doris.py\n- benchbox/cli/database.py\n- benchbox/cli/platform_hooks.py\n- benchbox/core/platform_config.py\n\nEnd state:\n- builder returns resolved DatabaseConfig\n- from_config() maps/resolves only benchmark-derived database naming that\n truly belongs there\n- __init__() consumes resolved values instead of repeating env/default logic\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Doris config builder the single owner of standard CLI connection/env/default resolution", "wid": "w8"}, {"evidence": null, "item_id": "eliminate-non-data-loading-wrong-layer-compensation", "needs": ["w1"], "notes": "Apply the same ownership cleanup as Doris to SingleStore.\n\nPrimary files:\n- benchbox/platforms/singlestore.py\n- benchbox/cli/database.py\n- benchbox/cli/platform_hooks.py\n- benchbox/core/platform_config.py\n\nEnd state:\n- no adapter-side env/database fallback in the standard config-aware path\n- explicit database overrides still win\n- generated benchmark-scoped database naming still works when no explicit\n database is provided\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make SingleStore config builder the single owner of standard CLI connection/env/default resolution", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-08T00:00:00Z", "deferrals": [], "deps": [], "description": "Integrate the full tuning configuration directly into benchmark result output files.\n\nCurrently, benchmark results reference the tuning file path but do not capture the\nactual tuning configuration content. This is problematic because:\n- The tuning file could be modified by subsequent runs\n- Results become non-reproducible without the exact tuning config used\n- Comparing results across different tuning configs becomes unreliable\n\nUnlike query text (which is deterministically generated from benchmark definitions),\ntuning configurations are user-defined and can vary between runs. The actual tuning\ncontent must be captured at execution time to ensure result reproducibility.\n\nSolution: Embed the complete tuning configuration (clauses, indexes, constraints,\netc.) directly in the result output, either inline or as a companion file.", "id": "embed-tuning-config-in-results", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Embed Full Tuning Config in Result Output", "verifications": [], "work": [], "worktree": "result-object-migration"} -{"anti_patterns": [{"INSTEAD": "add the manifest helper as a separate function", "dont": "change _ensure_data_generated return type or call conventions", "why": "runner logic for reuse/force-regen depends on the bool return"}, {"INSTEAD": "catch all exceptions and return {}", "dont": "raise exceptions from _read_datagen_stats_from_manifest", "why": "stats are non-critical and a crash would block all datagen runs"}, {"INSTEAD": "only call _read_datagen_stats_from_manifest in the data_only branch", "dont": "read the manifest eagerly in every code path", "why": "manifest may not exist for all run types"}, {"INSTEAD": "only add duration_seconds before it", "dont": "rename or remove the **kwargs parameter from create_enhanced_benchmark_result", "why": "many call sites pass platform-specific kwargs"}], "approach": "Work through the three bugs in dependency order (builder \u2192 base_benchmark \u2192 runner).\n\nFor w1: add _total_duration_seconds: float | None = None to ResultBuilder.__init__\n(after _execution_phases_override), add set_total_duration(seconds: float) setter,\nand prepend a check in _calculate_duration():\n if self._total_duration_seconds is not None:\n return self._total_duration_seconds\n\nFor w2: add duration_seconds: Optional[float] = None as an explicit keyword parameter\nto create_enhanced_benchmark_result() (after performance_characteristics, before **kwargs).\nAfter builder construction, call: if duration_seconds is not None: builder.set_total_duration(duration_seconds)\n\nFor w3: in runner.py data-only block (line 643), compute datagen_duration_ms = int(datagen_duration * 1000)\nand include it in both places the phases dict appears.\n\nFor w4: add helper _read_datagen_stats_from_manifest(benchmark) near _ensure_data_generated().\nUse existing load_manifest() from benchbox.core.manifest.io. Handle both ManifestV1 (tables is\ndict[str, list[FileEntry]]) and ManifestV2 (tables is dict[str, TableFormats] with .formats dict).\nReturn {} on any failure - stats are non-critical. Merge returned dict into phase_data with **.\n\nFor w5: use pytest fixtures and mock benchmark objects with a temp manifest on disk.\nParametrize over tpch/tpcds/ssb to confirm benchmark-agnostic behavior.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-25T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": ["propagate-duration-seconds-in-enhanced-results"], "description": "Successful datagen-only runs (--phases generate) produce results JSON with\nnear-zero metrics: run.total_duration_ms is always 0, phases.data_generation\nhas only {\"status\": \"COMPLETED\"} with no duration_ms, and no per-table stats\nare surfaced anywhere.\n\nThree distinct bugs prevent existing timing and manifest data from reaching\nthe JSON output:\n\n1. duration_seconds kwarg silently dropped: runner.py captures datagen_duration\n and passes it to create_enhanced_benchmark_result(), but that method does\n not declare duration_seconds as an explicit parameter - it lands in **kwargs\n and is never extracted. ResultBuilder._calculate_duration() then returns 0.0\n (no query times, no loading time). Fix: add set_total_duration() to builder\n and wire the param through.\n\n2. duration_ms missing from phases dict: runner.py passes\n phases={\"data_generation\": {\"status\": \"COMPLETED\"}} without duration_ms.\n Fix: include \"duration_ms\": int(datagen_duration * 1000) in the dict.\n\n3. Manifest-level per-table stats never surfaced: the manifest\n (_datagen_manifest.json) written by every generator contains per-file\n row_count and size_bytes. After _ensure_data_generated() completes these\n sit on disk but are never read back. Fix: add _read_datagen_stats_from_manifest()\n helper that aggregates tables_generated, total_rows, total_size_bytes and\n merges them into the phases dict.", "id": "emit-datagen-metrics-in-results-json", "preserves": ["All existing callers of create_enhanced_benchmark_result() continue to work - duration_seconds was previously silently accepted via **kwargs; making it explicit must not break any call site", "ResultBuilder._calculate_duration() fallback logic (start/end times, then query time sum) remains intact for non-datagen paths", "Standard benchmark runs (load+power) report correct total_duration_ms and phase timings - no regression", "_ensure_data_generated() return type (bool) and calling convention in runner.py unchanged"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/builder.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/runner/test_datagen_metrics.py"}], "state": "done", "title": "Emit meaningful datagen metrics in results JSON", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_duration_in_run_block -v", "description": "run.total_duration_ms is non-zero after a datagen run", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_phase_has_duration_ms -v", "description": "phases.data_generation.duration_ms present and > 0 in result JSON", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_phase_has_table_stats_when_manifest_present -v", "description": "Manifest stats surfaced when manifest file exists on disk", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_phase_gracefully_degrades_without_manifest -v", "description": "No crash when manifest is absent; phase still has status and duration_ms", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Full test suite passes with no regression", "expected": "All tests pass with 0 failures", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add set_total_duration() to ResultBuilder; update _calculate_duration() to check it first", "wid": "w1"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add duration_seconds as explicit param to create_enhanced_benchmark_result(); call builder.set_total_duration() when provided", "wid": "w2"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Include duration_ms in phases dict and execution_metadata.phase_status in runner.py data-only result block", "wid": "w3"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add _read_datagen_stats_from_manifest() and _flatten_manifest_v2_entries() helpers; merge stats into phase dict", "wid": "w4"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests/unit/core/runner/test_datagen_metrics.py with 5 tests covering duration, phase stats, manifest presence/absence", "wid": "w5"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change _ensure_data_generated return type or call conventions", "instead": "add the manifest helper as a separate function", "why": "runner logic for reuse/force-regen depends on the bool return"}, {"dont": "raise exceptions from _read_datagen_stats_from_manifest", "instead": "catch all exceptions and return {}", "why": "stats are non-critical and a crash would block all datagen runs"}, {"dont": "read the manifest eagerly in every code path", "instead": "only call _read_datagen_stats_from_manifest in the data_only branch", "why": "manifest may not exist for all run types"}, {"dont": "rename or remove the **kwargs parameter from create_enhanced_benchmark_result", "instead": "only add duration_seconds before it", "why": "many call sites pass platform-specific kwargs"}], "approach": "Work through the three bugs in dependency order (builder \u2192 base_benchmark \u2192 runner).\n\nFor w1: add _total_duration_seconds: float | None = None to ResultBuilder.__init__\n(after _execution_phases_override), add set_total_duration(seconds: float) setter,\nand prepend a check in _calculate_duration():\n if self._total_duration_seconds is not None:\n return self._total_duration_seconds\n\nFor w2: add duration_seconds: Optional[float] = None as an explicit keyword parameter\nto create_enhanced_benchmark_result() (after performance_characteristics, before **kwargs).\nAfter builder construction, call: if duration_seconds is not None: builder.set_total_duration(duration_seconds)\n\nFor w3: in runner.py data-only block (line 643), compute datagen_duration_ms = int(datagen_duration * 1000)\nand include it in both places the phases dict appears.\n\nFor w4: add helper _read_datagen_stats_from_manifest(benchmark) near _ensure_data_generated().\nUse existing load_manifest() from benchbox.core.manifest.io. Handle both ManifestV1 (tables is\ndict[str, list[FileEntry]]) and ManifestV2 (tables is dict[str, TableFormats] with .formats dict).\nReturn {} on any failure - stats are non-critical. Merge returned dict into phase_data with **.\n\nFor w5: use pytest fixtures and mock benchmark objects with a temp manifest on disk.\nParametrize over tpch/tpcds/ssb to confirm benchmark-agnostic behavior.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-25T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": ["propagate-duration-seconds-in-enhanced-results"], "description": "Successful datagen-only runs (--phases generate) produce results JSON with\nnear-zero metrics: run.total_duration_ms is always 0, phases.data_generation\nhas only {\"status\": \"COMPLETED\"} with no duration_ms, and no per-table stats\nare surfaced anywhere.\n\nThree distinct bugs prevent existing timing and manifest data from reaching\nthe JSON output:\n\n1. duration_seconds kwarg silently dropped: runner.py captures datagen_duration\n and passes it to create_enhanced_benchmark_result(), but that method does\n not declare duration_seconds as an explicit parameter - it lands in **kwargs\n and is never extracted. ResultBuilder._calculate_duration() then returns 0.0\n (no query times, no loading time). Fix: add set_total_duration() to builder\n and wire the param through.\n\n2. duration_ms missing from phases dict: runner.py passes\n phases={\"data_generation\": {\"status\": \"COMPLETED\"}} without duration_ms.\n Fix: include \"duration_ms\": int(datagen_duration * 1000) in the dict.\n\n3. Manifest-level per-table stats never surfaced: the manifest\n (_datagen_manifest.json) written by every generator contains per-file\n row_count and size_bytes. After _ensure_data_generated() completes these\n sit on disk but are never read back. Fix: add _read_datagen_stats_from_manifest()\n helper that aggregates tables_generated, total_rows, total_size_bytes and\n merges them into the phases dict.", "id": "emit-datagen-metrics-in-results-json", "preserves": ["All existing callers of create_enhanced_benchmark_result() continue to work - duration_seconds was previously silently accepted via **kwargs; making it explicit must not break any call site", "ResultBuilder._calculate_duration() fallback logic (start/end times, then query time sum) remains intact for non-datagen paths", "Standard benchmark runs (load+power) report correct total_duration_ms and phase timings - no regression", "_ensure_data_generated() return type (bool) and calling convention in runner.py unchanged"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/base_benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/builder.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/runner/test_datagen_metrics.py"}], "state": "done", "title": "Emit meaningful datagen metrics in results JSON", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_duration_in_run_block -v", "description": "run.total_duration_ms is non-zero after a datagen run", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_phase_has_duration_ms -v", "description": "phases.data_generation.duration_ms present and > 0 in result JSON", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_phase_has_table_stats_when_manifest_present -v", "description": "Manifest stats surfaced when manifest file exists on disk", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core/runner/test_datagen_metrics.py::test_datagen_phase_gracefully_degrades_without_manifest -v", "description": "No crash when manifest is absent; phase still has status and duration_ms", "expected": "PASSED", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Full test suite passes with no regression", "expected": "All tests pass with 0 failures", "item_id": "emit-datagen-metrics-in-results-json", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add set_total_duration() to ResultBuilder; update _calculate_duration() to check it first", "wid": "w1"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add duration_seconds as explicit param to create_enhanced_benchmark_result(); call builder.set_total_duration() when provided", "wid": "w2"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Include duration_ms in phases dict and execution_metadata.phase_status in runner.py data-only result block", "wid": "w3"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add _read_datagen_stats_from_manifest() and _flatten_manifest_v2_entries() helpers; merge stats into phase dict", "wid": "w4"}, {"evidence": null, "item_id": "emit-datagen-metrics-in-results-json", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add tests/unit/core/runner/test_datagen_metrics.py with 5 tests covering duration, phase stats, manifest presence/absence", "wid": "w5"}], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Platform Expansion", "claimed_at": null, "claimed_by": null, "completed_at": "2025-12-24T00:00:00Z", "completed_pr": null, "created_at": "2025-12-24T00:00:00Z", "deferrals": [], "deps": ["aws-glue-platform"], "description": "Add Amazon EMR Serverless managed Spark platform adapter. EMR Serverless provides\ntrue serverless Spark with sub-second startup and automatic resource scaling.\n\nImplementation reuses S3 staging and IAM auth patterns from AWS Glue adapter.\nUses EMR Serverless API for application and job management.", "id": "emr-serverless-platform", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Amazon EMR Serverless Platform", "verifications": [], "work": [], "worktree": "platform-expansion"} -{"anti_patterns": [{"INSTEAD": "solve the problem via a supported configuration surface instead.", "dont": "monkey-patch `BROWSER_RUNTIME._globalFileInfo` from the page", "why": "it lives inside the DuckDB-WASM worker and crosses a private API boundary"}, {"INSTEAD": "fix the range-read path rather than replacing it.", "dont": "move the explorer off DuckDB-WASM's HTTP attach to regain range reads", "why": "the canonical browser read model depends on it"}], "approach": "Do not introduce a custom forked build of duckdb-wasm unless every\nother avenue fails. Prefer a library upgrade, then a supported config\nentry point, then an upstream fix. The goal is to keep the explorer's\nHTTP attach logic conventional so future maintainers do not carry a\nprivate patch.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Performance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-19T00:00:00Z", "deferrals": [], "deps": ["implement-results-explorer-browser-functional-tests"], "description": "The explorer attaches `results.duckdb` at\n`results-explorer/src/db.ts` via\n`registerFileURL(\"results.duckdb\", url, DuckDBDataProtocol.HTTP, true)`\nfollowed by `ATTACH 'results.duckdb' AS bench (READ_ONLY)`. The RG-2\ngate for the canonical browser read model requires that a cold\nsingle-row lookup transfers \u226410% of the on-disk DB, i.e. DuckDB-WASM\nshould use HTTP `Range:` requests rather than a single whole-file GET.\n\nIn practice, with duckdb-wasm 1.32.0, the runtime's HTTP file-info\ndefaults (`reliableHeadRequests=false`, `allowFullHttpReads=true`)\nshort-circuit the Range probe even when `directIO=true` is passed to\n`registerFileURL`, and the runtime falls back to a single whole-file\nGET on ATTACH. There is no stable per-URL API to override those\ndefaults from the main thread:\n\n- `db.open({filesystem: {reliableHeadRequests: true, allowFullHTTPReads: false}})`\n resets the WebFileSystem and breaks the already-registered URL.\n- `SET reliable_head_requests = true` affects only the httpfs extension\n HTTP client, not the runtime's built-in HTTP file reader.\n- The per-file `_globalFileInfo` object that carries these flags lives\n inside the DuckDB-WASM worker and is not reachable from the page.\n\nThe e2e harness already serves the snapshot with `Accept-Ranges: bytes`\nand honours `Range: bytes=...` with `206 Partial Content` (see\n`results-explorer/scripts/serve-browser-tests.mjs` and\n`results-explorer/e2e/capability/range-read-budget.spec.ts`), so the\nmoment the browser runtime starts issuing range reads, the RG-2\nbyte-budget assertion can be re-introduced without any server change.\n\nDeliverables:\n1. Confirm the library-level behaviour against the current\n duckdb-wasm version at the time of work start (it may have\n gained a supported override by then).\n2. Pick a path from: (a) upgrade duckdb-wasm to a version that\n supports per-URL range-read flags, (b) patch the runtime via a\n supported entry point such as a config object accepted by\n `registerFileURL`, or (c) file an upstream issue/PR and pin to a\n build that includes the fix.\n3. Re-enable the byte-budget assertion in\n `results-explorer/e2e/capability/range-read-budget.spec.ts` using\n the server-side `/__test/transfers` accounting already in place.\n", "id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "preserves": ["The server-side transfer log at `/__test/transfers` remains available for the re-enabled assertion.", "The test server's `Accept-Ranges`/`Range` handling in `results-explorer/scripts/serve-browser-tests.mjs` is not regressed.", "`results-explorer/src/db.ts` continues to ATTACH the snapshot READ_ONLY and must not fall back to a JSON-only path."], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "results-data/bundles/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/public/data/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/scripts/serve-browser-tests.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/capability/range-read-budget.spec.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/package-lock.json"}, {"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}], "state": "active", "title": "Enable DuckDB-WASM HTTP range-reads for URLs registered via registerFileURL", "verifications": [{"command": "uv run _project/scripts/validate_todo.py _project/TODO/main/planning/enable-duckdb-wasm-http-range-reads-for-registered-urls.yaml", "description": "The TODO item itself validates cleanly", "expected": "Validation passes with no schema or DAG errors", "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium -- --grep \"RG-2 range-read\"", "description": "After the fix lands, a cold explorer load transfers \u226410% of the snapshot", "expected": "The byte-budget assertion is re-enabled and passes", "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "needs": [], "notes": "Audited duckdb-wasm 1.32.0 (latest stable as of 2026-04-27;\n1.33.x exists only as dev pre-release builds). Findings:\n\n- `DuckDBConfig.filesystem` in\n `dist/types/src/bindings/config.d.ts` exposes three booleans:\n `reliableHeadRequests`, `allowFullHTTPReads`, and\n `forceFullHTTPReads`. There is no per-URL override on\n `registerFileURL`.\n- `db.open({filesystem: {...}})` is the public entry point.\n- Order matters: `open` AFTER `registerFileURL` resets the\n WebFileSystem and breaks the registration; `open` BEFORE\n registration is the supported sequence.\n- `allowFullHTTPReads: false` makes the runtime error on\n first ATTACH (\"IO Error: Cannot open database 'results.duckdb'\n in read-only mode: database does not exist\"). This matches\n upstream issue duckdb/duckdb-wasm#1984's analysis: when the\n flag is false the runtime \"always errors\". So we cannot use it\n to force range-only reads.\n- With `reliableHeadRequests: true` + `forceFullHTTPReads: false`\n + the default `allowFullHTTPReads: true`, the page loads but\n the runtime still issues a single whole-file GET against the\n registered URL (verified by the test harness transfer log:\n total 6,565,888 bytes = full DB size, no 206 responses).\n- Issue #1984 documents the underlying buggy code path: even\n when the runtime first attempts a range read, an extra \"full\n GET anyway\" code block runs after. This block has not been\n removed in 1.32.0.\n\nConclusion: there is no flag combination on duckdb-wasm 1.32.0\nthat yields range-only reads against an HTTP-registered URL.\nThe fix must come from upstream (or from a 1.33.x stable\nrelease once one ships). The byte-budget assertion has been\nadded to range-read-budget.spec.ts under `test.skip` so it\nauto-enables when the runtime starts issuing 206s.\n\nUpdate 2026-04-27: also verified against `@duckdb/duckdb-wasm@\n1.33.1-dev53.0` (the latest dev pre-release). Same behavior \u2014\nwhole-file GET on ATTACH, zero 206 responses, byte-budget\nexceeded by 10x. Upstream has not addressed this in any 1.33.x\nbuild to date. Reverted to 1.32.0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-audit duckdb-wasm runtime flags and the HTTP file-reader code path for the current version", "wid": "w1"}, {"evidence": null, "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "needs": ["w1"], "notes": "Update 2026-05-01: still blocked on upstream. Current npm metadata\nshows no stable `@duckdb/duckdb-wasm` release beyond `1.32.0`; the\npackage dist-tags point at dev builds (`latest=1.33.1-dev45.0`,\n`next=1.33.1-dev53.0`). Upstream issue duckdb/duckdb-wasm#1984 is\nstill open, with no maintainer resolution or release note indicating\na supported range-only path for HTTP-registered URLs.\n\n`/code review W2`: no required code changes are available under the\nTODO guardrails. Upgrading to a dev pre-release or carrying a private\npatched runtime would violate the `approach` constraint unless the\nmaintainer explicitly accepts that tradeoff.\n\nOf the three candidate paths:\n\n1. **Library upgrade** \u2014 no stable 1.33.x release is available\n (only dev builds 1.33.1-dev1 through 1.33.1-dev53 as of\n 2026-05-01). Pinning to a dev pre-release violates the\n `approach` constraint (\"Do not introduce a custom forked build\n of duckdb-wasm unless every other avenue fails\").\n2. **Supported runtime API** \u2014 w1 verified that no flag\n combination on the public `DuckDBConfig.filesystem` API yields\n range-only reads. The buggy \"Perform a full GET anyways\" code\n path in runtime_browser.ts is not toggleable from the page.\n3. **Upstream issue/PR** \u2014 this is the path forward. File or\n comment on duckdb/duckdb-wasm#1984 with a minimal repro\n showing that even `reliableHeadRequests: true` +\n `forceFullHTTPReads: false` produces a whole-file GET on\n ATTACH against an `HTTP, directIO=true` registration. Pin to\n the first 1.33.x stable that includes the fix.\n\nThis work unit unblocks once a 1.33.x stable release with the\nfix is published OR a private patched build is acceptable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Pick an unblock path and prototype it", "wid": "w2"}, {"evidence": null, "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "needs": ["w2"], "notes": "Partially landed: the byte-budget assertion is now coded in\n`results-explorer/e2e/capability/range-read-budget.spec.ts`\nunder `test.skip`. It asserts after a cold `goto` of `/results/`\nfollowed by `waitForDataLoaded` that:\n - the sum of `Content-Length` transfers under\n `/results/data/results.duckdb` is \u226410% of the on-disk DB\n size, and\n - at least one transfer is a 206 ranged response.\n\nThe test is currently `.skip` because w2 is blocked. Removing\n`.skip` is a one-line change once the runtime issues range\nreads. The capability tests survive as a diagnostic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-enable the byte-budget assertion and close the RG-2 gate", "wid": "w3"}], "worktree": "main"} +{"anti_patterns": [{"dont": "monkey-patch `BROWSER_RUNTIME._globalFileInfo` from the page", "instead": "solve the problem via a supported configuration surface instead.", "why": "it lives inside the DuckDB-WASM worker and crosses a private API boundary"}, {"dont": "move the explorer off DuckDB-WASM's HTTP attach to regain range reads", "instead": "fix the range-read path rather than replacing it.", "why": "the canonical browser read model depends on it"}], "approach": "Do not introduce a custom forked build of duckdb-wasm unless every\nother avenue fails. Prefer a library upgrade, then a supported config\nentry point, then an upstream fix. The goal is to keep the explorer's\nHTTP attach logic conventional so future maintainers do not carry a\nprivate patch.\n", "blocked_reason": "imported from YAML status: Blocked", "category": "Performance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-19T00:00:00Z", "deferrals": [], "deps": ["implement-results-explorer-browser-functional-tests"], "description": "The explorer attaches `results.duckdb` at\n`results-explorer/src/db.ts` via\n`registerFileURL(\"results.duckdb\", url, DuckDBDataProtocol.HTTP, true)`\nfollowed by `ATTACH 'results.duckdb' AS bench (READ_ONLY)`. The RG-2\ngate for the canonical browser read model requires that a cold\nsingle-row lookup transfers \u226410% of the on-disk DB, i.e. DuckDB-WASM\nshould use HTTP `Range:` requests rather than a single whole-file GET.\n\nIn practice, with duckdb-wasm 1.32.0, the runtime's HTTP file-info\ndefaults (`reliableHeadRequests=false`, `allowFullHttpReads=true`)\nshort-circuit the Range probe even when `directIO=true` is passed to\n`registerFileURL`, and the runtime falls back to a single whole-file\nGET on ATTACH. There is no stable per-URL API to override those\ndefaults from the main thread:\n\n- `db.open({filesystem: {reliableHeadRequests: true, allowFullHTTPReads: false}})`\n resets the WebFileSystem and breaks the already-registered URL.\n- `SET reliable_head_requests = true` affects only the httpfs extension\n HTTP client, not the runtime's built-in HTTP file reader.\n- The per-file `_globalFileInfo` object that carries these flags lives\n inside the DuckDB-WASM worker and is not reachable from the page.\n\nThe e2e harness already serves the snapshot with `Accept-Ranges: bytes`\nand honours `Range: bytes=...` with `206 Partial Content` (see\n`results-explorer/scripts/serve-browser-tests.mjs` and\n`results-explorer/e2e/capability/range-read-budget.spec.ts`), so the\nmoment the browser runtime starts issuing range reads, the RG-2\nbyte-budget assertion can be re-introduced without any server change.\n\nDeliverables:\n1. Confirm the library-level behaviour against the current\n duckdb-wasm version at the time of work start (it may have\n gained a supported override by then).\n2. Pick a path from: (a) upgrade duckdb-wasm to a version that\n supports per-URL range-read flags, (b) patch the runtime via a\n supported entry point such as a config object accepted by\n `registerFileURL`, or (c) file an upstream issue/PR and pin to a\n build that includes the fix.\n3. Re-enable the byte-budget assertion in\n `results-explorer/e2e/capability/range-read-budget.spec.ts` using\n the server-side `/__test/transfers` accounting already in place.\n", "id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "preserves": ["The server-side transfer log at `/__test/transfers` remains available for the re-enabled assertion.", "The test server's `Accept-Ranges`/`Range` handling in `results-explorer/scripts/serve-browser-tests.mjs` is not regressed.", "`results-explorer/src/db.ts` continues to ATTACH the snapshot READ_ONLY and must not fall back to a JSON-only path."], "prior_art": [], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "results-data/bundles/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/public/data/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/scripts/serve-browser-tests.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/capability/range-read-budget.spec.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/package-lock.json"}, {"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}], "state": "active", "title": "Enable DuckDB-WASM HTTP range-reads for URLs registered via registerFileURL", "verifications": [{"command": "uv run _project/scripts/validate_todo.py _project/TODO/main/planning/enable-duckdb-wasm-http-range-reads-for-registered-urls.yaml", "description": "The TODO item itself validates cleanly", "expected": "Validation passes with no schema or DAG errors", "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium -- --grep \"RG-2 range-read\"", "description": "After the fix lands, a cold explorer load transfers \u226410% of the snapshot", "expected": "The byte-budget assertion is re-enabled and passes", "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "needs": [], "notes": "Audited duckdb-wasm 1.32.0 (latest stable as of 2026-04-27;\n1.33.x exists only as dev pre-release builds). Findings:\n\n- `DuckDBConfig.filesystem` in\n `dist/types/src/bindings/config.d.ts` exposes three booleans:\n `reliableHeadRequests`, `allowFullHTTPReads`, and\n `forceFullHTTPReads`. There is no per-URL override on\n `registerFileURL`.\n- `db.open({filesystem: {...}})` is the public entry point.\n- Order matters: `open` AFTER `registerFileURL` resets the\n WebFileSystem and breaks the registration; `open` BEFORE\n registration is the supported sequence.\n- `allowFullHTTPReads: false` makes the runtime error on\n first ATTACH (\"IO Error: Cannot open database 'results.duckdb'\n in read-only mode: database does not exist\"). This matches\n upstream issue duckdb/duckdb-wasm#1984's analysis: when the\n flag is false the runtime \"always errors\". So we cannot use it\n to force range-only reads.\n- With `reliableHeadRequests: true` + `forceFullHTTPReads: false`\n + the default `allowFullHTTPReads: true`, the page loads but\n the runtime still issues a single whole-file GET against the\n registered URL (verified by the test harness transfer log:\n total 6,565,888 bytes = full DB size, no 206 responses).\n- Issue #1984 documents the underlying buggy code path: even\n when the runtime first attempts a range read, an extra \"full\n GET anyway\" code block runs after. This block has not been\n removed in 1.32.0.\n\nConclusion: there is no flag combination on duckdb-wasm 1.32.0\nthat yields range-only reads against an HTTP-registered URL.\nThe fix must come from upstream (or from a 1.33.x stable\nrelease once one ships). The byte-budget assertion has been\nadded to range-read-budget.spec.ts under `test.skip` so it\nauto-enables when the runtime starts issuing 206s.\n\nUpdate 2026-04-27: also verified against `@duckdb/duckdb-wasm@\n1.33.1-dev53.0` (the latest dev pre-release). Same behavior \u2014\nwhole-file GET on ATTACH, zero 206 responses, byte-budget\nexceeded by 10x. Upstream has not addressed this in any 1.33.x\nbuild to date. Reverted to 1.32.0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-audit duckdb-wasm runtime flags and the HTTP file-reader code path for the current version", "wid": "w1"}, {"evidence": null, "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "needs": ["w1"], "notes": "Update 2026-05-01: still blocked on upstream. Current npm metadata\nshows no stable `@duckdb/duckdb-wasm` release beyond `1.32.0`; the\npackage dist-tags point at dev builds (`latest=1.33.1-dev45.0`,\n`next=1.33.1-dev53.0`). Upstream issue duckdb/duckdb-wasm#1984 is\nstill open, with no maintainer resolution or release note indicating\na supported range-only path for HTTP-registered URLs.\n\n`/code review W2`: no required code changes are available under the\nTODO guardrails. Upgrading to a dev pre-release or carrying a private\npatched runtime would violate the `approach` constraint unless the\nmaintainer explicitly accepts that tradeoff.\n\nOf the three candidate paths:\n\n1. **Library upgrade** \u2014 no stable 1.33.x release is available\n (only dev builds 1.33.1-dev1 through 1.33.1-dev53 as of\n 2026-05-01). Pinning to a dev pre-release violates the\n `approach` constraint (\"Do not introduce a custom forked build\n of duckdb-wasm unless every other avenue fails\").\n2. **Supported runtime API** \u2014 w1 verified that no flag\n combination on the public `DuckDBConfig.filesystem` API yields\n range-only reads. The buggy \"Perform a full GET anyways\" code\n path in runtime_browser.ts is not toggleable from the page.\n3. **Upstream issue/PR** \u2014 this is the path forward. File or\n comment on duckdb/duckdb-wasm#1984 with a minimal repro\n showing that even `reliableHeadRequests: true` +\n `forceFullHTTPReads: false` produces a whole-file GET on\n ATTACH against an `HTTP, directIO=true` registration. Pin to\n the first 1.33.x stable that includes the fix.\n\nThis work unit unblocks once a 1.33.x stable release with the\nfix is published OR a private patched build is acceptable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Pick an unblock path and prototype it", "wid": "w2"}, {"evidence": null, "item_id": "enable-duckdb-wasm-http-range-reads-for-registered-urls", "needs": ["w2"], "notes": "Partially landed: the byte-budget assertion is now coded in\n`results-explorer/e2e/capability/range-read-budget.spec.ts`\nunder `test.skip`. It asserts after a cold `goto` of `/results/`\nfollowed by `waitForDataLoaded` that:\n - the sum of `Content-Length` transfers under\n `/results/data/results.duckdb` is \u226410% of the on-disk DB\n size, and\n - at least one transfer is a 206 ranged response.\n\nThe test is currently `.skip` because w2 is blocked. Removing\n`.skip` is a one-line change once the runtime issues range\nreads. The capability tests survive as a diagnostic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-enable the byte-budget assertion and close the RG-2 gate", "wid": "w3"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Data Quality and Compatibility Issues", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Tasks**:\"\n- \"**Files Affected**:\"\n- \"**Implementation Summary**:\"\n- \"**Impact**: \u2705 **COMPLETED** - Critical development workflow improvement achieved\"\n- \"**Verification Results**: \u2705 **ALL REQUIREMENTS SATISFIED**\"", "id": "enable-fractional-scale-factor-support", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Enable Fractional Scale Factor Support", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Data Quality and Compatibility Issues", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Tasks**:\"\n- \"**Final Result**:\"\n- \"**Files Affected**:\"", "id": "enable-ssb-clickhouse-embedded-platform-support", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Enable SSB ClickHouse Embedded Platform Support", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change timing metric filters (measurement-only) as part of tagging cleanup", "why": "this TODO is metadata-contract hardening, not metric-policy redesign"}, {"INSTEAD": "keep run_type as per-row execution role", "dont": "reinterpret run_type as execution mode (`power|throughput|maintenance`)", "why": "run mode is already carried by `test_execution_type`"}, {"INSTEAD": "retain compatibility while enforcing explicit tagging for new producer output", "dont": "remove fallback inference in query normalization immediately", "why": "historical artifacts may lack run_type"}], "approach": "1) Define and document canonical run_type vocabulary for query rows and\nsynthetic rows.\n2) Add explicit producer-side stamping in SQL standard/throughput/maintenance\npaths so run_type is present before normalization.\n3) Propagate run_type into enhanced QueryExecution phase objects.\n4) Keep fallback inference as compatibility-only safety net (not primary path).\n5) Add contract tests that fail if any producer emits query rows without run_type.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Query result rows currently rely on fallback inference for `run_type` in several\npaths (especially SQL `standard` execution), while other paths set run_type\nexplicitly (`warmup`/`measurement`, and DataFrame metadata/summary rows).\n\nThis creates ambiguity and repeated fallback logic in normalization/schema code,\nand can cause inconsistent downstream analytics semantics when producer paths\nomit run_type.\n\nThis TODO defines canonical run_type semantics and ensures all producer paths\nexplicitly stamp run_type before result building/export. It also aligns\nenhanced execution-phase objects and adds contract tests so missing run_type\ncannot regress silently.", "id": "enforce-explicit-query-run-type-across-executions", "preserves": ["Backward compatibility for loading historical results without explicit run_type remains intact via normalization fallback", "DataFrame metadata/summary synthetic rows keep their current run_type values (`metadata`/`summary`)", "Existing power/throughput behavior where warmup rows are excluded from primary timing metrics remains unchanged", "`test_execution_type` continues to represent run mode (`standard|power|throughput|maintenance|combined`) and is NOT replaced by run_type"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/query_normalizer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Enforce explicit query run_type tagging across SQL and DataFrame execution paths", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms -k 'run_type and (standard or throughput or maintenance or power)' -q", "description": "SQL adapter paths stamp run_type explicitly across execution types", "expected": "selected tests pass; run_type present assertions succeed", "item_id": "enforce-explicit-query-run-type-across-executions", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/results -k 'run_type or query_normalizer or schema' -q", "description": "Result-layer contract tests enforce explicit run_type and compatibility fallback", "expected": "selected tests pass; no missing run_type regressions", "item_id": "enforce-explicit-query-run-type-across-executions", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/mcp/test_mcp_results_and_analytics.py -k RunBenchmark -q", "description": "MCP and result export behavior remains stable", "expected": "selected run_benchmark MCP tests pass", "item_id": "enforce-explicit-query-run-type-across-executions", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define canonical run_type semantics and allowed values in a shared results-layer location", "wid": "w1"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add SQL producer-side run_type stamping for standard execution outputs", "wid": "w2"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ensure throughput/maintenance and fallback combined paths also emit explicit run_type for query rows", "wid": "w3"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Propagate run_type into `_create_standard_execution_phase()` QueryExecution objects", "wid": "w4"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retain normalization/schema run_type fallback as compatibility-only path with clarifying comments", "wid": "w5"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add platform adapter unit tests asserting run_type presence across execution types", "wid": "w6"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add results-layer contract tests for explicit run_type and fallback compatibility", "wid": "w7"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add MCP/result export regression tests to verify run_type continuity in payloads", "wid": "w8"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run focused verification suite and document any residual edge cases", "wid": "w9"}], "worktree": "core-functionality"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a second pytest_collection_modifyitems function to conftest.py", "why": "Python silently overrides the first; extend the existing hook body instead"}, {"INSTEAD": "(not specified)", "dont": "add new markers to tests without first checking for speed-lane conflicts", "why": "the hook catches it at collection time but a failing CI run wastes 10+ minutes before you see the error; check locally with --collect-only first"}, {"INSTEAD": "(not specified)", "dont": "change test logic while fixing markers", "why": "this is a metadata-only cleanup and any logic change risks introducing regressions that get blamed on the marker fix, making it harder to revert cleanly"}, {"INSTEAD": "(not specified)", "dont": "pipe the lint-markers pytest invocation into tail or any filter", "why": "the exit code must come from pytest directly, otherwise CI wiring silently passes on failure"}, {"INSTEAD": "(not specified)", "dont": "resolve conflicts by removing resource_heavy or live_integration from tests that genuinely need those markers", "why": "those exist to route tests to the right runner; remove the speed-lane marker instead"}, {"INSTEAD": "(not specified)", "dont": "use BENCHBOX_SKIP_MARKER_CHECK=1 in CI", "why": "it defeats the entire purpose"}], "approach": "Work units are sequenced: extend the existing pytest_collection_modifyitems hook\nin warning mode first (w1), placing conflict detection BEFORE the deselection\nslice so all 43 violations are visible regardless of -m filter. CI stays green\nwhile violations are fixed (w2-w13 can run in any order or in parallel - one\nfile per commit). Promote to error mode (w14) only after all violations are\ngone, confirmed by pytest --collect-only returning zero marker-conflict warnings.\nFinally add the make target (w15) to give pre-commit a fast entry point.\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "43 tests across 12 files currently carry mutually exclusive marker combinations\nthat undermine the fast/medium/slow speed-lane semantics and cause selection\nambiguity for both humans and automation.\n\nDiscovered violations (as of 2026-04-01):\n fast + slow : 13 tests in 5 files\n medium + slow : 26 tests in 6 files\n fast + resource_heavy: 4 tests in 1 file\n\nConflict rules to enforce:\n Speed-lane mutual exclusion: a test may carry at most one of {fast, medium, slow}\n Exclusion-set incompatibility: fast is incompatible with {slow, stress, resource_heavy, live_integration}\n\nThe fix has two parts:\n1. Conflict detection merged into the existing pytest_collection_modifyitems hook in\n tests/conftest.py, running BEFORE the deselection slice so all collected items\n (including those about to be deselected by -m filters) are inspected. Starts in\n warning mode, promoted to error mode once all violations are remediated.\n2. Remediation of all 43 existing violations - each test gets a single correct\n speed marker based on what it actually does.\n\nThis makes the pytest_collection_modifyitems exclusion hook (added in\nd84c897a8) a belt-and-suspenders safety net rather than the primary guard.", "id": "enforce-no-conflicting-test-markers", "preserves": ["BENCHBOX_SKIP_MARKER_CHECK=1 suppresses validation errors after w14 promotion (escape hatch preserved); during w1-w13 migration, BENCHBOX_ENFORCE_MARKER_CHECK=1 opts into early error mode", "No test logic changes - only marker annotations modified", "make test-fast and uv run -- python -m pytest -m fast -q collect identical node ID sets (not just equal counts) - enforced by existing collection hook in d84c897a8", "make test-slow, make test-stress, make test-all still collect their expected sets - hook must not block explicit marker opt-in"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/lint.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "tests/conftest.py (validation hook)"}, {"kind": "only_modify", "path_glob": "tests/integration/core/data_organization/test_presorted_generation.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_nyctaxi_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_phase4_validation_integration.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tpch_skew_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tsbs_devops_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/test_tpcdi.py"}, {"kind": "only_modify", "path_glob": "tests/test_tpcds_direct.py"}, {"kind": "only_modify", "path_glob": "tests/unit/benchmarks/test_write_primitives_generator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/monitoring/test_profiler.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databricks_liquid_clustering.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_infrastructure.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_tpcdi_phase1.py"}], "state": "done", "title": "Enforce no conflicting test markers: detection hook + remediation of 43 existing violations", "verifications": [{"command": "uv run -- python -m pytest --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -m \"fast and slow\" -p no:warnings 2>&1 | tail -3", "description": "No fast+slow conflicts after w2-w6", "expected": "no tests collected", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -m \"medium and slow\" -p no:warnings 2>&1 | tail -3", "description": "No medium+slow conflicts after w7-w12", "expected": "no tests collected", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -m \"fast and resource_heavy\" -p no:warnings 2>&1 | tail -3", "description": "No fast+resource_heavy conflicts after w13", "expected": "no tests collected", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest --collect-only -q -p no:warnings 2>&1 | grep -c 'Marker conflict'", "description": "After w14: zero marker-conflict warnings in clean collect-only", "expected": "0", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 4}, {"command": "diff <(uv run -- python -m pytest -m fast --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -p no:warnings 2>&1 | grep '::' | sort) <(make test-fast ARGS='--collect-only -q -p no:warnings' 2>&1 | grep '::' | sort)", "description": "make test-fast node ID set is identical before and after marker remediation", "expected": "empty diff (no output)", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 5}, {"command": "make lint-markers", "description": "make lint-markers exits 0", "expected": "exit code 0", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Merge conflict validation into pytest_collection_modifyitems (warning mode)", "wid": "w1"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_tpcdi.py (2 tests)", "wid": "w10"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_release_infrastructure.py (1 test)", "wid": "w11"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_tpcdi_phase1.py (1 test)", "wid": "w12"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+resource_heavy: test_presorted_generation.py (4 tests)", "wid": "w13"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w10", "w11", "w12", "w13", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Promote validation hook from warning to error mode", "wid": "w14"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w14"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make lint-markers target for pre-commit and CI", "wid": "w15"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_nyctaxi_duckdb.py (3 tests)", "wid": "w2"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_tsbs_devops_duckdb.py (5 tests)", "wid": "w3"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_tpcds_direct.py (1 test)", "wid": "w4"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_profiler.py (3 tests)", "wid": "w5"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_databricks_liquid_clustering.py (1 test)", "wid": "w6"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_write_primitives_generator.py (18 tests)", "wid": "w7"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_tpch_skew_duckdb.py (4 tests)", "wid": "w8"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_phase4_validation_integration.py (2 tests)", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "adopt an external charting library", "why": "the render()->str pattern, benchmark-specific features, and sub-character precision cannot be replicated; custom implementation evaluated as superior to all 10+ candidates"}, {"INSTEAD": "(not specified)", "dont": "compress the entire bar scale for outliers", "why": "it makes most bars unreadably small; truncate outlier bars instead with overflow indicators"}, {"INSTEAD": "(not specified)", "dont": "hardcode terminal width", "why": "base.py already handles terminal detection; use self.width from ASCIIChartBase"}, {"INSTEAD": "(not specified)", "dont": "print to stdout", "why": "all ASCII charts must return strings for MCP tool responses and CLI piping"}, {"INSTEAD": "(not specified)", "dont": "skip Unicode fallback", "why": "ASCII_BLOCK_CHARS and ASCII_BOX_CHARS exist for terminals without Unicode support"}], "approach": "Follow existing inheritance pattern: new classes inherit from ASCIIChartBase in ascii/ module.\nUse existing BLOCK_CHARS, INTENSITY_CHARS, BOX_CHARS from base.py - do not duplicate.\nFollow factory function pattern (from_*_data) for data conversion.\nMirror _render_single_ascii_chart() dispatch pattern in visualization.py for new types.\nReference existing bar_chart.py for 1/8 block precision rendering technique.\nReference existing histogram.py for auto-chunking pattern if comparison charts have many queries.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": [], "description": "Enhance the existing ASCII charting module (benchbox/core/visualization/ascii/) with new chart\ntypes and improved comparison visualizations. The current implementation has 6 chart types across\n~2,255 lines and is well-architected with terminal capability detection, ANSI color support, and\nUnicode fallbacks.\n\nKey enhancements needed:\n1. Paired comparison bar chart - side-by-side bars (e.g., SQL vs DataFrame) with inline percentage\n change annotations, regression/improvement color coding, and scale notes\n2. Diverging bar chart - centered-zero horizontal bars showing regression/improvement distribution,\n sorted by magnitude, with overflow arrows for outliers\n3. Summary box - bordered panel with key aggregate metrics (geo mean, total time, improved/stable/\n regressed counts, worst/best queries)\n4. Inline percentage annotations - \"+23%\", \"-57%\", \"\u2190 !\" markers alongside bars\n5. Intelligent truncation - handle outliers (e.g., Q21 +726%) by truncating bars with indicators\n rather than compressing the entire scale\n\nLibrary evaluation conclusion: Keep custom implementation. Evaluated 10+ terminal charting libraries\n(plotext, plotext-plus, termgraph, plotille, uniplot, rich, textual, asciichartpy, drawille,\nbashplotlib). None can replace the existing module without significant adaptation. Key reasons:\n- BenchBox needs render() -> str pattern; most libraries print to stdout\n- Benchmark-specific features (Pareto frontiers, best/worst markers, grouped multi-platform bars,\n auto-chunking) don't exist in any library\n- Sub-character bar precision (1/8 Unicode blocks) exceeds all library implementations\n- 4-tier terminal color degradation (truecolor -> 256 -> 16 -> none) with NO_COLOR support is\n more robust than any library\n- Zero dependency advantage is significant\nClosest competitor was plotext-plus (7/10 fit) but still needs wrappers for string output,\nbenchmark annotations, and regression highlighting. Full evaluation documented in this item.", "id": "enhance-ascii-charting", "preserves": ["4-tier color degradation chain (truecolor -> 256 -> 16 -> none) must work for new chart types", "ASCIIChartBase interface and render() -> str contract must not change", "ASCIIChartOptions(use_color=False) must produce clean output without ANSI codes for new charts", "Existing 6 ASCII chart types (bar, histogram, heatmap, box, line, scatter) must remain fully functional", "Factory functions from_bar_data(), from_query_latency_data(), etc. must remain backward-compatible", "MCP generate_chart tool must continue to work for existing chart_type values", "Unicode -> ASCII fallback must work for all new chart types"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/*.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "tests/test_ascii_charts.py"}], "state": "done", "title": "Enhance and extend ASCII charting with new chart types and comparison visualizations", "verifications": [{"command": "uv run -- python -m pytest tests/ -k 'ascii' -q", "description": "All existing and new ASCII chart tests pass", "expected": "all tests pass, 0 failures", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'visualization' -q", "description": "No regressions in visualization tests", "expected": "all tests pass, 0 failures", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check benchbox/core/visualization/ascii/", "description": "No lint errors in new or modified files", "expected": "All checks passed or no output (clean)", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.visualization.ascii import ASCIIComparisonBar, ASCIIDivergingBar, ASCIISummaryBox; print('imports ok')\"", "description": "New classes are properly exported from ascii module", "expected": "imports ok", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 4}], "work": [], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "change timing metric filters (measurement-only) as part of tagging cleanup", "instead": "(not specified)", "why": "this TODO is metadata-contract hardening, not metric-policy redesign"}, {"dont": "reinterpret run_type as execution mode (`power|throughput|maintenance`)", "instead": "keep run_type as per-row execution role", "why": "run mode is already carried by `test_execution_type`"}, {"dont": "remove fallback inference in query normalization immediately", "instead": "retain compatibility while enforcing explicit tagging for new producer output", "why": "historical artifacts may lack run_type"}], "approach": "1) Define and document canonical run_type vocabulary for query rows and\nsynthetic rows.\n2) Add explicit producer-side stamping in SQL standard/throughput/maintenance\npaths so run_type is present before normalization.\n3) Propagate run_type into enhanced QueryExecution phase objects.\n4) Keep fallback inference as compatibility-only safety net (not primary path).\n5) Add contract tests that fail if any producer emits query rows without run_type.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-02-18T00:00:00Z", "deferrals": [], "deps": [], "description": "Query result rows currently rely on fallback inference for `run_type` in several\npaths (especially SQL `standard` execution), while other paths set run_type\nexplicitly (`warmup`/`measurement`, and DataFrame metadata/summary rows).\n\nThis creates ambiguity and repeated fallback logic in normalization/schema code,\nand can cause inconsistent downstream analytics semantics when producer paths\nomit run_type.\n\nThis TODO defines canonical run_type semantics and ensures all producer paths\nexplicitly stamp run_type before result building/export. It also aligns\nenhanced execution-phase objects and adds contract tests so missing run_type\ncannot regress silently.", "id": "enforce-explicit-query-run-type-across-executions", "preserves": ["Backward compatibility for loading historical results without explicit run_type remains intact via normalization fallback", "DataFrame metadata/summary synthetic rows keep their current run_type values (`metadata`/`summary`)", "Existing power/throughput behavior where warmup rows are excluded from primary timing metrics remains unchanged", "`test_execution_type` continues to represent run mode (`standard|power|throughput|maintenance|combined`) and is NOT replaced by run_type"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/query_normalizer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/adapter.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/mcp/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Enforce explicit query run_type tagging across SQL and DataFrame execution paths", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms -k 'run_type and (standard or throughput or maintenance or power)' -q", "description": "SQL adapter paths stamp run_type explicitly across execution types", "expected": "selected tests pass; run_type present assertions succeed", "item_id": "enforce-explicit-query-run-type-across-executions", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/results -k 'run_type or query_normalizer or schema' -q", "description": "Result-layer contract tests enforce explicit run_type and compatibility fallback", "expected": "selected tests pass; no missing run_type regressions", "item_id": "enforce-explicit-query-run-type-across-executions", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/mcp/test_mcp_results_and_analytics.py -k RunBenchmark -q", "description": "MCP and result export behavior remains stable", "expected": "selected run_benchmark MCP tests pass", "item_id": "enforce-explicit-query-run-type-across-executions", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define canonical run_type semantics and allowed values in a shared results-layer location", "wid": "w1"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add SQL producer-side run_type stamping for standard execution outputs", "wid": "w2"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Ensure throughput/maintenance and fallback combined paths also emit explicit run_type for query rows", "wid": "w3"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Propagate run_type into `_create_standard_execution_phase()` QueryExecution objects", "wid": "w4"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retain normalization/schema run_type fallback as compatibility-only path with clarifying comments", "wid": "w5"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add platform adapter unit tests asserting run_type presence across execution types", "wid": "w6"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add results-layer contract tests for explicit run_type and fallback compatibility", "wid": "w7"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add MCP/result export regression tests to verify run_type continuity in payloads", "wid": "w8"}, {"evidence": null, "item_id": "enforce-explicit-query-run-type-across-executions", "needs": ["w8"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run focused verification suite and document any residual edge cases", "wid": "w9"}], "worktree": "core-functionality"} +{"anti_patterns": [{"dont": "add a second pytest_collection_modifyitems function to conftest.py", "instead": "(not specified)", "why": "Python silently overrides the first; extend the existing hook body instead"}, {"dont": "add new markers to tests without first checking for speed-lane conflicts", "instead": "(not specified)", "why": "the hook catches it at collection time but a failing CI run wastes 10+ minutes before you see the error; check locally with --collect-only first"}, {"dont": "change test logic while fixing markers", "instead": "(not specified)", "why": "this is a metadata-only cleanup and any logic change risks introducing regressions that get blamed on the marker fix, making it harder to revert cleanly"}, {"dont": "pipe the lint-markers pytest invocation into tail or any filter", "instead": "(not specified)", "why": "the exit code must come from pytest directly, otherwise CI wiring silently passes on failure"}, {"dont": "resolve conflicts by removing resource_heavy or live_integration from tests that genuinely need those markers", "instead": "(not specified)", "why": "those exist to route tests to the right runner; remove the speed-lane marker instead"}, {"dont": "use BENCHBOX_SKIP_MARKER_CHECK=1 in CI", "instead": "(not specified)", "why": "it defeats the entire purpose"}], "approach": "Work units are sequenced: extend the existing pytest_collection_modifyitems hook\nin warning mode first (w1), placing conflict detection BEFORE the deselection\nslice so all 43 violations are visible regardless of -m filter. CI stays green\nwhile violations are fixed (w2-w13 can run in any order or in parallel - one\nfile per commit). Promote to error mode (w14) only after all violations are\ngone, confirmed by pytest --collect-only returning zero marker-conflict warnings.\nFinally add the make target (w15) to give pre-commit a fast entry point.\n", "blocked_reason": null, "category": "Testing & Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-01T00:00:00Z", "completed_pr": null, "created_at": "2026-04-01T00:00:00Z", "deferrals": [], "deps": [], "description": "43 tests across 12 files currently carry mutually exclusive marker combinations\nthat undermine the fast/medium/slow speed-lane semantics and cause selection\nambiguity for both humans and automation.\n\nDiscovered violations (as of 2026-04-01):\n fast + slow : 13 tests in 5 files\n medium + slow : 26 tests in 6 files\n fast + resource_heavy: 4 tests in 1 file\n\nConflict rules to enforce:\n Speed-lane mutual exclusion: a test may carry at most one of {fast, medium, slow}\n Exclusion-set incompatibility: fast is incompatible with {slow, stress, resource_heavy, live_integration}\n\nThe fix has two parts:\n1. Conflict detection merged into the existing pytest_collection_modifyitems hook in\n tests/conftest.py, running BEFORE the deselection slice so all collected items\n (including those about to be deselected by -m filters) are inspected. Starts in\n warning mode, promoted to error mode once all violations are remediated.\n2. Remediation of all 43 existing violations - each test gets a single correct\n speed marker based on what it actually does.\n\nThis makes the pytest_collection_modifyitems exclusion hook (added in\nd84c897a8) a belt-and-suspenders safety net rather than the primary guard.", "id": "enforce-no-conflicting-test-markers", "preserves": ["BENCHBOX_SKIP_MARKER_CHECK=1 suppresses validation errors after w14 promotion (escape hatch preserved); during w1-w13 migration, BENCHBOX_ENFORCE_MARKER_CHECK=1 opts into early error mode", "No test logic changes - only marker annotations modified", "make test-fast and uv run -- python -m pytest -m fast -q collect identical node ID sets (not just equal counts) - enforced by existing collection hook in d84c897a8", "make test-slow, make test-stress, make test-all still collect their expected sets - hook must not block explicit marker opt-in"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/lint.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "tests/conftest.py (validation hook)"}, {"kind": "only_modify", "path_glob": "tests/integration/core/data_organization/test_presorted_generation.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_nyctaxi_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_phase4_validation_integration.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tpch_skew_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/integration/test_tsbs_devops_duckdb.py"}, {"kind": "only_modify", "path_glob": "tests/test_tpcdi.py"}, {"kind": "only_modify", "path_glob": "tests/test_tpcds_direct.py"}, {"kind": "only_modify", "path_glob": "tests/unit/benchmarks/test_write_primitives_generator.py"}, {"kind": "only_modify", "path_glob": "tests/unit/monitoring/test_profiler.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_databricks_liquid_clustering.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_release_infrastructure.py"}, {"kind": "only_modify", "path_glob": "tests/unit/test_tpcdi_phase1.py"}], "state": "done", "title": "Enforce no conflicting test markers: detection hook + remediation of 43 existing violations", "verifications": [{"command": "uv run -- python -m pytest --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -m \"fast and slow\" -p no:warnings 2>&1 | tail -3", "description": "No fast+slow conflicts after w2-w6", "expected": "no tests collected", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -m \"medium and slow\" -p no:warnings 2>&1 | tail -3", "description": "No medium+slow conflicts after w7-w12", "expected": "no tests collected", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -m \"fast and resource_heavy\" -p no:warnings 2>&1 | tail -3", "description": "No fast+resource_heavy conflicts after w13", "expected": "no tests collected", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest --collect-only -q -p no:warnings 2>&1 | grep -c 'Marker conflict'", "description": "After w14: zero marker-conflict warnings in clean collect-only", "expected": "0", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 4}, {"command": "diff <(uv run -- python -m pytest -m fast --collect-only -q --override-ini=\"addopts=-p _benchbox_pytest_xdist_safety --strict-markers --strict-config -p no:cov\" -p no:warnings 2>&1 | grep '::' | sort) <(make test-fast ARGS='--collect-only -q -p no:warnings' 2>&1 | grep '::' | sort)", "description": "make test-fast node ID set is identical before and after marker remediation", "expected": "empty diff (no output)", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 5}, {"command": "make lint-markers", "description": "make lint-markers exits 0", "expected": "exit code 0", "item_id": "enforce-no-conflicting-test-markers", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Merge conflict validation into pytest_collection_modifyitems (warning mode)", "wid": "w1"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_tpcdi.py (2 tests)", "wid": "w10"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_release_infrastructure.py (1 test)", "wid": "w11"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_tpcdi_phase1.py (1 test)", "wid": "w12"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+resource_heavy: test_presorted_generation.py (4 tests)", "wid": "w13"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w10", "w11", "w12", "w13", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Promote validation hook from warning to error mode", "wid": "w14"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w14"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add make lint-markers target for pre-commit and CI", "wid": "w15"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_nyctaxi_duckdb.py (3 tests)", "wid": "w2"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_tsbs_devops_duckdb.py (5 tests)", "wid": "w3"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_tpcds_direct.py (1 test)", "wid": "w4"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_profiler.py (3 tests)", "wid": "w5"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix fast+slow: test_databricks_liquid_clustering.py (1 test)", "wid": "w6"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_write_primitives_generator.py (18 tests)", "wid": "w7"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_tpch_skew_duckdb.py (4 tests)", "wid": "w8"}, {"evidence": null, "item_id": "enforce-no-conflicting-test-markers", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix medium+slow: test_phase4_validation_integration.py (2 tests)", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "adopt an external charting library", "instead": "(not specified)", "why": "the render()->str pattern, benchmark-specific features, and sub-character precision cannot be replicated; custom implementation evaluated as superior to all 10+ candidates"}, {"dont": "compress the entire bar scale for outliers", "instead": "(not specified)", "why": "it makes most bars unreadably small; truncate outlier bars instead with overflow indicators"}, {"dont": "hardcode terminal width", "instead": "(not specified)", "why": "base.py already handles terminal detection; use self.width from ASCIIChartBase"}, {"dont": "print to stdout", "instead": "(not specified)", "why": "all ASCII charts must return strings for MCP tool responses and CLI piping"}, {"dont": "skip Unicode fallback", "instead": "(not specified)", "why": "ASCII_BLOCK_CHARS and ASCII_BOX_CHARS exist for terminals without Unicode support"}], "approach": "Follow existing inheritance pattern: new classes inherit from ASCIIChartBase in ascii/ module.\nUse existing BLOCK_CHARS, INTENSITY_CHARS, BOX_CHARS from base.py - do not duplicate.\nFollow factory function pattern (from_*_data) for data conversion.\nMirror _render_single_ascii_chart() dispatch pattern in visualization.py for new types.\nReference existing bar_chart.py for 1/8 block precision rendering technique.\nReference existing histogram.py for auto-chunking pattern if comparison charts have many queries.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-02-06T00:00:00Z", "completed_pr": null, "created_at": "2026-02-06T00:00:00Z", "deferrals": [], "deps": [], "description": "Enhance the existing ASCII charting module (benchbox/core/visualization/ascii/) with new chart\ntypes and improved comparison visualizations. The current implementation has 6 chart types across\n~2,255 lines and is well-architected with terminal capability detection, ANSI color support, and\nUnicode fallbacks.\n\nKey enhancements needed:\n1. Paired comparison bar chart - side-by-side bars (e.g., SQL vs DataFrame) with inline percentage\n change annotations, regression/improvement color coding, and scale notes\n2. Diverging bar chart - centered-zero horizontal bars showing regression/improvement distribution,\n sorted by magnitude, with overflow arrows for outliers\n3. Summary box - bordered panel with key aggregate metrics (geo mean, total time, improved/stable/\n regressed counts, worst/best queries)\n4. Inline percentage annotations - \"+23%\", \"-57%\", \"\u2190 !\" markers alongside bars\n5. Intelligent truncation - handle outliers (e.g., Q21 +726%) by truncating bars with indicators\n rather than compressing the entire scale\n\nLibrary evaluation conclusion: Keep custom implementation. Evaluated 10+ terminal charting libraries\n(plotext, plotext-plus, termgraph, plotille, uniplot, rich, textual, asciichartpy, drawille,\nbashplotlib). None can replace the existing module without significant adaptation. Key reasons:\n- BenchBox needs render() -> str pattern; most libraries print to stdout\n- Benchmark-specific features (Pareto frontiers, best/worst markers, grouped multi-platform bars,\n auto-chunking) don't exist in any library\n- Sub-character bar precision (1/8 Unicode blocks) exceeds all library implementations\n- 4-tier terminal color degradation (truecolor -> 256 -> 16 -> none) with NO_COLOR support is\n more robust than any library\n- Zero dependency advantage is significant\nClosest competitor was plotext-plus (7/10 fit) but still needs wrappers for string output,\nbenchmark annotations, and regression highlighting. Full evaluation documented in this item.", "id": "enhance-ascii-charting", "preserves": ["4-tier color degradation chain (truecolor -> 256 -> 16 -> none) must work for new chart types", "ASCIIChartBase interface and render() -> str contract must not change", "ASCIIChartOptions(use_color=False) must produce clean output without ANSI codes for new charts", "Existing 6 ASCII chart types (bar, histogram, heatmap, box, line, scatter) must remain fully functional", "Factory functions from_bar_data(), from_query_latency_data(), etc. must remain backward-compatible", "MCP generate_chart tool must continue to work for existing chart_type values", "Unicode -> ASCII fallback must work for all new chart types"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/*.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/ascii/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/visualization/exporters.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/visualization.py"}, {"kind": "only_modify", "path_glob": "tests/test_ascii_charts.py"}], "state": "done", "title": "Enhance and extend ASCII charting with new chart types and comparison visualizations", "verifications": [{"command": "uv run -- python -m pytest tests/ -k 'ascii' -q", "description": "All existing and new ASCII chart tests pass", "expected": "all tests pass, 0 failures", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/ -k 'visualization' -q", "description": "No regressions in visualization tests", "expected": "all tests pass, 0 failures", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run ruff check benchbox/core/visualization/ascii/", "description": "No lint errors in new or modified files", "expected": "All checks passed or no output (clean)", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -c \"from benchbox.core.visualization.ascii import ASCIIComparisonBar, ASCIIDivergingBar, ASCIISummaryBox; print('imports ok')\"", "description": "New classes are properly exported from ascii module", "expected": "imports ok", "item_id": "enhance-ascii-charting", "last_result": null, "last_run": null, "seq": 4}], "work": [], "worktree": "core-functionality"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "\"Technical Debt & Refactoring\"", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Completed (commit 3c5fa74)", "id": "enhanced-cli-preflight-data-validation-for-tpc-ds-generation-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Enhanced CLI Preflight Data Validation for TPC-DS Generation - COMPLETED \u2705", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Supporting Infrastructure", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Add CPU/memory/disk/network monitoring during benchmark execution with\nbottleneck identification and resource utilization charts.", "id": "enhanced-resource-monitoring-profiling", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Enhanced Resource Monitoring & Profiling", "verifications": [], "work": [], "worktree": "infrastructure-monitoring"} -{"anti_patterns": [{"INSTEAD": "keep runtime checks local/optional.", "dont": "broaden runtime parity tests to live cloud platforms in this item", "why": "credentials and live_integration approvals are separate concerns"}, {"INSTEAD": "use semantics-preserving SQL or skip_on instead.", "dont": "keep related-but-not-equivalent fallbacks as comparable variants", "why": "they produce misleading benchmark signals"}, {"INSTEAD": "validate declared result_contract metadata.", "dont": "rely only on column-name equality", "why": "nested JSON/array/struct shape can diverge behind identical aliases"}, {"INSTEAD": "preserve the current bypass behavior.", "dont": "rewrite variants through SQLGlot", "why": "existing platform variants are authored as final target SQL"}], "approach": "Start with the shared query catalog loader because read_primitives uses the\ngeneric query catalog path. Add result contract parsing in a backward-compatible\nway, then enforce the stricter contract only for read_primitives queries with\nvariants or high-risk nested/semi-structured outputs.\n\nUse the existing variant tests in tests/unit/core/read_primitives as the local\npattern. Keep the first implementation focused on read_primitives; do not\ngeneralize the policy to metadata_primitives or ai_primitives unless needed for\nshared loader compatibility.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-29T00:00:00Z", "completed_pr": null, "created_at": "2026-04-29T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "ensure-read-primitives-variant-comparability", "id": 153, "reason": "The first implementation should fix benchmark correctness at the catalog/test layer; CLI/result-report integration can follow once the contract shape stabilizes.", "resolution": "promoted", "resolved_item": "expose-query-comparability-metadata-in-cli-output-and-persis", "resolved_reason": null, "summary": "Expose query comparability metadata in CLI output and persisted result artifacts"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ensure-read-primitives-variant-comparability", "id": 154, "reason": "The first pass targets the highest-risk families. Follow-up coverage should include array_unnest (clickhouse, bigquery), array_agg_simple (clickhouse), struct_construction (clickhouse), list_transform (clickhouse), and the duckdb/clickhouse paths of map_construction so retained variants do not drift silently.", "resolution": "promoted", "resolved_item": "expand-runtime-parity-coverage-for-retained-read-primitives", "resolved_reason": null, "summary": "Expand runtime parity coverage for retained read_primitives variants not covered by the first local parity suite"}], "deps": [], "description": "Previous review found that `read_primitives` platform variants can diverge enough\nto produce misleading cross-platform benchmark signals. Q75/json_aggregates exposed\nthe issue: matching column names and successful execution are not enough when the\nnested output shape or measured capability differs.\n\nApply the policy that every active platform variant must be meaningfully comparable\nto the base query. If a platform can only express a related-but-not-equivalent\noperation, skip that query on that platform rather than keeping it in cross-platform\nbenchmark scoring.", "id": "ensure-read-primitives-variant-comparability", "preserves": ["Catalog variants remain final target SQL and continue to bypass SQLGlot retranslation in ReadPrimitivesBenchmark.get_queries().", "Existing skip_on behavior continues to exclude unsupported queries without counting them as failures.", "Runnable but non-equivalent variants are skipped rather than retained as benchmark-comparable queries."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/ai_primitives/catalog/queries.yaml"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/metadata_primitives/catalog/queries.yaml"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}], "state": "done", "title": "Ensure read_primitives query variants are meaningfully comparable", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/read_primitives/test_catalog_loader.py tests/unit/core/read_primitives/test_query_manager_variants.py tests/unit/core/read_primitives/test_benchmark_variants.py -q", "description": "Catalog loader and existing variant behavior still pass", "expected": "all tests pass", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/read_primitives/test_read_primitives_variant_contracts.py -q", "description": "New static comparability contracts pass", "expected": "all tests pass", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/integration/validation/test_read_primitives_variant_parity.py -q", "description": "Focused runtime parity tests pass or skip cleanly for missing optional engines", "expected": "tests pass; optional engines may skip when unavailable", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after this item is created", "expected": "no cycles or dangling dependencies", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add result_contract metadata support to the read_primitives catalog model and loader", "wid": "w1"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add static comparability linting for base/variant output columns, nested shape, row identity, and measured capability", "wid": "w2"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or skip scalar and capability-divergent variants: fulltext, window_moving_frame, statistical_percentiles, timeseries_trend_analysis, and any_value", "wid": "w3"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or skip semi-structured variants with nested-shape risk: JSON, arrays, structs, maps, and lambda/list operations", "wid": "w4"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add focused runtime parity tests for risky read_primitives families against DuckDB reference at SF=0.01", "wid": "w5"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update read_primitives catalog docs and skip reference to require comparable variants or skip_on", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "keep the widening unconditional", "why": "a value oracle that equates NaN with NULL or ignores trailing spaces by default cannot be trusted to catch those bug classes."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The shared result comparator silently widens equivalence in two engine-justified\nbut globally-applied ways, each of which can mask a real cross-surface value\ndivergence:\n - validation.py:718 (_numeric_values_equal) treats float('nan') and None as\n both \"missing\" and reports two missing values as EQUAL. Combined with\n _normalize_value mapping NaN->None, a DataFrame surface that produces a\n spurious NaN (e.g. 0/0 from a division bug) compares EQUAL to a SQL NULL.\n - validation.py:693 strips leading/trailing whitespace unconditionally\n (val1.strip() == val2.strip()), so \"foo \" vs \"foo\" is declared equal - masking\n a real transcription difference.\nBoth are justified for specific engine decode paths (e.g. ClickHouse Arrow/pandas\nNULL decode, CHAR padding) but are applied engine-agnostically with no flag to\ndisable, so they widen equivalence for every gate.\n\nMake both behaviors explicit comparator flags (e.g. treat_nan_as_null,\nstrip_strings) that default to STRICT, enabled only where an engine genuinely\nneeds them and documented at that call site.", "id": "equivalence-comparator-value-widening-opt-in", "preserves": ["Engine decode paths that genuinely need NaN-as-null or CHAR-padding tolerance still pass, via the explicit opt-in flag at that site.", "Float relative-tolerance (1e-10) comparison semantics are unchanged."], "prior_art": [{"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py / dataframe_surface.py (_normalize_value)', 'note': 'Where NaN->None normalization happens; enable the widening flags only where a specific engine decode requires it.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py:693', 'note': 'Unconditional string .strip() compare - add a strip_strings flag, default False.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py:718', 'note': '_numeric_values_equal NaN/None handling - add a treat_nan_as_null flag, default False (strict).'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/equivalence/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Make equivalence-comparator value-widening (NaN==NULL, whitespace strip) opt-in instead of unconditional", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/equivalence -q && make ssb-cross-surface-equivalence-report", "description": "Strict default catches widened divergences; enforced gates stay green", "expected": "new strict-default tests pass; all enforced gates still 0 divergent", "item_id": "equivalence-comparator-value-widening-opt-in", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "equivalence-comparator-value-widening-opt-in", "needs": [], "notes": "Add treat_nan_as_null and strip_strings flags (additive, default False so the\nstrict behavior is the default). Thread them through the comparator entry\npoints. Enable them ONLY at the specific engine decode sites that genuinely\nneed them, with an inline comment justifying each. Keep TPC-Havoc and all\nenforced gates green (verify none silently depended on the widening - if one\ndoes, that is a divergence to surface, not to preserve).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate NaN==NULL and whitespace-strip behind comparator flags, default strict", "wid": "w1"}, {"evidence": null, "item_id": "equivalence-comparator-value-widening-opt-in", "needs": ["w1"], "notes": "Unit tests asserting, under the strict default: a DataFrame-side 0/0->NaN vs\nSQL NULL is CAUGHT, and \"foo \" vs \"foo\" is CAUGHT. And that the opt-in flags,\nwhere set, still accept the engine-justified case.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test that the strict default catches the masked divergences", "wid": "w2"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add review_by enforcement as a hard FAIL by default without checking it would not immediately break an existing gate", "why": "a surprise new failure mode on merge day for a benchmark whose owner is not in the room is worse than a warning."}, {"INSTEAD": "(not specified)", "dont": "conflate review_by-past-due with the existing resolved-divergence stale-baseline failure", "why": "they are different signals (one means 'the divergence is GONE, prune the entry'; the other means 'the divergence may still be valid but a human should re-look') and must not share one code path or one message format that a reader could confuse."}, {"INSTEAD": "one keys on backend+query, the other on query+variant).", "dont": "fork _report per module to add this", "why": "both modules already have their own _report; extend each in place, do not introduce a third shared helper unless the two are trivially unifiable (they are not"}, {"INSTEAD": "keep it opt-in.", "dont": "make review_by mandatory on every waiver", "why": "most waivers (e.g. the h2odb Q9 DECIMAL-scale residue, an inherent engine behavior) may never need re-review and forcing a date on them is busywork, not signal"}], "approach": "Decide the carrier shape and warn-vs-fail policy first (w1), since it\ndetermines whether cross_surface.py's known_divergences Union type or\nequivalence.py's dict[str, str] baselines need a type change vs a\nside-table. Implement the check in each module's _report (w2, w3), document\nthe policy in both module docstrings (w4), and retrofit one worked example\nper module with a test proving the check fires (w5).\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-03T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "equivalence-waiver-expiry-policy", "id": 312, "reason": "Out of scope; this item only adds the detectable signal (a warning/failure at report time). Automating a proactive nudge is a separate CI/scheduling concern.", "resolution": "promoted", "resolved_item": "automatically-opening-an-issue-pr-or-paging-an-owner-when-a", "resolved_reason": null, "summary": "Automatically opening an issue/PR or paging an owner when a review_by date passes."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "equivalence-waiver-expiry-policy", "id": 313, "reason": "review_by is optional by design (per the task framing: 'Keep it lightweight'); mandating it on every existing entry is a larger migration this item does not need to force.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: review_by is optional by design per the parent item's own framing; retroactively mandating it on all waivers was deliberately rejected.", "summary": "Requiring review_by on every existing waiver retroactively."}], "deps": [], "description": "SOUNDNESS-CRITICAL COMPARATOR SURFACE: this TODO's work touches\nbenchbox/core/equivalence/cross_surface.py (covered by .github/CODEOWNERS as\n`benchbox/core/equivalence/** @joeharris76`) and\nbenchbox/core/tpchavoc/equivalence.py (adjacent to, and sharing the same\nwaiver-baseline discipline as, benchbox/core/tpchavoc/validation.py, which\nIS covered as `benchbox/core/**/validation.py @joeharris76`; note that\ntpchavoc/equivalence.py itself is not literally matched by an existing\nCODEOWNERS glob - verified against the current 3-line .github/CODEOWNERS -\nbut should be treated as soundness-critical and reviewed by the code owner\nregardless). Its PR will require code-owner review before merge.\n\nVerified by reading both modules on this branch: no known-divergence waiver\nentry, in either file, carries any date or expiry field.\n - cross_surface.py's `known_divergences: dict[str, str | ClassifiedDivergence]`\n (CrossSurfaceGate field, line 371) holds bare strings (e.g.\n `_READ_PRIMITIVES_KNOWN_DIVERGENCES`, cross_surface.py:859-885) and\n `ClassifiedDivergence` instances (cross_surface.py:338-357, e.g.\n `_H2ODB_KNOWN_DIVERGENCES` at 813-816, `_CLICKBENCH_TIE_AMBIGUOUS` at\n 619-624+) - neither the `str` nor the `ClassifiedDivergence` dataclass\n (fields: `reason`, `accepts`, `requires_live_divergence`) has a date field.\n - equivalence.py's four engine-keyed baselines (`KNOWN_DIVERGENCES`,\n `POSTGRES_KNOWN_DIVERGENCES`, `DATAFUSION_KNOWN_DIVERGENCES`,\n `CLICKHOUSE_KNOWN_DIVERGENCES`, all `dict[str, str]`) are plain\n key->reason-string maps with no date field either.\n\nThe only existing staleness signal in either module is \"stale-baseline\nfailure\": a waiver whose divergence has stopped reproducing is caught (in\ncross_surface.py's `_report`, the `resolved` check at line ~1188-1192; in\nequivalence.py's shared `_report`, a `resolved` PRINT only - see the sibling\nTODO tpchavoc-duckdb-gate-vacuity-guard w3 for that gap). But a waiver whose\ndivergence is STILL LIVE (e.g. an \"irreducible engine-semantic difference\" or\nan \"approximate by construction\" classification) never gets re-examined - it\ncan sit correct-when-written but stale-in-spirit indefinitely: the engine\nversion that produced the documented behavior may have changed, the\nbenchmark's data/queries may have evolved, or the original rationale may no\nlonger hold, and nothing forces a human to re-look.\n\nThis TODO adds a lightweight, OPTIONAL `review_by:` date to waiver entries\n(both the bare-string form, by upgrading it to a small structured wrapper or\na side-table, and the `ClassifiedDivergence` form, as a new optional field),\nand a warning-or-failure when a `review_by` date has passed, so long-lived\nwaivers are periodically forced back into review even when their divergence\nnever disappears on its own.", "id": "equivalence-waiver-expiry-policy", "preserves": ["The existing stale-baseline (resolved-divergence) failure behavior in both modules' _report functions is unchanged by this addition - review_by is a new, independent check, not a replacement.", "cross_surface.py's GATES stay green and equivalence.py's tpchavoc-equivalence-report* targets stay green after this lands (no waiver in either module should already have an accidentally-past-due review_by introduced by this change).", "review_by is OPTIONAL - every existing waiver entry with no review_by continues to behave exactly as today (no forced migration, no new failure for entries that omit it)."], "prior_art": [{"concept": "benchbox/core/equivalence/cross_surface.py:1173-1196 _report - where a new review_by warning/failure check is added, alongside the existing new/resolved/missing_backends/unclassified_empty checks.", "decision": "reuse", "path": "benchbox/core/equivalence/cross_surface.py:1173-1196"}, {"concept": "benchbox/core/equivalence/cross_surface.py:338-357 ClassifiedDivergence - the dataclass to extend with an optional review_by field (or a documented decision not to, if the bare-string form is chosen as the carrier instead).", "decision": "extend", "path": "benchbox/core/equivalence/cross_surface.py:338-357"}, {"concept": "benchbox/core/equivalence/cross_surface.py:371 CrossSurfaceGate.known_divergences: dict[str, str | ClassifiedDivergence] - the Union type that must accommodate whatever wrapper form w1 chooses for bare-string entries with a review_by date.", "decision": "reuse", "path": "benchbox/core/equivalence/cross_surface.py:371"}, {"concept": "benchbox/core/tpchavoc/equivalence.py:434-470 _report (KNOWN_DIVERGENCES/POSTGRES_KNOWN_DIVERGENCES/DATAFUSION_KNOWN_DIVERGENCES/CLICKHOUSE_KNOWN_DIVERGENCES, all dict[str, str]) - the second _report that needs the same review_by check, in a shape compatible with its plain string-valued dicts.", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py:434-470"}, {"concept": "cross-surface-baseline-update-flag.yaml (planning) - a closely related, already-filed TODO for a --update-baseline flag to DROP resolved entries; this item is complementary (forcing review of LIVE entries) and should reuse its w0's re-confirm-the-live-file-and-lines discipline, not duplicate its writer.", "decision": "reuse", "path": "cross-surface-baseline-update-flag.yaml"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpchavoc/validation.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/equivalence/cross_surface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/equivalence.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/unit/equivalence/ (actually tests/unit/core/equivalence/ - see verification deviation note)"}], "state": "done", "title": "Add an optional review_by expiry date to known-divergence waivers so live waivers are periodically forced back into review", "verifications": [{"command": "uv run -- python -m pytest tests/unit/equivalence/ -k review_by -v", "description": "A synthetic past-due review_by waiver triggers the new signal in cross_surface.py's report", "expected": "the seeded past-due waiver produces the documented warning (or failure, per w1) line; a future-dated or absent review_by does not", "item_id": "equivalence-waiver-expiry-policy", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpchavoc/ -k review_by -v", "description": "A synthetic past-due review_by waiver triggers the new signal in equivalence.py's report", "expected": "same behavior, ported to the TPC-Havoc engine-keyed baselines", "item_id": "equivalence-waiver-expiry-policy", "last_result": null, "last_run": null, "seq": 2}, {"command": "make ssb-cross-surface-equivalence-report && make read-primitives-cross-surface-equivalence-report && make tpchavoc-equivalence-report", "description": "Existing gates stay green after the change lands", "expected": "all three exit 0, unchanged from before this TODO", "item_id": "equivalence-waiver-expiry-policy", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": [], "notes": "DECISION, matching the open_questions defaults exactly (directed as the\npolicy for this batch and independently confirmed sound):\n (a) cross_surface.py: option (i) - added `review_by: date | None = None`\n as a third field on `ClassifiedDivergence`. Bare-`str` entries\n CANNOT carry a review_by (documented explicitly in both the\n dataclass docstring and the module's new \"waiver review policy\"\n section) - no bare-string waiver was converted to\n ClassifiedDivergence to add one. Verified this does not complicate\n the sibling read-primitives-waivers-classified-divergence TODO:\n `_READ_PRIMITIVES_KNOWN_DIVERGENCES` (read_primitives gate) still\n has its existing bare-string entries untouched; nothing in this\n change forces or blocks that sibling's future conversion.\n (b) equivalence.py (tpchavoc): option (ii) - a side-table\n `dict[str, date]` per baseline, keyed identically to the baseline\n it annotates (e.g. `CLICKHOUSE_KNOWN_DIVERGENCES_REVIEW_BY`), passed\n into the shared `_report` as a new optional `review_by` parameter.\n No existing `dict[str, str]` baseline's value type changed.\n (c) WARN, not fail - matches the default exactly. A past-due\n `review_by` prints a \"WAIVER REVIEW DUE - ...\" line in both\n modules' `_report`s but never changes the return code, kept\n deliberately distinct in wording and code path from the existing\n `resolved` GATE FAILURE (per the anti_patterns).\nPrior notes (superseded by the decision above, kept for context):\nTwo open design questions to resolve before implementing:\n (a) Carrier shape for cross_surface.py's bare-`str` waivers (the majority\n of entries - see _READ_PRIMITIVES_KNOWN_DIVERGENCES, and any others\n not yet converted to ClassifiedDivergence by the sibling TODO\n read-primitives-waivers-classified-divergence). Options: (i) add\n `review_by` as a THIRD optional field on `ClassifiedDivergence` and\n require every review_by-bearing waiver to at least be wrapped in\n `ClassifiedDivergence(reason=..., accepts=lambda _: True, review_by=...)`\n even if it has no real predicate yet (accepts-everything is\n explicit, not implicit, in this form) - this reuses one type instead\n of two; or (ii) introduce a small side-table\n `dict[str, date]` (e.g. `_REVIEW_BY: dict[str, date]`) keyed by the\n same divergence key, checked independently of the waiver's own\n shape - avoids touching the `known_divergences` Union type at all.\n Prefer (i) if the sibling TODO's ClassifiedDivergence conversion is\n expected to land first (fewer bare strings to carry the concern for\n long-term); prefer (ii) if this item lands independently and should\n not force every bare-string waiver into a dataclass. State and\n justify the choice here.\n (b) Same question for equivalence.py's plain `dict[str, str]` baselines\n (KNOWN_DIVERGENCES et al.) - these have no ClassifiedDivergence\n equivalent at all today, so a side-table (ii)-style\n `dict[str, date]` per baseline, or a small `@dataclass` wrapper\n (`KnownDivergence(reason: str, review_by: date | None = None)`)\n replacing the bare `str` value type, is the realistic choice. Note\n this is a wider blast radius (touches every existing baseline\n entry's type) - if it proves too invasive for zero-value-change\n entries, the side-table is likely the pragmatic default.\n (c) Warn vs fail: decide in this work unit, do not defer. Given the\n module already treats a stale-baseline `resolved` divergence as a\n hard gate FAILURE in cross_surface.py (line ~1191), a past-due\n `review_by` on a STILL-LIVE waiver is a weaker signal (the waiver\n may still be entirely correct) - default to WARN (print prominently,\n do not fail exit code) unless a strong case for FAIL emerges (e.g.\n \"the gate is meant to be a hard forcing function and a warning is\n too easy to ignore\"). Record the decision and rationale.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the review_by carrier for both waiver shapes and decide warn-vs-fail on a past-due date", "wid": "w1"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w1"], "notes": "Added `review_by: date | None = None` to `ClassifiedDivergence`. In\n`_report`, computes `review_due = sorted(key for key, entry in\nknown.items() if isinstance(entry, ClassifiedDivergence) and\nentry.review_by is not None and entry.review_by < date.today())` and, for\neach, prints `\"WAIVER REVIEW DUE - {key}: review_by {entry.review_by} has\npassed - {entry.reason}\"` - a distinct prefix from \"GATE FAILURE\" so a\nreader (or a naive grep for FAILURE) cannot confuse the two signals. The\nreturn statement (`1 if (new or resolved or missing_backends or\nunclassified_empty) else 0`) is UNCHANGED - review_due never enters it,\nper w1(c)'s warn decision.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the review_by field/side-table and wire the warning/failure check into cross_surface.py's _report", "wid": "w2"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w1"], "notes": "Added an optional `review_by: dict[str, date] | None = None` parameter to\nthe shared `_report` (equivalence.py). Computes `review_due = sorted(key\nfor key, due in review_by.items() if key in known and due <\ndate.today())` and prints the same \"WAIVER REVIEW DUE - ...\" line style as\ncross_surface.py (message text intentionally parallel, code NOT shared -\nper the anti_patterns, the two `_report`s stay separate functions).\nVerified live all four report modes after the change: DuckDB hard gate\n(220/220, exit 0), DataFusion sample (206/206, exit 0), ClickHouse sample\n(195/202 + 7 classified, exit 0, no review-due warning since the worked\nexample's date is not yet past), Postgres sample unreachable in this\nenvironment (clean SKIP, exit 0, unaffected - POSTGRES_KNOWN_DIVERGENCES\nis empty so review_by never applies there regardless).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the review_by field/side-table and wire the equivalent check into equivalence.py's shared _report", "wid": "w3"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w2", "w3"], "notes": "Added a \"Waiver review policy\" paragraph to both module docstrings\n(cross_surface.py, after the \"Currently gated\" paragraph; equivalence.py,\nafter the LIMIT-stripping paragraph) stating: an entry MAY carry an\noptional review_by date; a past-due date WARNS (never fails); this is\nindependent of and does not replace the existing stale-baseline\n(`resolved`) failure - one means \"the divergence is gone, delete the\nentry\", the other means \"the divergence may still be valid, re-look\nanyway\". Also added matching explanatory paragraphs to the `ClassifiedDivergence`\ndataclass docstring and both `_report` docstrings.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the policy in both modules' docstrings", "wid": "w4"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w2", "w3", "w4"], "notes": "cross_surface.py: retrofitted `_CLICKBENCH_TIE_AMBIGUOUS` (ClickBench Q18,\nan order-less top-N with NO ORDER BY over ~97k groups - structurally\npermanent, `requires_live_divergence=False` already documents it can never\ntrip the `resolved` check on its own, so review_by is the ONLY forcing\nfunction available for it) with `review_by=date(2027, 1, 3)` (6 months\nfrom created_date 2026-07-03).\nequivalence.py: retrofitted CLICKHOUSE_KNOWN_DIVERGENCES's \"1_v4\"\ndecimal-division-truncation entry (inherent to ClickHouse's Decimal\narithmetic, unlikely to ever resolve) via a new\n`CLICKHOUSE_KNOWN_DIVERGENCES_REVIEW_BY = {\"1_v4\": date(2027, 1, 3)}`\nside-table, wired into `run_clickhouse_sample`'s `_report(...)` call.\nTests added:\n - tests/unit/core/equivalence/test_cross_surface.py:\n test_review_by_past_due_warns_but_does_not_fail_the_gate,\n test_review_by_future_date_does_not_warn,\n test_review_by_absent_by_default_produces_no_warning\n - tests/unit/core/tpchavoc/test_tpchavoc_review_by.py (new file):\n test_review_by_past_due_warns_but_does_not_fail_the_report,\n test_review_by_past_due_warns_on_a_still_reproducing_divergence,\n test_review_by_future_date_does_not_warn,\n test_review_by_absent_by_default_produces_no_warning,\n test_clickhouse_review_by_worked_example_is_not_yet_due (regression\n guard: fails loudly if the worked example's date is ever left to go\n stale unnoticed).\nAll 8 pass; confirmed neither worked-example date (2027-01-03) is\ncurrently past-due (today is 2026-07-03), so no gate/sample newly warns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retrofit review_by onto at least one existing long-lived waiver per module as a worked example, and add a test", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "broaden runtime parity tests to live cloud platforms in this item", "instead": "keep runtime checks local/optional.", "why": "credentials and live_integration approvals are separate concerns"}, {"dont": "keep related-but-not-equivalent fallbacks as comparable variants", "instead": "use semantics-preserving SQL or skip_on instead.", "why": "they produce misleading benchmark signals"}, {"dont": "rely only on column-name equality", "instead": "validate declared result_contract metadata.", "why": "nested JSON/array/struct shape can diverge behind identical aliases"}, {"dont": "rewrite variants through SQLGlot", "instead": "preserve the current bypass behavior.", "why": "existing platform variants are authored as final target SQL"}], "approach": "Start with the shared query catalog loader because read_primitives uses the\ngeneric query catalog path. Add result contract parsing in a backward-compatible\nway, then enforce the stricter contract only for read_primitives queries with\nvariants or high-risk nested/semi-structured outputs.\n\nUse the existing variant tests in tests/unit/core/read_primitives as the local\npattern. Keep the first implementation focused on read_primitives; do not\ngeneralize the policy to metadata_primitives or ai_primitives unless needed for\nshared loader compatibility.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-29T00:00:00Z", "completed_pr": null, "created_at": "2026-04-29T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "ensure-read-primitives-variant-comparability", "id": 153, "reason": "The first implementation should fix benchmark correctness at the catalog/test layer; CLI/result-report integration can follow once the contract shape stabilizes.", "resolution": "promoted", "resolved_item": "expose-query-comparability-metadata-in-cli-output-and-persis", "resolved_reason": null, "summary": "Expose query comparability metadata in CLI output and persisted result artifacts"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "ensure-read-primitives-variant-comparability", "id": 154, "reason": "The first pass targets the highest-risk families. Follow-up coverage should include array_unnest (clickhouse, bigquery), array_agg_simple (clickhouse), struct_construction (clickhouse), list_transform (clickhouse), and the duckdb/clickhouse paths of map_construction so retained variants do not drift silently.", "resolution": "promoted", "resolved_item": "expand-runtime-parity-coverage-for-retained-read-primitives", "resolved_reason": null, "summary": "Expand runtime parity coverage for retained read_primitives variants not covered by the first local parity suite"}], "deps": [], "description": "Previous review found that `read_primitives` platform variants can diverge enough\nto produce misleading cross-platform benchmark signals. Q75/json_aggregates exposed\nthe issue: matching column names and successful execution are not enough when the\nnested output shape or measured capability differs.\n\nApply the policy that every active platform variant must be meaningfully comparable\nto the base query. If a platform can only express a related-but-not-equivalent\noperation, skip that query on that platform rather than keeping it in cross-platform\nbenchmark scoring.", "id": "ensure-read-primitives-variant-comparability", "preserves": ["Catalog variants remain final target SQL and continue to bypass SQLGlot retranslation in ReadPrimitivesBenchmark.get_queries().", "Existing skip_on behavior continues to exclude unsupported queries without counting them as failures.", "Runnable but non-equivalent variants are skipped rather than retained as benchmark-comparable queries."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/ai_primitives/catalog/queries.yaml"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/metadata_primitives/catalog/queries.yaml"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}], "state": "done", "title": "Ensure read_primitives query variants are meaningfully comparable", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/read_primitives/test_catalog_loader.py tests/unit/core/read_primitives/test_query_manager_variants.py tests/unit/core/read_primitives/test_benchmark_variants.py -q", "description": "Catalog loader and existing variant behavior still pass", "expected": "all tests pass", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/read_primitives/test_read_primitives_variant_contracts.py -q", "description": "New static comparability contracts pass", "expected": "all tests pass", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/integration/validation/test_read_primitives_variant_parity.py -q", "description": "Focused runtime parity tests pass or skip cleanly for missing optional engines", "expected": "tests pass; optional engines may skip when unavailable", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after this item is created", "expected": "no cycles or dangling dependencies", "item_id": "ensure-read-primitives-variant-comparability", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add result_contract metadata support to the read_primitives catalog model and loader", "wid": "w1"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add static comparability linting for base/variant output columns, nested shape, row identity, and measured capability", "wid": "w2"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or skip scalar and capability-divergent variants: fulltext, window_moving_frame, statistical_percentiles, timeseries_trend_analysis, and any_value", "wid": "w3"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix or skip semi-structured variants with nested-shape risk: JSON, arrays, structs, maps, and lambda/list operations", "wid": "w4"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add focused runtime parity tests for risky read_primitives families against DuckDB reference at SF=0.01", "wid": "w5"}, {"evidence": null, "item_id": "ensure-read-primitives-variant-comparability", "needs": ["w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update read_primitives catalog docs and skip reference to require comparable variants or skip_on", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "keep the widening unconditional", "instead": "(not specified)", "why": "a value oracle that equates NaN with NULL or ignores trailing spaces by default cannot be trusted to catch those bug classes."}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-28T00:00:00Z", "completed_pr": null, "created_at": "2026-06-28T00:00:00Z", "deferrals": [], "deps": [], "description": "The shared result comparator silently widens equivalence in two engine-justified\nbut globally-applied ways, each of which can mask a real cross-surface value\ndivergence:\n - validation.py:718 (_numeric_values_equal) treats float('nan') and None as\n both \"missing\" and reports two missing values as EQUAL. Combined with\n _normalize_value mapping NaN->None, a DataFrame surface that produces a\n spurious NaN (e.g. 0/0 from a division bug) compares EQUAL to a SQL NULL.\n - validation.py:693 strips leading/trailing whitespace unconditionally\n (val1.strip() == val2.strip()), so \"foo \" vs \"foo\" is declared equal - masking\n a real transcription difference.\nBoth are justified for specific engine decode paths (e.g. ClickHouse Arrow/pandas\nNULL decode, CHAR padding) but are applied engine-agnostically with no flag to\ndisable, so they widen equivalence for every gate.\n\nMake both behaviors explicit comparator flags (e.g. treat_nan_as_null,\nstrip_strings) that default to STRICT, enabled only where an engine genuinely\nneeds them and documented at that call site.", "id": "equivalence-comparator-value-widening-opt-in", "preserves": ["Engine decode paths that genuinely need NaN-as-null or CHAR-padding tolerance still pass, via the explicit opt-in flag at that site.", "Float relative-tolerance (1e-10) comparison semantics are unchanged."], "prior_art": [{"concept": "{'ref': 'benchbox/core/equivalence/cross_surface.py / dataframe_surface.py (_normalize_value)', 'note': 'Where NaN->None normalization happens; enable the widening flags only where a specific engine decode requires it.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py:693', 'note': 'Unconditional string .strip() compare - add a strip_strings flag, default False.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/core/tpchavoc/validation.py:718', 'note': '_numeric_values_equal NaN/None handling - add a treat_nan_as_null flag, default False (strict).'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/equivalence/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/"}], "state": "done", "title": "Make equivalence-comparator value-widening (NaN==NULL, whitespace strip) opt-in instead of unconditional", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/equivalence -q && make ssb-cross-surface-equivalence-report", "description": "Strict default catches widened divergences; enforced gates stay green", "expected": "new strict-default tests pass; all enforced gates still 0 divergent", "item_id": "equivalence-comparator-value-widening-opt-in", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "equivalence-comparator-value-widening-opt-in", "needs": [], "notes": "Add treat_nan_as_null and strip_strings flags (additive, default False so the\nstrict behavior is the default). Thread them through the comparator entry\npoints. Enable them ONLY at the specific engine decode sites that genuinely\nneed them, with an inline comment justifying each. Keep TPC-Havoc and all\nenforced gates green (verify none silently depended on the widening - if one\ndoes, that is a divergence to surface, not to preserve).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Gate NaN==NULL and whitespace-strip behind comparator flags, default strict", "wid": "w1"}, {"evidence": null, "item_id": "equivalence-comparator-value-widening-opt-in", "needs": ["w1"], "notes": "Unit tests asserting, under the strict default: a DataFrame-side 0/0->NaN vs\nSQL NULL is CAUGHT, and \"foo \" vs \"foo\" is CAUGHT. And that the opt-in flags,\nwhere set, still accept the engine-justified case.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Test that the strict default catches the masked divergences", "wid": "w2"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add review_by enforcement as a hard FAIL by default without checking it would not immediately break an existing gate", "instead": "(not specified)", "why": "a surprise new failure mode on merge day for a benchmark whose owner is not in the room is worse than a warning."}, {"dont": "conflate review_by-past-due with the existing resolved-divergence stale-baseline failure", "instead": "(not specified)", "why": "they are different signals (one means 'the divergence is GONE, prune the entry'; the other means 'the divergence may still be valid but a human should re-look') and must not share one code path or one message format that a reader could confuse."}, {"dont": "fork _report per module to add this", "instead": "one keys on backend+query, the other on query+variant).", "why": "both modules already have their own _report; extend each in place, do not introduce a third shared helper unless the two are trivially unifiable (they are not"}, {"dont": "make review_by mandatory on every waiver", "instead": "keep it opt-in.", "why": "most waivers (e.g. the h2odb Q9 DECIMAL-scale residue, an inherent engine behavior) may never need re-review and forcing a date on them is busywork, not signal"}], "approach": "Decide the carrier shape and warn-vs-fail policy first (w1), since it\ndetermines whether cross_surface.py's known_divergences Union type or\nequivalence.py's dict[str, str] baselines need a type change vs a\nside-table. Implement the check in each module's _report (w2, w3), document\nthe policy in both module docstrings (w4), and retrofit one worked example\nper module with a test proving the check fires (w5).\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-03T00:00:00Z", "completed_pr": null, "created_at": "2026-07-03T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "equivalence-waiver-expiry-policy", "id": 312, "reason": "Out of scope; this item only adds the detectable signal (a warning/failure at report time). Automating a proactive nudge is a separate CI/scheduling concern.", "resolution": "promoted", "resolved_item": "automatically-opening-an-issue-pr-or-paging-an-owner-when-a", "resolved_reason": null, "summary": "Automatically opening an issue/PR or paging an owner when a review_by date passes."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "equivalence-waiver-expiry-policy", "id": 313, "reason": "review_by is optional by design (per the task framing: 'Keep it lightweight'); mandating it on every existing entry is a larger migration this item does not need to force.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: review_by is optional by design per the parent item's own framing; retroactively mandating it on all waivers was deliberately rejected.", "summary": "Requiring review_by on every existing waiver retroactively."}], "deps": [], "description": "SOUNDNESS-CRITICAL COMPARATOR SURFACE: this TODO's work touches\nbenchbox/core/equivalence/cross_surface.py (covered by .github/CODEOWNERS as\n`benchbox/core/equivalence/** @joeharris76`) and\nbenchbox/core/tpchavoc/equivalence.py (adjacent to, and sharing the same\nwaiver-baseline discipline as, benchbox/core/tpchavoc/validation.py, which\nIS covered as `benchbox/core/**/validation.py @joeharris76`; note that\ntpchavoc/equivalence.py itself is not literally matched by an existing\nCODEOWNERS glob - verified against the current 3-line .github/CODEOWNERS -\nbut should be treated as soundness-critical and reviewed by the code owner\nregardless). Its PR will require code-owner review before merge.\n\nVerified by reading both modules on this branch: no known-divergence waiver\nentry, in either file, carries any date or expiry field.\n - cross_surface.py's `known_divergences: dict[str, str | ClassifiedDivergence]`\n (CrossSurfaceGate field, line 371) holds bare strings (e.g.\n `_READ_PRIMITIVES_KNOWN_DIVERGENCES`, cross_surface.py:859-885) and\n `ClassifiedDivergence` instances (cross_surface.py:338-357, e.g.\n `_H2ODB_KNOWN_DIVERGENCES` at 813-816, `_CLICKBENCH_TIE_AMBIGUOUS` at\n 619-624+) - neither the `str` nor the `ClassifiedDivergence` dataclass\n (fields: `reason`, `accepts`, `requires_live_divergence`) has a date field.\n - equivalence.py's four engine-keyed baselines (`KNOWN_DIVERGENCES`,\n `POSTGRES_KNOWN_DIVERGENCES`, `DATAFUSION_KNOWN_DIVERGENCES`,\n `CLICKHOUSE_KNOWN_DIVERGENCES`, all `dict[str, str]`) are plain\n key->reason-string maps with no date field either.\n\nThe only existing staleness signal in either module is \"stale-baseline\nfailure\": a waiver whose divergence has stopped reproducing is caught (in\ncross_surface.py's `_report`, the `resolved` check at line ~1188-1192; in\nequivalence.py's shared `_report`, a `resolved` PRINT only - see the sibling\nTODO tpchavoc-duckdb-gate-vacuity-guard w3 for that gap). But a waiver whose\ndivergence is STILL LIVE (e.g. an \"irreducible engine-semantic difference\" or\nan \"approximate by construction\" classification) never gets re-examined - it\ncan sit correct-when-written but stale-in-spirit indefinitely: the engine\nversion that produced the documented behavior may have changed, the\nbenchmark's data/queries may have evolved, or the original rationale may no\nlonger hold, and nothing forces a human to re-look.\n\nThis TODO adds a lightweight, OPTIONAL `review_by:` date to waiver entries\n(both the bare-string form, by upgrading it to a small structured wrapper or\na side-table, and the `ClassifiedDivergence` form, as a new optional field),\nand a warning-or-failure when a `review_by` date has passed, so long-lived\nwaivers are periodically forced back into review even when their divergence\nnever disappears on its own.", "id": "equivalence-waiver-expiry-policy", "preserves": ["The existing stale-baseline (resolved-divergence) failure behavior in both modules' _report functions is unchanged by this addition - review_by is a new, independent check, not a replacement.", "cross_surface.py's GATES stay green and equivalence.py's tpchavoc-equivalence-report* targets stay green after this lands (no waiver in either module should already have an accidentally-past-due review_by introduced by this change).", "review_by is OPTIONAL - every existing waiver entry with no review_by continues to behave exactly as today (no forced migration, no new failure for entries that omit it)."], "prior_art": [{"concept": "benchbox/core/equivalence/cross_surface.py:1173-1196 _report - where a new review_by warning/failure check is added, alongside the existing new/resolved/missing_backends/unclassified_empty checks.", "decision": "reuse", "path": "benchbox/core/equivalence/cross_surface.py:1173-1196"}, {"concept": "benchbox/core/equivalence/cross_surface.py:338-357 ClassifiedDivergence - the dataclass to extend with an optional review_by field (or a documented decision not to, if the bare-string form is chosen as the carrier instead).", "decision": "extend", "path": "benchbox/core/equivalence/cross_surface.py:338-357"}, {"concept": "benchbox/core/equivalence/cross_surface.py:371 CrossSurfaceGate.known_divergences: dict[str, str | ClassifiedDivergence] - the Union type that must accommodate whatever wrapper form w1 chooses for bare-string entries with a review_by date.", "decision": "reuse", "path": "benchbox/core/equivalence/cross_surface.py:371"}, {"concept": "benchbox/core/tpchavoc/equivalence.py:434-470 _report (KNOWN_DIVERGENCES/POSTGRES_KNOWN_DIVERGENCES/DATAFUSION_KNOWN_DIVERGENCES/CLICKHOUSE_KNOWN_DIVERGENCES, all dict[str, str]) - the second _report that needs the same review_by check, in a shape compatible with its plain string-valued dicts.", "decision": "reuse", "path": "benchbox/core/tpchavoc/equivalence.py:434-470"}, {"concept": "cross-surface-baseline-update-flag.yaml (planning) - a closely related, already-filed TODO for a --update-baseline flag to DROP resolved entries; this item is complementary (forcing review of LIVE entries) and should reuse its w0's re-confirm-the-live-file-and-lines discipline, not duplicate its writer.", "decision": "reuse", "path": "cross-surface-baseline-update-flag.yaml"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/tpchavoc/validation.py"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/equivalence/cross_surface.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpchavoc/equivalence.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/tpchavoc/"}, {"kind": "only_modify", "path_glob": "tests/unit/equivalence/ (actually tests/unit/core/equivalence/ - see verification deviation note)"}], "state": "done", "title": "Add an optional review_by expiry date to known-divergence waivers so live waivers are periodically forced back into review", "verifications": [{"command": "uv run -- python -m pytest tests/unit/equivalence/ -k review_by -v", "description": "A synthetic past-due review_by waiver triggers the new signal in cross_surface.py's report", "expected": "the seeded past-due waiver produces the documented warning (or failure, per w1) line; a future-dated or absent review_by does not", "item_id": "equivalence-waiver-expiry-policy", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/tpchavoc/ -k review_by -v", "description": "A synthetic past-due review_by waiver triggers the new signal in equivalence.py's report", "expected": "same behavior, ported to the TPC-Havoc engine-keyed baselines", "item_id": "equivalence-waiver-expiry-policy", "last_result": null, "last_run": null, "seq": 2}, {"command": "make ssb-cross-surface-equivalence-report && make read-primitives-cross-surface-equivalence-report && make tpchavoc-equivalence-report", "description": "Existing gates stay green after the change lands", "expected": "all three exit 0, unchanged from before this TODO", "item_id": "equivalence-waiver-expiry-policy", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": [], "notes": "DECISION, matching the open_questions defaults exactly (directed as the\npolicy for this batch and independently confirmed sound):\n (a) cross_surface.py: option (i) - added `review_by: date | None = None`\n as a third field on `ClassifiedDivergence`. Bare-`str` entries\n CANNOT carry a review_by (documented explicitly in both the\n dataclass docstring and the module's new \"waiver review policy\"\n section) - no bare-string waiver was converted to\n ClassifiedDivergence to add one. Verified this does not complicate\n the sibling read-primitives-waivers-classified-divergence TODO:\n `_READ_PRIMITIVES_KNOWN_DIVERGENCES` (read_primitives gate) still\n has its existing bare-string entries untouched; nothing in this\n change forces or blocks that sibling's future conversion.\n (b) equivalence.py (tpchavoc): option (ii) - a side-table\n `dict[str, date]` per baseline, keyed identically to the baseline\n it annotates (e.g. `CLICKHOUSE_KNOWN_DIVERGENCES_REVIEW_BY`), passed\n into the shared `_report` as a new optional `review_by` parameter.\n No existing `dict[str, str]` baseline's value type changed.\n (c) WARN, not fail - matches the default exactly. A past-due\n `review_by` prints a \"WAIVER REVIEW DUE - ...\" line in both\n modules' `_report`s but never changes the return code, kept\n deliberately distinct in wording and code path from the existing\n `resolved` GATE FAILURE (per the anti_patterns).\nPrior notes (superseded by the decision above, kept for context):\nTwo open design questions to resolve before implementing:\n (a) Carrier shape for cross_surface.py's bare-`str` waivers (the majority\n of entries - see _READ_PRIMITIVES_KNOWN_DIVERGENCES, and any others\n not yet converted to ClassifiedDivergence by the sibling TODO\n read-primitives-waivers-classified-divergence). Options: (i) add\n `review_by` as a THIRD optional field on `ClassifiedDivergence` and\n require every review_by-bearing waiver to at least be wrapped in\n `ClassifiedDivergence(reason=..., accepts=lambda _: True, review_by=...)`\n even if it has no real predicate yet (accepts-everything is\n explicit, not implicit, in this form) - this reuses one type instead\n of two; or (ii) introduce a small side-table\n `dict[str, date]` (e.g. `_REVIEW_BY: dict[str, date]`) keyed by the\n same divergence key, checked independently of the waiver's own\n shape - avoids touching the `known_divergences` Union type at all.\n Prefer (i) if the sibling TODO's ClassifiedDivergence conversion is\n expected to land first (fewer bare strings to carry the concern for\n long-term); prefer (ii) if this item lands independently and should\n not force every bare-string waiver into a dataclass. State and\n justify the choice here.\n (b) Same question for equivalence.py's plain `dict[str, str]` baselines\n (KNOWN_DIVERGENCES et al.) - these have no ClassifiedDivergence\n equivalent at all today, so a side-table (ii)-style\n `dict[str, date]` per baseline, or a small `@dataclass` wrapper\n (`KnownDivergence(reason: str, review_by: date | None = None)`)\n replacing the bare `str` value type, is the realistic choice. Note\n this is a wider blast radius (touches every existing baseline\n entry's type) - if it proves too invasive for zero-value-change\n entries, the side-table is likely the pragmatic default.\n (c) Warn vs fail: decide in this work unit, do not defer. Given the\n module already treats a stale-baseline `resolved` divergence as a\n hard gate FAILURE in cross_surface.py (line ~1191), a past-due\n `review_by` on a STILL-LIVE waiver is a weaker signal (the waiver\n may still be entirely correct) - default to WARN (print prominently,\n do not fail exit code) unless a strong case for FAIL emerges (e.g.\n \"the gate is meant to be a hard forcing function and a warning is\n too easy to ignore\"). Record the decision and rationale.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the review_by carrier for both waiver shapes and decide warn-vs-fail on a past-due date", "wid": "w1"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w1"], "notes": "Added `review_by: date | None = None` to `ClassifiedDivergence`. In\n`_report`, computes `review_due = sorted(key for key, entry in\nknown.items() if isinstance(entry, ClassifiedDivergence) and\nentry.review_by is not None and entry.review_by < date.today())` and, for\neach, prints `\"WAIVER REVIEW DUE - {key}: review_by {entry.review_by} has\npassed - {entry.reason}\"` - a distinct prefix from \"GATE FAILURE\" so a\nreader (or a naive grep for FAILURE) cannot confuse the two signals. The\nreturn statement (`1 if (new or resolved or missing_backends or\nunclassified_empty) else 0`) is UNCHANGED - review_due never enters it,\nper w1(c)'s warn decision.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the review_by field/side-table and wire the warning/failure check into cross_surface.py's _report", "wid": "w2"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w1"], "notes": "Added an optional `review_by: dict[str, date] | None = None` parameter to\nthe shared `_report` (equivalence.py). Computes `review_due = sorted(key\nfor key, due in review_by.items() if key in known and due <\ndate.today())` and prints the same \"WAIVER REVIEW DUE - ...\" line style as\ncross_surface.py (message text intentionally parallel, code NOT shared -\nper the anti_patterns, the two `_report`s stay separate functions).\nVerified live all four report modes after the change: DuckDB hard gate\n(220/220, exit 0), DataFusion sample (206/206, exit 0), ClickHouse sample\n(195/202 + 7 classified, exit 0, no review-due warning since the worked\nexample's date is not yet past), Postgres sample unreachable in this\nenvironment (clean SKIP, exit 0, unaffected - POSTGRES_KNOWN_DIVERGENCES\nis empty so review_by never applies there regardless).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the review_by field/side-table and wire the equivalent check into equivalence.py's shared _report", "wid": "w3"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w2", "w3"], "notes": "Added a \"Waiver review policy\" paragraph to both module docstrings\n(cross_surface.py, after the \"Currently gated\" paragraph; equivalence.py,\nafter the LIMIT-stripping paragraph) stating: an entry MAY carry an\noptional review_by date; a past-due date WARNS (never fails); this is\nindependent of and does not replace the existing stale-baseline\n(`resolved`) failure - one means \"the divergence is gone, delete the\nentry\", the other means \"the divergence may still be valid, re-look\nanyway\". Also added matching explanatory paragraphs to the `ClassifiedDivergence`\ndataclass docstring and both `_report` docstrings.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the policy in both modules' docstrings", "wid": "w4"}, {"evidence": null, "item_id": "equivalence-waiver-expiry-policy", "needs": ["w2", "w3", "w4"], "notes": "cross_surface.py: retrofitted `_CLICKBENCH_TIE_AMBIGUOUS` (ClickBench Q18,\nan order-less top-N with NO ORDER BY over ~97k groups - structurally\npermanent, `requires_live_divergence=False` already documents it can never\ntrip the `resolved` check on its own, so review_by is the ONLY forcing\nfunction available for it) with `review_by=date(2027, 1, 3)` (6 months\nfrom created_date 2026-07-03).\nequivalence.py: retrofitted CLICKHOUSE_KNOWN_DIVERGENCES's \"1_v4\"\ndecimal-division-truncation entry (inherent to ClickHouse's Decimal\narithmetic, unlikely to ever resolve) via a new\n`CLICKHOUSE_KNOWN_DIVERGENCES_REVIEW_BY = {\"1_v4\": date(2027, 1, 3)}`\nside-table, wired into `run_clickhouse_sample`'s `_report(...)` call.\nTests added:\n - tests/unit/core/equivalence/test_cross_surface.py:\n test_review_by_past_due_warns_but_does_not_fail_the_gate,\n test_review_by_future_date_does_not_warn,\n test_review_by_absent_by_default_produces_no_warning\n - tests/unit/core/tpchavoc/test_tpchavoc_review_by.py (new file):\n test_review_by_past_due_warns_but_does_not_fail_the_report,\n test_review_by_past_due_warns_on_a_still_reproducing_divergence,\n test_review_by_future_date_does_not_warn,\n test_review_by_absent_by_default_produces_no_warning,\n test_clickhouse_review_by_worked_example_is_not_yet_due (regression\n guard: fails loudly if the worked example's date is ever left to go\n stale unnoticed).\nAll 8 pass; confirmed neither worked-example date (2027-01-03) is\ncurrently past-due (today is 2026-07-03), so no gate/sample newly warns.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retrofit review_by onto at least one existing long-lived waiver per module as a worked example, and add a test", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-12-12T00:00:00Z", "deferrals": [], "deps": [], "description": "Systematic review and improvement of error messages across all platform adapters\nand CLI commands.\n\nRecent work (Presto/Trino catalog handling) demonstrated the value of actionable\nerror messages that suggest next steps. This pattern should be applied consistently\nacross the codebase.\n\nFocus areas:\n- Connection failures (credentials, network, permissions)\n- Missing dependencies (optional packages not installed)\n- Invalid configuration (wrong parameter combinations)\n- Data loading errors (file not found, format mismatch)", "id": "error-message-polish", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Error Message Polish", "verifications": [], "work": [], "worktree": "release-readiness"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:36:32Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #491 of testing-approach-correctness-gates.\nDeferred: Evaluate mutation testing on dialect/result-model core\nReason deferred: Useful after correctness lanes exist; not a prerequisite for closing the current CI routing gap.", "id": "evaluate-mutation-testing-on-dialect-result-model-core", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Evaluate mutation testing on dialect/result-model core", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "design the extracted-repo architecture in this TODO", "why": "that's the work that follows the decision; designing it now anchors the decision."}, {"INSTEAD": "(not specified)", "dont": "pick triggers that require manual measurement", "why": "they will be ignored; favor checks the metrics script can compute."}, {"INSTEAD": "(not specified)", "dont": "pre-commit to extraction", "why": "the trade-offs depend on numbers we don't have yet; this TODO is a forcing function, not a decision."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-27T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["define-phase-3-promotion-metrics", "implement-pr-based-result-submission-workflow"], "description": "The strategy doc says: \"A dedicated `benchbox-results` repository remains\nan optional future extraction if corpus size or contribution volume starts\nto overwhelm the main repo. It is not a Phase 1 or early Phase 2\nrequirement.\"\n\nWithout an explicit checkpoint, this becomes a perpetually deferred\nquestion that surfaces ad-hoc the first time someone gets annoyed about\nrepo size. The decision has real trade-offs (CI permissions, contributor\nfriction, SHA tracking across two repos, history rewrite risk) that\ndeserve a structured evaluation rather than a hot take.\n\nThis item creates the forcing function: a defined trigger condition, a\npre-written ADR template, and a lightweight evaluation procedure. When\nthe trigger fires, the evaluation produces a yes/no answer with rationale.\nUntil then, this TODO stays in planning as a marker that the decision is\nintentional, not forgotten.", "id": "evaluate-results-data-repo-extraction-checkpoint", "preserves": ["Existing `results-data/` location and structure on `published-results` branch.", "The strategy doc's \"extract only if churn justifies it\" disposition - this TODO encodes the trigger, doesn't pre-decide the extraction.", "Working CI workflows that reference `results-data/`."], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/results-data-extraction-trigger.md"}, {"kind": "only_modify", "path_glob": "docs/development/adr/TEMPLATE-results-data-extraction.md"}, {"kind": "only_modify", "path_glob": "scripts/phase2_metrics.py"}], "state": "done", "title": "Schedule a checkpoint to evaluate extracting results-data/ to its own repo", "verifications": [{"command": "test -f _project/analysis/results-data-extraction-trigger.md && grep -c 'trigger' _project/analysis/results-data-extraction-trigger.md", "description": "Trigger doc exists with quantitative + qualitative conditions", "expected": ">= 2", "item_id": "evaluate-results-data-repo-extraction-checkpoint", "last_result": null, "last_run": null, "seq": 1}, {"command": "test -f docs/development/adr/TEMPLATE-results-data-extraction.md", "description": "ADR template exists with decision matrix scaffold", "expected": "file exists", "item_id": "evaluate-results-data-repo-extraction-checkpoint", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": [], "notes": "Pick 2-3 quantitative triggers (any one fires the evaluation):\n - `results-data/` exceeds 250MB in the main repo\n - Sustained 20+ community PRs/month (3 consecutive months)\n - Contributor-reported fork/clone times > 60 seconds on broadband\nAnd 1-2 qualitative triggers:\n - A maintainer formally requests evaluation citing operational pain\n - Phase 3 design begins (extraction may be a precondition)\nDocument in `_project/analysis/results-data-extraction-trigger.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the evaluation trigger conditions", "wid": "w1"}, {"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": ["w1"], "notes": "Template `docs/development/adr/TEMPLATE-results-data-extraction.md`:\n - Context (repo size, PR volume at time of decision)\n - Options:\n A. Stay in monorepo (status quo)\n B. Extract to `joeharris76/benchbox-results` (or similar)\n C. Extract with selective sync (data repo + lightweight pointer\n in main repo)\n - For each option:\n - CI implications (workflows currently in main repo)\n - Contributor friction (one-repo vs two-repo PR flow)\n - Hash stability (SHAs change if history is rewritten on extract)\n - Cross-repo SHA tracking (does the explorer pin a data repo\n commit?)\n - Operational cost\n - Recommended decision matrix scoring template\n - Decision section (filled in at evaluation time, not now)\n - Rollback path: how to merge back into the monorepo if extraction\n proves wrong\nReference: `git filter-repo` for the extraction mechanic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write the ADR template covering the trade-offs", "wid": "w2"}, {"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": ["w1"], "notes": "Extend `scripts/phase2_metrics.py` (from `define-phase-3-promotion-metrics`)\nwith a final section that runs the quantitative checks from w1 and\nprints a clear \"EXTRACTION EVALUATION RECOMMENDED: \" message\nwhen triggered. This piggybacks on the existing quarterly review\ncadence - no separate scheduling needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a one-line check in the Phase 3 metrics script that flags trigger", "wid": "w3"}, {"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": ["w2", "w3"], "notes": "Append to `_project/analysis/results-data-extraction-trigger.md`:\n Procedure when triggered:\n 1. Open new TODO `evaluate-results-data-extraction-decision`\n (active, owner = current maintainer rotation).\n 2. Copy the ADR template to `docs/development/adr/{date}-results-data-extraction.md`.\n 3. Fill in current measurements, score the matrix, write the\n recommendation.\n 4. Get review from at least one other maintainer.\n 5. Commit decision; if \"extract\", file a follow-up implementation\n TODO with explicit migration plan including history-preservation\n strategy.\n 6. Mark this checkpoint TODO Completed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the evaluation procedure", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "design the extracted-repo architecture in this TODO", "instead": "(not specified)", "why": "that's the work that follows the decision; designing it now anchors the decision."}, {"dont": "pick triggers that require manual measurement", "instead": "(not specified)", "why": "they will be ignored; favor checks the metrics script can compute."}, {"dont": "pre-commit to extraction", "instead": "(not specified)", "why": "the trade-offs depend on numbers we don't have yet; this TODO is a forcing function, not a decision."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-27T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["define-phase-3-promotion-metrics", "implement-pr-based-result-submission-workflow"], "description": "The strategy doc says: \"A dedicated `benchbox-results` repository remains\nan optional future extraction if corpus size or contribution volume starts\nto overwhelm the main repo. It is not a Phase 1 or early Phase 2\nrequirement.\"\n\nWithout an explicit checkpoint, this becomes a perpetually deferred\nquestion that surfaces ad-hoc the first time someone gets annoyed about\nrepo size. The decision has real trade-offs (CI permissions, contributor\nfriction, SHA tracking across two repos, history rewrite risk) that\ndeserve a structured evaluation rather than a hot take.\n\nThis item creates the forcing function: a defined trigger condition, a\npre-written ADR template, and a lightweight evaluation procedure. When\nthe trigger fires, the evaluation produces a yes/no answer with rationale.\nUntil then, this TODO stays in planning as a marker that the decision is\nintentional, not forgotten.", "id": "evaluate-results-data-repo-extraction-checkpoint", "preserves": ["Existing `results-data/` location and structure on `published-results` branch.", "The strategy doc's \"extract only if churn justifies it\" disposition - this TODO encodes the trigger, doesn't pre-decide the extraction.", "Working CI workflows that reference `results-data/`."], "prior_art": [], "priority": "low", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/results-data-extraction-trigger.md"}, {"kind": "only_modify", "path_glob": "docs/development/adr/TEMPLATE-results-data-extraction.md"}, {"kind": "only_modify", "path_glob": "scripts/phase2_metrics.py"}], "state": "done", "title": "Schedule a checkpoint to evaluate extracting results-data/ to its own repo", "verifications": [{"command": "test -f _project/analysis/results-data-extraction-trigger.md && grep -c 'trigger' _project/analysis/results-data-extraction-trigger.md", "description": "Trigger doc exists with quantitative + qualitative conditions", "expected": ">= 2", "item_id": "evaluate-results-data-repo-extraction-checkpoint", "last_result": null, "last_run": null, "seq": 1}, {"command": "test -f docs/development/adr/TEMPLATE-results-data-extraction.md", "description": "ADR template exists with decision matrix scaffold", "expected": "file exists", "item_id": "evaluate-results-data-repo-extraction-checkpoint", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": [], "notes": "Pick 2-3 quantitative triggers (any one fires the evaluation):\n - `results-data/` exceeds 250MB in the main repo\n - Sustained 20+ community PRs/month (3 consecutive months)\n - Contributor-reported fork/clone times > 60 seconds on broadband\nAnd 1-2 qualitative triggers:\n - A maintainer formally requests evaluation citing operational pain\n - Phase 3 design begins (extraction may be a precondition)\nDocument in `_project/analysis/results-data-extraction-trigger.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define the evaluation trigger conditions", "wid": "w1"}, {"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": ["w1"], "notes": "Template `docs/development/adr/TEMPLATE-results-data-extraction.md`:\n - Context (repo size, PR volume at time of decision)\n - Options:\n A. Stay in monorepo (status quo)\n B. Extract to `joeharris76/benchbox-results` (or similar)\n C. Extract with selective sync (data repo + lightweight pointer\n in main repo)\n - For each option:\n - CI implications (workflows currently in main repo)\n - Contributor friction (one-repo vs two-repo PR flow)\n - Hash stability (SHAs change if history is rewritten on extract)\n - Cross-repo SHA tracking (does the explorer pin a data repo\n commit?)\n - Operational cost\n - Recommended decision matrix scoring template\n - Decision section (filled in at evaluation time, not now)\n - Rollback path: how to merge back into the monorepo if extraction\n proves wrong\nReference: `git filter-repo` for the extraction mechanic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write the ADR template covering the trade-offs", "wid": "w2"}, {"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": ["w1"], "notes": "Extend `scripts/phase2_metrics.py` (from `define-phase-3-promotion-metrics`)\nwith a final section that runs the quantitative checks from w1 and\nprints a clear \"EXTRACTION EVALUATION RECOMMENDED: \" message\nwhen triggered. This piggybacks on the existing quarterly review\ncadence - no separate scheduling needed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a one-line check in the Phase 3 metrics script that flags trigger", "wid": "w3"}, {"evidence": null, "item_id": "evaluate-results-data-repo-extraction-checkpoint", "needs": ["w2", "w3"], "notes": "Append to `_project/analysis/results-data-extraction-trigger.md`:\n Procedure when triggered:\n 1. Open new TODO `evaluate-results-data-extraction-decision`\n (active, owner = current maintainer rotation).\n 2. Copy the ADR template to `docs/development/adr/{date}-results-data-extraction.md`.\n 3. Fill in current measurements, score the matrix, write the\n recommendation.\n 4. Get review from at least one other maintainer.\n 5. Commit decision; if \"extract\", file a follow-up implementation\n TODO with explicit migration plan including history-preservation\n strategy.\n 6. Mark this checkpoint TODO Completed.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document the evaluation procedure", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "New Feature Requests", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Modernize DuckDB benchmark examples to use the 3-line adapter pattern\"\n- \"**Tasks**:\"\n- \"**Files Affected**: `examples/duckdb_tpch*.py`, `examples/duckdb_tpcds.py`\"\n- \"**Impact**: \u2705 Eliminates outdated anti-patterns, demonstrates correct framework usage, reduces maintenance overhead by 85%, and provides clear guidance for new users on the modern adapter pattern approach.\"", "id": "example-modernization-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Example Modernization \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Pre-Release Medium Priority Tasks (Nice-to-Have)", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "Created a progressive suite of getting-started scripts for DuckDB, Databricks, and BigQuery, refreshed the documentation to explain the learning path, and added automated tests that exercise the new examples without requiring external credentials.", "id": "example-simplification", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "\u2705 Example Simplification", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "New Feature Requests", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: Medium\"\n- \"**Status**: \u2705 Completed\"\n- \"**Description**: Add tuning configuration options to DuckDB TPC-H and TPC-DS examples to demonstrate modern benchmarking capabilities\"\n- \"**Tasks**:\"\n- \"**Files Affected**: `examples/duckdb_tpch.py`, `examples/duckdb_tpcds.py`\"\n- \"**Impact**: \u2705 Demonstrates production-ready benchmarking practices, enables performance comparison testing, and showcases the full capabilities of the tuning system for new users. Examples now include comprehensive tuning controls with validation, clear help documentation, and detailed result reporting of applied configurations.\"", "id": "example-tuning-configuration-integration-completed", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Example Tuning Configuration Integration \u2705 COMPLETED", "verifications": [], "work": [], "worktree": "unknown"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:10Z", "deferrals": [], "deps": [], "description": "- \"**Priority**: High\"\n- \"**Status**: \u2705 **COMPLETED**\"\n- \"**Description**: Eliminate substantial code duplication across 24 example files by consolidating repeated functionality into core BenchBox modules, reducing maintenance burden and improving consistency\"\n- \"**Context**: Analysis reveals 24 files contain identical utility functions (`format_duration`, `format_bytes`), 22 files have repetitive argument parsing patterns, 17 files duplicate dependency checking logic, and 17 files implement similar result display functions\"\n- \"**Duplication Analysis**:\"\n- \"**Consolidation Strategy**: Integrate duplicated functionality into existing core modules rather than creating shared example utilities, maintaining standalone example principle\"\n- \"**Tasks**:\"\n- \"**Files Affected**:\"\n- \"**Implementation Benefits**:\"\n- \"**Integration Points**:\"\n- \"**Success Metrics Achieved**:\"\n- \"**Implementation**: \u2705 **COMPLETED** - All phases implemented with comprehensive testing\"\n- \"**Impact**: \u2705 **ACHIEVED** - Dramatically reduced maintenance burden for examples, ensured consistency across all example scripts, maintained standalone example principle while leveraging core infrastructure, improved code quality and eliminated duplication-related bugs\"", "id": "examples-code-duplication-consolidation", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "done", "title": "Examples Code Duplication Consolidation", "verifications": [], "work": [], "worktree": "unknown"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a second dispatch mechanism; extend resolve_*_harness.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "change execution semantics while extracting", "why": "this is a structural move only, proven by unchanged results on the correctness/equivalence gates."}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-harness-extraction-throughput-maintenance-combined", "id": 393, "reason": "Only TPC has multi-phase power/throughput/maintenance shapes today; generalization is speculative until a second family needs it.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Generalizing the multi-phase harness registry to non-TPC families is already tracked as an open planning item. (tracked by generalize-multiphase-harness-registry-non-tpc)", "summary": "Generalize the harness registry to non-TPC benchmark families."}], "deps": [], "description": "Follow-up to structural-debt-base-layer-tpc-and-module-split (PR #914, DONE).\n#914's w1 success_metric \u2014 \"platforms/base/execution.py carries no literal TPC\nbenchmark names in its dispatch path\" \u2014 was met ONLY for the power-test path:\n_execute_power_test (execution.py:682 on origin/develop) now dispatches via\nresolve_power_harness(benchmark_id) with zero literals. But _execute_throughput_test,\n_execute_maintenance_test, and _execute_combined_test still branch on literal\n`benchmark_id == \"tpch\"` / `== \"tpcds\"` (origin/develop execution.py ~713-779).\nThe guard test test_power_dispatch_path_has_no_literal_tpc_benchmark_names\ninspects only _execute_power_test, so the broader metric reads as met when it is\nnot. This item extends the capability-registry dispatch to the remaining three\ntest types and broadens the guard test to all four.\n\nBehavior-preserving refactor: throughput/maintenance/combined carry stream/cursor\nand connection-factory wiring (the TPC-DS path uses connection_factory +\n_make_stream_cursor for thread-safe cursors); that wiring must be preserved\nexactly, as #914 did for the power path.", "id": "execution-harness-extraction-throughput-maintenance-combined", "preserves": ["Byte-identical benchmark execution: query order, stream/cursor wiring, connection_factory + _make_stream_cursor thread-safe semantics, set_config_validation_mode call point.", "Public dispatch entry points (_execute_throughput_test etc.) keep their signatures and call contracts."], "prior_art": [{"concept": "{'ref': 'benchbox/core/power_harnesses.py', 'note': 'resolve_power_harness + the per-benchmark power harness modules #914 introduced (core/tpch/platform_power.py, core/tpcds/platform_power.py). Extend this registry with throughput/maintenance/combined harness resolution; do not invent a parallel dispatch.'}", "decision": "extend", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/base/execution.py:682', 'note': 'The already-extracted _execute_power_test as the template: capability lookup, harness invocation, preserved connection/cursor wiring.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/platforms/', 'note': 'test_power_dispatch_path_has_no_literal_tpc_benchmark_names \u2014 broaden it to assert no literal TPC names across all four dispatch methods.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/power_harnesses.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "only_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Extend capability-based power-harness dispatch to throughput, maintenance, and combined tests", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -k literal_tpc_benchmark_names -q", "description": "No literal TPC names remain in any execution.py dispatch method", "expected": "guard test asserts power, throughput, maintenance, and combined dispatch carry no literal benchmark_id == tpch/tpcds branches", "item_id": "execution-harness-extraction-throughput-maintenance-combined", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests -m fast -k 'execution or power or throughput or maintenance' -q", "description": "Benchmark execution is unchanged", "expected": "execution behavior unchanged; all green", "item_id": "execution-harness-extraction-throughput-maintenance-combined", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": [], "notes": "Grep execution.py for `benchmark_id == \"tpc` and record each method + line\n(line numbers in this TODO are origin/develop at review time and will drift).\nConfirm the guard test currently inspects only _execute_power_test. Summarize\nin the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm the residual literals and the guard test scope on fresh develop", "wid": "w0"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w0"], "notes": "Move the TPC-H/TPC-DS throughput bodies behind a resolve_*_harness lookup,\npreserving stream/cursor and connection-factory wiring exactly. Per-benchmark\nbodies live under core/tpc{h,ds}/, mirroring the power split.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract throughput dispatch to capability resolution", "wid": "w1"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w0"], "notes": "Same treatment for _execute_maintenance_test, preserving the\nset_config_validation_mode call point and DML/refresh semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract maintenance dispatch to capability resolution", "wid": "w2"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w1", "w2"], "notes": "_execute_combined_test composes power+throughput+maintenance; route it through\nthe now-extracted harnesses. No literal benchmark_id branch should remain.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract combined dispatch to capability resolution", "wid": "w3"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w3"], "notes": "Extend test_power_dispatch_path_has_no_literal_tpc_benchmark_names (or add a\nsibling) to assert no literal TPC names in throughput/maintenance/combined too.\nThen correct the #914 DONE success_metric back to the full-dispatch claim (or\ncoordinate with holistic-review-followup-metadata-truth, which reworded it to\nthe interim power-path-only wording).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Broaden the guard test to all four dispatch methods and restore the strong metric", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "decide DataFrame mode in runner.py by hard-coding family names again", "why": "future families must not require runner edits."}, {"INSTEAD": "(not specified)", "dont": "delete `dataframe_runner.py` just because it is deprecated", "why": "dry-run and tests still import it."}, {"INSTEAD": "(not specified)", "dont": "import optional DataFrame libraries in mode-detection helpers.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "let dry-run call a different TPC-DS variant fallback implementation than real execution.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "replace bidirectional Q-prefix filters with canonical stripping unless tests prove all caller contracts still work.", "why": "(unstated)"}], "approach": "Split the work into two small centers of gravity: one query-resolution helper\nand one execution-mode predicate. Route production mixin, dry-run, CLI, and\ndeprecated compatibility helpers through those centers. Keep compatibility\nwrappers until deletion is separately authorized.\n", "blocked_reason": null, "category": "DataFrame Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-dataframe-routing-and-query-resolution", "id": 394, "reason": "The current public/backward-compat registry describes it as a deprecated internal compatibility runner; deletion needs a separate removal-window decision after helper imports migrate.", "resolution": "promoted", "resolved_item": "delete-benchbox-core-runner-dataframe-runner-py-completely", "resolved_reason": null, "summary": "Delete `benchbox/core/runner/dataframe_runner.py` completely"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-dataframe-routing-and-query-resolution", "id": 395, "reason": "This TODO centralizes existing routing/query-resolution behavior; broader registry redesign should wait until drift is removed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Broader DataFrame registry redesign explicitly deferred until routing/query-resolution drift is removed; no evidence of that, no open item tracks it.", "summary": "Redesign DataFrame query registries for all benchmark families"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-dataframe-routing-and-query-resolution", "id": 396, "reason": "Can be done after the shared helper lands and tests stop requiring MagicMock internals.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: dataframe_runner.py/benchmark_mixin.py housing the MagicMock-aware hook detection are marked deprecated-internal-compatibility-runner in favor of adapter.run_benchmark+BenchmarkExecutionMixin; cleanup moot once removed.", "summary": "Remove MagicMock-aware hook handling from production modules"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Independent verification confirmed that DataFrame execution has two related\nsources of drift.\n\nThe deprecated internal compatibility module\n`benchbox/core/runner/dataframe_runner.py` is still load-bearing for CLI and\ndry-run helpers. Production DataFrame execution uses\n`adapter.run_benchmark()` through\n`benchbox/platforms/dataframe/benchmark_mixin.py`, but dry-run imports\n`get_execution_mode()` and `_get_queries_for_benchmark()` from the deprecated\nmodule. Both the deprecated runner and the production mixin duplicate TPC-H\nstream ordering, TPC-DS stream/variant fallback, and bidirectional query\nsubset filters.\n\nDataFrame-vs-SQL routing is also detected two ways. The lifecycle runner\ndecides DataFrame execution from adapter `family in (\"expression\", \"pandas\")`,\nwhile adapter factory, capabilities, CLI/dry-run helpers, and the deprecated\nrunner use platform suffix/default-mode logic. A new DataFrame family or\nregistry default can therefore disagree with the lifecycle branch.", "id": "execution-results-dataframe-routing-and-query-resolution", "preserves": ["Dry-run DataFrame query previews remain available for supported DataFrame benchmarks.", "Existing `dataframe_runner.py` compatibility imports keep working unless the public-contract map explicitly approves removal.", "No new DataFrame platform dependency is imported eagerly.", "Production DataFrame execution remains `adapter.run_benchmark()` plus `BenchmarkExecutionMixin`.", "TPC-H stream permutation and TPC-DS stream/variant fallback behavior does not change."], "prior_art": [{"concept": "benchbox/core/results/query_normalizer.py: existing one-way query ID normalization - reuse only if extended to support bidirectional filter sets.", "decision": "reuse", "path": "benchbox/core/results/query_normalizer.py"}, {"concept": "benchbox/core/runner/dataframe_runner.py: deprecated compatibility runner and current dry-run helper source - delegate rather than deleting first.", "decision": "reuse", "path": "benchbox/core/runner/dataframe_runner.py"}, {"concept": "benchbox/platforms/__init__.py: current `is_dataframe_platform` helper - reconcile with broader predicate rather than adding another list.", "decision": "reuse", "path": "benchbox/platforms/__init__.py"}, {"concept": "benchbox/platforms/adapter_factory.py: platform string normalization and default-mode logic - reuse for config/platform predicate.", "decision": "reuse", "path": "benchbox/platforms/adapter_factory.py"}, {"concept": "benchbox/platforms/dataframe/benchmark_mixin.py: production DataFrame adapter execution path - preserve behavior while extracting helpers.", "decision": "reuse", "path": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"concept": "docs/reference/backward-compatibility.md: deprecated dataframe_runner lifecycle row - preserve or update explicitly.", "decision": "reuse", "path": "docs/reference/backward-compatibility.md"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/query_normalizer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/adapter_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "docs/reference/backward-compatibility.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/runner/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_dataframe_factory.py"}], "state": "done", "title": "Execution/results hardening: unify DataFrame routing, query resolution, and query-id filters", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/runner/test_dataframe_runner.py tests/unit/core/runner/test_dataframe_runner_lifecycle.py tests/unit/platforms/dataframe/test_benchmark_mixin.py tests/unit/platforms/test_dataframe_factory.py -q", "description": "Focused DataFrame routing and compatibility tests pass.", "expected": "Focused DataFrame runner, mixin, and factory tests pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_dryrun.py tests/unit/core/test_dryrun_behavioral.py tests/unit/cli/test_cli_dryrun.py --tb=short", "description": "Dry-run tests pass after moving query extraction off deprecated helpers.", "expected": "Dry-run DataFrame query extraction tests pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite catches execution-mode routing regressions.", "expected": "Fast tests pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": [], "notes": "Re-run current-tree checks before editing and record checked SHA plus a\ncompact evidence summary in the implementation PR body or TODO notes.\n\nRequired checks:\n - `rg -n \"from benchbox\\\\.core\\\\.runner\\\\.dataframe_runner import|benchbox\\\\.core\\\\.runner\\\\.dataframe_runner|run_dataframe_benchmark\\\\(|is_dataframe_execution\\\\(|get_execution_mode\\\\(|_get_queries_for_benchmark\\\\(\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - `rg -n \"adapter\\\\.family|is_dataframe_mode|is_dataframe_platform|endswith\\\\(\\\\\\\"-df\\\\\\\"\\\\)|endswith\\\\('-df'\\\\)\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - Compare `benchbox/core/runner/dataframe_runner.py` TPC-H/TPC-DS query resolution against `benchbox/platforms/dataframe/benchmark_mixin.py`.\n - Check docs/reference/backward-compatibility.md and docs/reference/public-contracts.md for the current deprecated compatibility promise.\n\nDecision gate: do not delete `dataframe_runner.py` in this TODO unless\nthe compatibility registry and tests are updated under an explicit\nremoval decision. The expected path is shared helper extraction first.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate DataFrame execution-mode predicates, deprecated helper imports, and duplicated query resolution", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w0"], "notes": "Extract DataFrame benchmark query resolution into a single internal helper\nmodule. The helper should own:\n - benchmark ID normalization;\n - TPC-H stream permutation ordering;\n - TPC-DS standard stream generation;\n - TPC-DS variant lookup and fallback behavior;\n - ClickBench DataFrame query registry retrieval;\n - benchmark-instance `get_dataframe_queries()` fallback;\n - MagicMock/synthetic hook rejection if still needed.\n\nCandidate destination: `benchbox/core/dataframe/query_resolution.py` or a\nsimilarly focused core DataFrame module. Keep it internal unless the\npublic-contract map is updated.\n\nThe production mixin and dry-run query extraction must call the helper.\nThe deprecated runner may call the helper while remaining as a thin\ncompatibility wrapper.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create a shared DataFrame query-resolution helper used by mixin and dry-run compatibility paths", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w0"], "notes": "Add one predicate that can answer \"does this adapter/config/platform mean\nDataFrame execution?\" without relying on two unrelated sources of truth.\n\nThe predicate must account for:\n - explicit execution mode in database config;\n - `-df` suffix and deployment suffix handling;\n - PlatformRegistry default mode;\n - actual adapter DataFrame capability;\n - existing family values `expression` and `pandas`;\n - future DataFrame family values without modifying runner.py.\n\nCandidate shape: adapter property/protocol `is_dataframe` plus a factory\nresolver for config/platform strings. If a property is added to adapters,\ninclude compatibility fallback for older adapters during the migration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unify DataFrame execution-mode detection behind one predicate", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w1"], "notes": "Extract the exact bidirectional query filter behavior currently duplicated\nin `BenchmarkExecutionMixin._build_query_filter()` and\n`dataframe_runner._execute_dataframe_queries()`:\n - input `Q1` matches both `Q1` and `1`;\n - input `1` matches both `1` and `Q1`;\n - case-insensitive input;\n - variant IDs such as `Q14a` remain usable.\n\nDo not accidentally replace this with the existing one-way\n`normalize_query_id()` helper unless the helper is extended to preserve\nthe bidirectional filter-set contract.\n\nReconcile, but do not necessarily replace, SQL adapter\n`_apply_query_subset()`: it resolves against actual query dict keys and\nis related rather than identical.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace duplicated bidirectional Q-prefix filter normalization with one helper", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w1", "w2", "w3"], "notes": "Update `benchbox/cli/orchestrator.py` and `benchbox/core/dryrun.py` to use\nthe shared execution-mode and query-resolution helpers rather than\nimporting from `benchbox.core.runner.dataframe_runner`.\n\nLeave `dataframe_runner.py` as a deprecated internal compatibility module\nif tests or public-contract docs require it. Its helper functions should\ndelegate to the new shared helpers so behavior cannot drift.\n\nUpdate docs/reference/backward-compatibility.md only if the lifecycle\nstatus or removal timing changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove production-path dependency on deprecated dataframe_runner helpers while preserving compatibility imports", "wid": "w4"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w4"], "notes": "Tests must prove:\n - lifecycle runner, adapter factory, CLI/orchestrator, and dry-run agree\n on DataFrame-vs-SQL mode for representative platforms including\n `polars`, `polars-df`, `pandas-df`, `datafusion-df`, `duckdb`, and a\n deployment-suffixed DataFrame platform;\n - dry-run query extraction and production mixin query resolution return\n the same ordered query IDs for TPC-H and TPC-DS at representative\n stream IDs;\n - TPC-DS variant fallback strict/allow modes remain unchanged;\n - Q-prefix query subset filtering still accepts both prefixed and bare\n inputs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add parity tests for DataFrame routing, dry-run query extraction, and production mixin query resolution", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add another wrapper helper in runner.py that recreates the same field map under a new name.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "change result JSON key names for driver metadata as part of helper consolidation.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "remove runner platform_info driver metadata population without replacing it in the shared helper", "why": "existing exports may rely on those setdefault fields."}, {"INSTEAD": "(not specified)", "dont": "turn all validation exceptions into `None`", "why": "that would hide real failed validation attempts."}], "approach": "Centralize metadata propagation by strengthening the existing shared helper,\nthen reduce runner-local code to a wrapper. Treat validation applicability as\na small lifecycle contract: absent capability means no validation record;\nattempted validation failures remain failures.\n", "blocked_reason": null, "category": "Results", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-driver-metadata-and-validation-semantics", "id": 397, "reason": "This TODO fixes the confirmed not-applicable vs failed conflation; a typed validation model is larger result-schema work.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Redesign all validation stages as typed applicability/pass/fail objects"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-driver-metadata-and-validation-semantics", "id": 398, "reason": "Monitoring import-gate tests patch this helper; a compatibility wrapper is safer unless a focused audit proves no external/test reliance remains.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Removal gated on a focused reliance audit proving no external/test dependency on the patch point; audit not done.", "summary": "Remove `_enrich_driver_runtime_metadata()` public patch point immediately"}], "deps": ["architecture-contract-map-and-support-taxonomy", "result-schema-policy-and-explorer-input-gate"], "description": "Independent verification found two lifecycle/result consistency risks in\n`benchbox/core/runner/runner.py`.\n\nDriver metadata propagation exists in two places: the shared\n`benchbox/core/results/driver_metadata.py:apply_driver_metadata()` helper\nused by CLI/orchestrator paths, and runner-local\n`_enrich_driver_runtime_metadata()` plus `_apply_driver_meta_to_dicts()`.\nThe original critic overstated precedence divergence: both paths prefer\nadapter values over database config fallbacks. The real divergence is that\nthe runner also writes driver fields into `result.platform_info` with\n`setdefault()`, while the shared helper only writes result attributes and\n`execution_metadata`.\n\nPost-load validation also conflates not-applicable validation with failed\nvalidation. `_run_postload_validation()` returns `None` for `adapter is None`,\nbut returns a synthetic invalid `ValidationResult` when an adapter lacks\n`create_connection`/`close_connection`. Consumers already treat `None` as\n\"no validation record\" and aggregate `is_valid=False` as a failed stage.", "id": "execution-results-driver-metadata-and-validation-semantics", "preserves": ["Actual post-load validation exceptions still surface as failed validation, not as not-run.", "Adapter-sourced driver metadata remains higher precedence than database_config fallbacks.", "DataFrame post-load validation remains skipped unless a separate TODO changes DataFrame validation support.", "Driver metadata still serializes through schema-v2 platform/execution blocks as before.", "Existing platform_info keys are not overwritten by driver metadata propagation."], "prior_art": [{"concept": "benchbox/core/results/driver_metadata.py: shared metadata helper already used by CLI paths - extend this rather than keeping runner-local duplication.", "decision": "extend", "path": "benchbox/core/results/driver_metadata.py"}, {"concept": "benchbox/core/results/schema.py: driver metadata collection for export - verify exported fields remain stable.", "decision": "reuse", "path": "benchbox/core/results/schema.py"}, {"concept": "benchbox/core/runner/runner.py: current platform_info setdefault behavior - preserve while routing through shared helper.", "decision": "reuse", "path": "benchbox/core/runner/runner.py"}, {"concept": "tests/unit/cli/test_run_metadata_contract.py: existing shared-helper contract tests - expand for platform_info and runner parity.", "decision": "reuse", "path": "tests/unit/cli/test_run_metadata_contract.py"}, {"concept": "tests/unit/core/test_runner_helpers.py: existing post-load validation tests - update to the explicit applicability semantics.", "decision": "reuse", "path": "tests/unit/core/test_runner_helpers.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/driver_metadata.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "docs/reference/result-formats.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_run_metadata_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_runner_helpers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_runner_lifecycle.py"}], "state": "done", "title": "Execution/results hardening: unify driver metadata propagation and validation applicability semantics", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/test_run_metadata_contract.py tests/unit/core/test_runner_helpers.py -q", "description": "Driver metadata contract tests pass.", "expected": "Metadata and validation helper tests pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_runner_lifecycle.py tests/unit/core/results/test_schema_timing_contract.py --tb=short", "description": "Focused lifecycle tests pass.", "expected": "Lifecycle result metadata and schema contract tests pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite catches path parity regressions.", "expected": "Fast tests pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": [], "notes": "Re-run source checks before editing and capture a compact summary with\nchecked SHA, command, and counts. Do not commit raw stdout.\n\nRequired checks:\n - `rg -n \"apply_driver_metadata|_enrich_driver_runtime_metadata|_apply_driver_meta_to_dicts|_CONFIG_FIELD_MAP|_EXEC_META_SETDEFAULT\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - `rg -n \"_run_postload_validation|postload_result|validation_records|_finalize_validation_metadata|ValidationResult\\\\(is_valid=False\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - Inspect `tests/unit/core/test_runner_helpers.py` for tests pinning current missing-method invalid behavior.\n\nDecision gate: if result-schema policy or public-contract docs currently\npromise a failed stage for validation-not-applicable, preserve that public\nbehavior and document why this TODO should not change it.\n\nCompleted 2026-05-22 at checked SHA\n`f997b0aebc354f69bf32ca6b431dd4bd7e177404`: driver metadata rg returned\n28 lines, post-load validation rg returned 55 lines. Public contracts\nreserve `summary.validation=\"passed\"` for actual validation evidence and\nsay missing validation records finalize as `not_run`; no contract requires\nunsupported connection validation to record a failed stage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate driver metadata call sites and post-load validation consumers", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w0"], "notes": "Update `benchbox/core/results/driver_metadata.py` so the shared helper\ncan also populate `result.platform_info` using the same `setdefault()`\nbehavior currently present in runner-local `_apply_driver_meta_to_dicts()`.\n\nPreserve these semantics:\n - adapter values win over database config values;\n - config values fill only missing adapter values;\n - `driver_version_resolved` falls back to requested `driver_version`;\n - `driver_package` and `driver_version_requested` in execution metadata\n use `setdefault`;\n - `driver_auto_install_used` remains recorded even when false;\n - platform_info population never overwrites existing platform metadata.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend the shared driver metadata helper to preserve runner platform_info behavior", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w1"], "notes": "Replace the runner-local duplication in\n`_enrich_driver_runtime_metadata()` with calls to\n`apply_driver_metadata(...)`. Delete or shrink `_DRIVER_STR_FIELDS`,\n`_CONFIG_FIELD_MAP`, `_EXEC_META_SETDEFAULT`, and\n`_apply_driver_meta_to_dicts()` if they become unused.\n\nKeep `_enrich_driver_runtime_metadata()` as a small compatibility wrapper\nif tests or monitoring import gates patch it directly. The wrapper should\nbe boring: call the shared helper and return the result.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Route runner lifecycle metadata enrichment through the shared helper", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w0"], "notes": "Decide and implement one explicit semantic rule:\n - not applicable: return `None`, do not add a validation stage;\n - attempted and failed: return `ValidationResult(is_valid=False)`;\n - attempted and passed/warned: return the engine result.\n\nApply that rule to `_run_postload_validation()`. The likely change is\nreturning `None` when the adapter lacks `create_connection` or\n`close_connection`, while preserving synthetic failure for exceptions\nraised after validation is actually attempted.\n\nUpdate tests currently expecting invalid for missing adapter methods to\nassert `None` if the public-contract check in w0 allows the change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define validation applicability semantics and update post-load validation behavior", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w2", "w3"], "notes": "Cover:\n - CLI/orchestrator path and runner lifecycle path produce the same\n top-level driver fields and `execution_metadata` fields;\n - runner path still backfills `platform_info` without overwriting\n existing platform_info values;\n - missing connection validation hooks produce no validation stage after\n the semantics change;\n - actual post-load validation exceptions still produce a failed stage.\n\nPrefer focused unit tests around `apply_driver_metadata()`,\n`_enrich_driver_runtime_metadata()`, `_run_postload_validation()`, and\n`_finalize_validation_metadata()` before broad lifecycle tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add metadata and validation regression tests for lifecycle and CLI parity", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add a catch-all dict that bypasses schema-v2 validation policy.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "delete `_benchmark_id_override` until result filename, loader, and builder behavior have an explicit replacement.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "move `native_comparison` or migration fields into `platform_extensions` in one step without loader, explorer, MCP, and public-submission tests.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "treat the dataclass field count as the user-facing problem", "why": "exported result compatibility is the primary risk."}], "approach": "Treat `BenchmarkResults` cleanup as a compatibility design problem, not a\nmechanical field move. First document the extension policy, then inventory\ncurrent fields and consumers, then add contract tests, and only then make a\nminimal internal-only or backward-compatible change.\n", "blocked_reason": null, "category": "Results", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-result-model-extension-contract", "id": 399, "reason": "This is the risky change the TODO is designed to evaluate; do it only after policy, inventory, and consumer tests prove compatibility.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Move all existing platform-specific result fields into `platform_extensions`"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-result-model-extension-contract", "id": 400, "reason": "Result schema policy currently supports schema-v2 consumers; a major break requires a separate migration plan.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: A major result-schema version bump needs a separate migration plan that doesn't exist; current schema-v2 policy still holds.", "summary": "Bump result schema major version for a broad model cleanup"}], "deps": ["architecture-contract-map-and-support-taxonomy", "result-schema-policy-and-explorer-input-gate"], "description": "Independent verification partially confirmed that `BenchmarkResults` is doing\ntoo much, but also found that the original critic overstated the facts. The\ndataclass currently has 78 fields, not roughly 205. It has several platform\nmetadata dictionaries, `_benchmark_id_override`, and a platform-specific\n`native_comparison` field. The claimed `migration` top-level model field is\nnot a `BenchmarkResults` field; schema-v2 exports migration as a phase when\n`ExecutionPhases.migration` is present, while a separate pg_mooncake script\nemits a top-level `migration` block.\n\nThe suggested fix, moving platform-specific results into\n`platform_extensions: dict[str, Any]`, is therefore directionally useful but\nrisky. Result JSON is a product boundary consumed by runtime loaders, CLI\ncompare/export/show, MCP tools, public submissions, hosted result bundles,\nand the explorer pipeline. This TODO creates the migration contract and a\nnarrow extension policy before any field move.", "id": "execution-results-result-model-extension-contract", "preserves": ["Do not conflate internal dataclass cleanup with exported JSON schema cleanup.", "Existing schema-v2 result bundles remain loadable by `load_result_file()`.", "Explorer pipeline input gate behavior from result-schema-policy work is not weakened.", "No existing top-level exported key is removed without same-PR compatibility and docs updates.", "Public submission validation remains compatible with accepted schema-v2 bundles."], "prior_art": [{"concept": "_project/scripts/explorer_pipeline/transformer.py: hosted/explorer consumer - include in downstream compatibility checks.", "decision": "reuse", "path": "_project/scripts/explorer_pipeline/transformer.py"}, {"concept": "benchbox/core/results/loader.py: reconstruction contract for historical result files - test before moving fields.", "decision": "reuse", "path": "benchbox/core/results/loader.py"}, {"concept": "benchbox/core/results/schema.py: current schema-v2 payload builder - exported keys are the compatibility boundary.", "decision": "reuse", "path": "benchbox/core/results/schema.py"}, {"concept": "benchbox/core/results/schema_policy.py: consumer-specific schema acceptance - preserve these risk postures.", "decision": "reuse", "path": "benchbox/core/results/schema_policy.py"}, {"concept": "docs/reference/public-contracts.md: result JSON bundle contract tier - extend before moving exported fields.", "decision": "extend", "path": "docs/reference/public-contracts.md"}, {"concept": "scripts/benchmark_mooncake_migration.py: standalone top-level migration payload - decide whether to align or leave separate.", "decision": "reuse", "path": "scripts/benchmark_mooncake_migration.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/results.py"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "docs/reference/result-formats.md"}, {"kind": "only_modify", "path_glob": "scripts/benchmark_mooncake_migration.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/commands/test_submit.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}], "state": "done", "title": "Execution/results hardening: define result-model extension contract before platform-specific field migration", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/results -q", "description": "Focused result schema/model tests pass.", "expected": "Result model, loader, schema, and normalizer tests pass.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline tests/unit/cli/commands/test_submit.py --tb=short", "description": "Explorer and submission consumers remain compatible.", "expected": "Explorer pipeline and submission command tests pass.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/validate_submission.py results-data/bundles/", "description": "Published bundle validator still accepts existing corpus.", "expected": "Existing public result bundles validate.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": [], "notes": "Re-run the current-tree evidence before proposing schema or model changes.\nCapture checked SHA, commands, and compact results in the implementation\nPR body or TODO notes; do not commit raw stdout.\n\nRequired checks:\n - `uv run -- python - <<'PY'` snippet using `dataclasses.fields(BenchmarkResults)` to print field count and platform/native/migration-related fields.\n - `rg -n \"native_comparison|\\\\[\\\\\\\"migration\\\\\\\"\\\\]|platform_info|platform_metadata|platform_raw|BenchmarkResults\\\\(|build_result_payload|reconstruct_benchmark_results|load_result_file|ResultExporter|PUBLIC_SUBMISSION_SCHEMA_POLICY\" benchbox _project scripts tests docs/reference --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - Inspect `benchbox/core/results/schema.py`, `benchbox/core/results/loader.py`, `benchbox/core/results/exporter.py`, `benchbox/mcp/tools/benchmark.py`, `benchbox/mcp/tools/results.py`, and `_project/scripts/explorer_pipeline/`.\n\nDecision gate: do not move or remove any serialized result key in the\nimplementation PR unless the migration policy explicitly classifies the\nchange as backward-compatible or includes a same-PR migration/loader gate.\n\nCompleted 2026-05-22 at checked SHA\n`185006fde1a04b460e4c30d52d4035afeb1861eb`: dataclass field count is 78;\nplatform/native/migration-related fields are `platform`,\n`platform_deployment`, `platform_cloud`, `platform_compute`,\n`platform_storage`, `platform_raw_config`, `platform_raw_metadata`,\n`platform_info`, `platform_metadata`, and `native_comparison`;\n`BenchmarkResults` has no `migration` field. Consumer rg returned 2504\nbroad hits. Inspection confirmed `comparisons.native_duckdb`,\n`phases.migration`, platform metadata blocks, loader reconstruction, MCP\nresult loading, and explorer projection are the relevant compatibility\nboundaries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate BenchmarkResults field count, platform-specific fields, and schema consumers", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w0"], "notes": "Add a policy section to docs/reference/public-contracts.md or a focused\nresult-schema doc that answers:\n - where platform-specific structured data should live in the internal\n model;\n - where it should serialize in schema-v2 payloads;\n - when top-level schema keys are allowed;\n - when a new `platform_extensions` block is acceptable;\n - how loader, normalizer, public submission, and explorer consumers\n must handle unknown extension keys;\n - what evidence is required before moving an existing field.\n\nThe policy must distinguish internal dataclass fields from exported JSON\nkeys. Do not let a model cleanup imply a schema break.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write a result-model extension policy covering platform-specific data and schema-v2 compatibility", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w1"], "notes": "Build a concise inventory covering at least:\n - `native_comparison` and its `comparisons.native_duckdb` export;\n - `ExecutionPhases.migration` and `phases.migration` export;\n - the standalone pg_mooncake migration script's top-level `migration`\n payload;\n - `platform_info`, `platform_metadata`, `platform_raw_config`,\n `platform_raw_metadata`;\n - `platform_deployment`, `platform_cloud`, `platform_compute`,\n `platform_storage`;\n - `execution_context` and `_benchmark_id_override`.\n\nFor each row record current owner, exported key, loader reconstruction\nbehavior, downstream consumers, and recommended action:\n - keep;\n - alias;\n - move internally only;\n - migrate exported schema with compatibility;\n - reject as not worth it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create an extension inventory and migration matrix for current platform-specific fields", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w2"], "notes": "Before changing model shape, add focused tests or fixtures that prove\ncurrent behavior for `comparisons.native_duckdb`, `phases.migration`, and\nplatform metadata blocks through:\n - `build_result_payload()`;\n - `reconstruct_benchmark_results()`;\n - public submission validation where applicable;\n - explorer pipeline transformation where applicable.\n\nIf the matrix chooses no field migration yet, this work still protects\nfuture refactors by making the current contract explicit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add consumer tests or fixtures that lock current extension behavior before migration", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w3"], "notes": "Choose one narrow change from the w2 matrix that does not break exported\nresult JSON. Examples:\n - add an internal `platform_extensions` field while continuing to\n serialize existing keys unchanged;\n - route one new platform-specific result through an extension block\n without moving existing keys;\n - add loader passthrough for unknown extension blocks while preserving\n known fields.\n\nIf every field move is risky, record that as the outcome and stop after\npolicy, inventory, and tests. A \"no migration yet\" result is acceptable\nif it is evidence-backed.\n\nOutcome: no existing field was moved. The only code cleanup is accepting\nthe existing producer-emitted top-level `comparisons` key in schema-v2\nvalidation. Current extension locations are locked by tests and policy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement only the lowest-risk internal extension cleanup approved by the matrix", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change exported result keys such as `summary.tpc_metrics` or query result `test_type` while cleaning up internals.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "delete `benchbox/core/tpc_metrics.py` solely because static import grep is clean", "why": "first audit whether its validation/reporting behavior is needed elsewhere."}, {"INSTEAD": "(not specified)", "dont": "introduce a new generic TPC execution framework in this TODO", "why": "the confirmed duplication is result conversion, not the entire driver lifecycle."}, {"INSTEAD": "(not specified)", "dont": "replace explicit failure sentinels with empty query lists", "why": "that can make failed power-test setup look like a successful run with zero queries."}], "approach": "Treat the legacy TPC metrics module as a retirement candidate, not as already\nsafe to delete. First prove the import graph and unique behavior. Then either\ndelete it or fold the useful surface into the canonical results helpers. In\nparallel, extract the small repeated power-result dict construction logic in\nthe base execution adapter without changing the surrounding TPC driver flow.\n", "blocked_reason": null, "category": "Results", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-tpc-metrics-and-power-result-normalization", "id": 401, "reason": "This TODO removes confirmed duplication in the power conversion path only; a broader driver rewrite needs separate perf and compliance risk review.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: The confirmed power-conversion duplication was already removed by this item. A full rewrite of ALL TPC-H/DS power+throughput drivers around a shared adapter is speculative on a compliance-critical path (the item flags it needs separate perf/compliance risk review); no remaining defect demonstrated.", "summary": "Rewrite all TPC-H/TPC-DS power and throughput execution drivers around one shared result adapter"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-tpc-metrics-and-power-result-normalization", "id": 402, "reason": "Only do this if w1 proves the legacy reporter has product value; otherwise it is dead internal surface.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on w1 proving legacy reporter has product value; otherwise explicitly 'dead internal surface' \u2014 no evidence condition met.", "summary": "Promote rich metrics reporting to a user-facing CLI command"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Independent verification of the execution/results architecture review found\ntwo adjacent duplication risks in the TPC metrics and power-test result path.\n\nFirst, `benchbox/core/tpc_metrics.py` is an 877-line legacy module with no\nproduction importers in the current tree. Production code imports\n`benchbox.core.results.metrics.TPCMetricsCalculator`, which is re-exported\nfrom `benchbox.core.results`. Only `tests/unit/core/test_tpc_metrics.py`\nimports the legacy module. The module is therefore a deletion candidate, but\nit contains richer validation/reporting classes that should be audited before\nremoval so useful behavior is not silently lost.\n\nSecond, `benchbox/platforms/base/execution.py` repeats TPC-H and TPC-DS\npower-test result-dict conversion in four blocks, plus four\n`power_test_error` sentinels. The duplication is smaller than the original\ncritic claimed, but it is still enough to make failure/result schema drift\nlikely when one TPC driver changes.", "id": "execution-results-tpc-metrics-and-power-result-normalization", "preserves": ["Do not weaken TPC compliance or row-count validation behavior as part of cleanup.", "No result JSON schema change from metrics-module retirement or conversion-helper extraction.", "Power-test failure sentinels still make all-query factory failures visible as FAILED runs.", "TPC-H and TPC-DS Power@Size calculations continue using the canonical `results.metrics.TPCMetricsCalculator` formulas."], "prior_art": [{"concept": "benchbox/core/results/builder.py: result-builder metric population - preserve as the production metric source.", "decision": "reuse", "path": "benchbox/core/results/builder.py"}, {"concept": "benchbox/core/results/metrics.py: canonical lightweight TPC metric formulas - extend this rather than keeping a second calculator.", "decision": "extend", "path": "benchbox/core/results/metrics.py"}, {"concept": "benchbox/platforms/base/execution.py: current TPC-H/TPC-DS power conversion blocks - consolidate locally first.", "decision": "reuse", "path": "benchbox/platforms/base/execution.py"}, {"concept": "docs/reference/public-contracts.md: contract tier map - consult before deleting any import path that may be public.", "decision": "reuse", "path": "docs/reference/public-contracts.md"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/builder.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpc_metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "only_modify", "path_glob": "docs/reference/backward-compatibility.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_metrics.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_tpc_metrics.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter.py"}], "state": "done", "title": "Execution/results hardening: retire legacy TPC metrics and normalize power-result conversion", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/results/test_metrics.py tests/unit/platforms/test_base_adapter.py -q", "description": "Focused TPC metrics and base adapter tests pass.", "expected": "All focused metrics and base-adapter tests pass.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 1}, {"command": "rg -n \"^from benchbox\\.core\\.tpc_metrics import|^import benchbox\\.core\\.tpc_metrics|from benchbox\\.core import tpc_metrics|benchbox\\.core\\.tpc_metrics\" benchbox tests pyproject.toml --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'", "description": "Legacy TPC metrics import graph has an explicit terminal state.", "expected": "No matches if deletion path is chosen; otherwise only documented compatibility imports remain and the PR explains why deletion was rejected.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite catches result-shape regressions.", "expected": "Fast tests pass.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": [], "notes": "Revalidated against checked SHA\n`7a4262f8f6457e97b334e6132c9e58b2640c785f`. `wc -l` reported\n`benchbox/core/tpc_metrics.py` at 877 lines and\n`benchbox/core/results/metrics.py` at 211 lines. The legacy import grep\nfound only `tests/unit/core/test_tpc_metrics.py`; no production importer\nexists in the current tree. The canonical results metrics grep found\nproduction and test use of `benchbox.core.results.metrics` /\n`TPCMetricsCalculator`. The power conversion grep found the duplicated\nTPC-H/TPC-DS normal result blocks and `power_test_error` sentinels in\n`benchbox/platforms/base/execution.py`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate TPC metrics import graph, unique legacy surface, and power-result conversion count", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w0"], "notes": "Audit decision table:\n - Power@Size, Throughput@Size, and composite Qph formulas are covered\n by `benchbox/core/results/metrics.py`; focused tests now pin the\n legacy formula expectations on the canonical calculator.\n - `PowerMetrics`, `ThroughputMetrics`, `CompositeMetrics`,\n `MetricsValidator`, and `calculate_full_metrics()` are unimported\n legacy object/reporting surfaces, not part of current schema-v2\n result construction.\n - `StatisticalAnalyzer` and `MetricsReporter` may be useful product\n ideas, but they are uncalled internal reporting helpers and should be\n promoted only through a separate CLI/reporting TODO.\nNo unique production-useful behavior required migration before deleting\nthe legacy module.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit legacy tpc_metrics.py for behavior not present in results.metrics", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w1"], "notes": "Deleted `benchbox/core/tpc_metrics.py` and\n`tests/unit/core/test_tpc_metrics.py`. No public/backward-compat docs\nwere changed because the revalidated import graph did not show an\nintentionally supported import path. Removed the stale mutation-tool\nexclusion for the deleted module from `pyproject.toml` as a narrow scope\nexception found during self-review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire or fold the legacy TPC metrics module through an explicit compatibility decision", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w0"], "notes": "Added `_power_query_result()` and `_power_test_error_result()` in\n`benchbox/platforms/base/execution.py` and replaced the duplicated\nTPC-H/TPC-DS power conversion blocks and sentinels. Throughput,\nmaintenance, and generic power conversion paths were left unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract power-test result conversion and sentinel construction helpers", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w2", "w3"], "notes": "Added canonical formula coverage in\n`tests/unit/core/results/test_metrics.py` and conversion-helper coverage\nin `tests/unit/platforms/test_base_adapter.py`. The final import grep is\nexpected to terminate with no matches for `benchbox.core.tpc_metrics`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add focused regression tests for canonical TPC metrics and power-result conversion", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add a second dispatch mechanism; extend resolve_*_harness.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "change execution semantics while extracting", "instead": "(not specified)", "why": "this is a structural move only, proven by unchanged results on the correctness/equivalence gates."}], "approach": null, "blocked_reason": null, "category": "Code Quality and Technical Debt", "claimed_at": null, "claimed_by": null, "completed_at": "2026-06-30T00:00:00Z", "completed_pr": null, "created_at": "2026-06-30T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-harness-extraction-throughput-maintenance-combined", "id": 393, "reason": "Only TPC has multi-phase power/throughput/maintenance shapes today; generalization is speculative until a second family needs it.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Generalizing the multi-phase harness registry to non-TPC families is already tracked as an open planning item. (tracked by generalize-multiphase-harness-registry-non-tpc)", "summary": "Generalize the harness registry to non-TPC benchmark families."}], "deps": [], "description": "Follow-up to structural-debt-base-layer-tpc-and-module-split (PR #914, DONE).\n#914's w1 success_metric \u2014 \"platforms/base/execution.py carries no literal TPC\nbenchmark names in its dispatch path\" \u2014 was met ONLY for the power-test path:\n_execute_power_test (execution.py:682 on origin/develop) now dispatches via\nresolve_power_harness(benchmark_id) with zero literals. But _execute_throughput_test,\n_execute_maintenance_test, and _execute_combined_test still branch on literal\n`benchmark_id == \"tpch\"` / `== \"tpcds\"` (origin/develop execution.py ~713-779).\nThe guard test test_power_dispatch_path_has_no_literal_tpc_benchmark_names\ninspects only _execute_power_test, so the broader metric reads as met when it is\nnot. This item extends the capability-registry dispatch to the remaining three\ntest types and broadens the guard test to all four.\n\nBehavior-preserving refactor: throughput/maintenance/combined carry stream/cursor\nand connection-factory wiring (the TPC-DS path uses connection_factory +\n_make_stream_cursor for thread-safe cursors); that wiring must be preserved\nexactly, as #914 did for the power path.", "id": "execution-harness-extraction-throughput-maintenance-combined", "preserves": ["Byte-identical benchmark execution: query order, stream/cursor wiring, connection_factory + _make_stream_cursor thread-safe semantics, set_config_validation_mode call point.", "Public dispatch entry points (_execute_throughput_test etc.) keep their signatures and call contracts."], "prior_art": [{"concept": "{'ref': 'benchbox/core/power_harnesses.py', 'note': 'resolve_power_harness + the per-benchmark power harness modules #914 introduced (core/tpch/platform_power.py, core/tpcds/platform_power.py). Extend this registry with throughput/maintenance/combined harness resolution; do not invent a parallel dispatch.'}", "decision": "extend", "path": "{'ref'"}, {"concept": "{'ref': 'benchbox/platforms/base/execution.py:682', 'note': 'The already-extracted _execute_power_test as the template: capability lookup, harness invocation, preserved connection/cursor wiring.'}", "decision": "reuse", "path": "{'ref'"}, {"concept": "{'ref': 'tests/unit/platforms/', 'note': 'test_power_dispatch_path_has_no_literal_tpc_benchmark_names \u2014 broaden it to assert no literal TPC names across all four dispatch methods.'}", "decision": "reuse", "path": "{'ref'"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/power_harnesses.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpcds/"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpch/"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "only_modify", "path_glob": "tests/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/"}], "state": "done", "title": "Extend capability-based power-harness dispatch to throughput, maintenance, and combined tests", "verifications": [{"command": "uv run -- python -m pytest tests/unit/platforms/ -k literal_tpc_benchmark_names -q", "description": "No literal TPC names remain in any execution.py dispatch method", "expected": "guard test asserts power, throughput, maintenance, and combined dispatch carry no literal benchmark_id == tpch/tpcds branches", "item_id": "execution-harness-extraction-throughput-maintenance-combined", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests -m fast -k 'execution or power or throughput or maintenance' -q", "description": "Benchmark execution is unchanged", "expected": "execution behavior unchanged; all green", "item_id": "execution-harness-extraction-throughput-maintenance-combined", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": [], "notes": "Grep execution.py for `benchmark_id == \"tpc` and record each method + line\n(line numbers in this TODO are origin/develop at review time and will drift).\nConfirm the guard test currently inspects only _execute_power_test. Summarize\nin the PR body.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm the residual literals and the guard test scope on fresh develop", "wid": "w0"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w0"], "notes": "Move the TPC-H/TPC-DS throughput bodies behind a resolve_*_harness lookup,\npreserving stream/cursor and connection-factory wiring exactly. Per-benchmark\nbodies live under core/tpc{h,ds}/, mirroring the power split.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract throughput dispatch to capability resolution", "wid": "w1"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w0"], "notes": "Same treatment for _execute_maintenance_test, preserving the\nset_config_validation_mode call point and DML/refresh semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract maintenance dispatch to capability resolution", "wid": "w2"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w1", "w2"], "notes": "_execute_combined_test composes power+throughput+maintenance; route it through\nthe now-extracted harnesses. No literal benchmark_id branch should remain.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract combined dispatch to capability resolution", "wid": "w3"}, {"evidence": null, "item_id": "execution-harness-extraction-throughput-maintenance-combined", "needs": ["w3"], "notes": "Extend test_power_dispatch_path_has_no_literal_tpc_benchmark_names (or add a\nsibling) to assert no literal TPC names in throughput/maintenance/combined too.\nThen correct the #914 DONE success_metric back to the full-dispatch claim (or\ncoordinate with holistic-review-followup-metadata-truth, which reworded it to\nthe interim power-path-only wording).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Broaden the guard test to all four dispatch methods and restore the strong metric", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "decide DataFrame mode in runner.py by hard-coding family names again", "instead": "(not specified)", "why": "future families must not require runner edits."}, {"dont": "delete `dataframe_runner.py` just because it is deprecated", "instead": "(not specified)", "why": "dry-run and tests still import it."}, {"dont": "import optional DataFrame libraries in mode-detection helpers.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "let dry-run call a different TPC-DS variant fallback implementation than real execution.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "replace bidirectional Q-prefix filters with canonical stripping unless tests prove all caller contracts still work.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Split the work into two small centers of gravity: one query-resolution helper\nand one execution-mode predicate. Route production mixin, dry-run, CLI, and\ndeprecated compatibility helpers through those centers. Keep compatibility\nwrappers until deletion is separately authorized.\n", "blocked_reason": null, "category": "DataFrame Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-21T00:00:00Z", "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-dataframe-routing-and-query-resolution", "id": 394, "reason": "The current public/backward-compat registry describes it as a deprecated internal compatibility runner; deletion needs a separate removal-window decision after helper imports migrate.", "resolution": "promoted", "resolved_item": "delete-benchbox-core-runner-dataframe-runner-py-completely", "resolved_reason": null, "summary": "Delete `benchbox/core/runner/dataframe_runner.py` completely"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-dataframe-routing-and-query-resolution", "id": 395, "reason": "This TODO centralizes existing routing/query-resolution behavior; broader registry redesign should wait until drift is removed.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Broader DataFrame registry redesign explicitly deferred until routing/query-resolution drift is removed; no evidence of that, no open item tracks it.", "summary": "Redesign DataFrame query registries for all benchmark families"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-dataframe-routing-and-query-resolution", "id": 396, "reason": "Can be done after the shared helper lands and tests stop requiring MagicMock internals.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: dataframe_runner.py/benchmark_mixin.py housing the MagicMock-aware hook detection are marked deprecated-internal-compatibility-runner in favor of adapter.run_benchmark+BenchmarkExecutionMixin; cleanup moot once removed.", "summary": "Remove MagicMock-aware hook handling from production modules"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Independent verification confirmed that DataFrame execution has two related\nsources of drift.\n\nThe deprecated internal compatibility module\n`benchbox/core/runner/dataframe_runner.py` is still load-bearing for CLI and\ndry-run helpers. Production DataFrame execution uses\n`adapter.run_benchmark()` through\n`benchbox/platforms/dataframe/benchmark_mixin.py`, but dry-run imports\n`get_execution_mode()` and `_get_queries_for_benchmark()` from the deprecated\nmodule. Both the deprecated runner and the production mixin duplicate TPC-H\nstream ordering, TPC-DS stream/variant fallback, and bidirectional query\nsubset filters.\n\nDataFrame-vs-SQL routing is also detected two ways. The lifecycle runner\ndecides DataFrame execution from adapter `family in (\"expression\", \"pandas\")`,\nwhile adapter factory, capabilities, CLI/dry-run helpers, and the deprecated\nrunner use platform suffix/default-mode logic. A new DataFrame family or\nregistry default can therefore disagree with the lifecycle branch.", "id": "execution-results-dataframe-routing-and-query-resolution", "preserves": ["Dry-run DataFrame query previews remain available for supported DataFrame benchmarks.", "Existing `dataframe_runner.py` compatibility imports keep working unless the public-contract map explicitly approves removal.", "No new DataFrame platform dependency is imported eagerly.", "Production DataFrame execution remains `adapter.run_benchmark()` plus `BenchmarkExecutionMixin`.", "TPC-H stream permutation and TPC-DS stream/variant fallback behavior does not change."], "prior_art": [{"concept": "benchbox/core/results/query_normalizer.py: existing one-way query ID normalization - reuse only if extended to support bidirectional filter sets.", "decision": "reuse", "path": "benchbox/core/results/query_normalizer.py"}, {"concept": "benchbox/core/runner/dataframe_runner.py: deprecated compatibility runner and current dry-run helper source - delegate rather than deleting first.", "decision": "reuse", "path": "benchbox/core/runner/dataframe_runner.py"}, {"concept": "benchbox/platforms/__init__.py: current `is_dataframe_platform` helper - reconcile with broader predicate rather than adding another list.", "decision": "reuse", "path": "benchbox/platforms/__init__.py"}, {"concept": "benchbox/platforms/adapter_factory.py: platform string normalization and default-mode logic - reuse for config/platform predicate.", "decision": "reuse", "path": "benchbox/platforms/adapter_factory.py"}, {"concept": "benchbox/platforms/dataframe/benchmark_mixin.py: production DataFrame adapter execution path - preserve behavior while extracting helpers.", "decision": "reuse", "path": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"concept": "docs/reference/backward-compatibility.md: deprecated dataframe_runner lifecycle row - preserve or update explicitly.", "decision": "reuse", "path": "docs/reference/backward-compatibility.md"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/cli/orchestrator.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/dataframe/"}, {"kind": "only_modify", "path_glob": "benchbox/core/dryrun.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/query_normalizer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/adapter_factory.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/dataframe/benchmark_mixin.py"}, {"kind": "only_modify", "path_glob": "docs/reference/backward-compatibility.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_cli_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/runner/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_dryrun_behavioral.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/dataframe/"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_dataframe_factory.py"}], "state": "done", "title": "Execution/results hardening: unify DataFrame routing, query resolution, and query-id filters", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/runner/test_dataframe_runner.py tests/unit/core/runner/test_dataframe_runner_lifecycle.py tests/unit/platforms/dataframe/test_benchmark_mixin.py tests/unit/platforms/test_dataframe_factory.py -q", "description": "Focused DataFrame routing and compatibility tests pass.", "expected": "Focused DataFrame runner, mixin, and factory tests pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_dryrun.py tests/unit/core/test_dryrun_behavioral.py tests/unit/cli/test_cli_dryrun.py --tb=short", "description": "Dry-run tests pass after moving query extraction off deprecated helpers.", "expected": "Dry-run DataFrame query extraction tests pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite catches execution-mode routing regressions.", "expected": "Fast tests pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-dataframe-routing-and-query-resolution", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": [], "notes": "Re-run current-tree checks before editing and record checked SHA plus a\ncompact evidence summary in the implementation PR body or TODO notes.\n\nRequired checks:\n - `rg -n \"from benchbox\\\\.core\\\\.runner\\\\.dataframe_runner import|benchbox\\\\.core\\\\.runner\\\\.dataframe_runner|run_dataframe_benchmark\\\\(|is_dataframe_execution\\\\(|get_execution_mode\\\\(|_get_queries_for_benchmark\\\\(\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - `rg -n \"adapter\\\\.family|is_dataframe_mode|is_dataframe_platform|endswith\\\\(\\\\\\\"-df\\\\\\\"\\\\)|endswith\\\\('-df'\\\\)\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - Compare `benchbox/core/runner/dataframe_runner.py` TPC-H/TPC-DS query resolution against `benchbox/platforms/dataframe/benchmark_mixin.py`.\n - Check docs/reference/backward-compatibility.md and docs/reference/public-contracts.md for the current deprecated compatibility promise.\n\nDecision gate: do not delete `dataframe_runner.py` in this TODO unless\nthe compatibility registry and tests are updated under an explicit\nremoval decision. The expected path is shared helper extraction first.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate DataFrame execution-mode predicates, deprecated helper imports, and duplicated query resolution", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w0"], "notes": "Extract DataFrame benchmark query resolution into a single internal helper\nmodule. The helper should own:\n - benchmark ID normalization;\n - TPC-H stream permutation ordering;\n - TPC-DS standard stream generation;\n - TPC-DS variant lookup and fallback behavior;\n - ClickBench DataFrame query registry retrieval;\n - benchmark-instance `get_dataframe_queries()` fallback;\n - MagicMock/synthetic hook rejection if still needed.\n\nCandidate destination: `benchbox/core/dataframe/query_resolution.py` or a\nsimilarly focused core DataFrame module. Keep it internal unless the\npublic-contract map is updated.\n\nThe production mixin and dry-run query extraction must call the helper.\nThe deprecated runner may call the helper while remaining as a thin\ncompatibility wrapper.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create a shared DataFrame query-resolution helper used by mixin and dry-run compatibility paths", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w0"], "notes": "Add one predicate that can answer \"does this adapter/config/platform mean\nDataFrame execution?\" without relying on two unrelated sources of truth.\n\nThe predicate must account for:\n - explicit execution mode in database config;\n - `-df` suffix and deployment suffix handling;\n - PlatformRegistry default mode;\n - actual adapter DataFrame capability;\n - existing family values `expression` and `pandas`;\n - future DataFrame family values without modifying runner.py.\n\nCandidate shape: adapter property/protocol `is_dataframe` plus a factory\nresolver for config/platform strings. If a property is added to adapters,\ninclude compatibility fallback for older adapters during the migration.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Unify DataFrame execution-mode detection behind one predicate", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w1"], "notes": "Extract the exact bidirectional query filter behavior currently duplicated\nin `BenchmarkExecutionMixin._build_query_filter()` and\n`dataframe_runner._execute_dataframe_queries()`:\n - input `Q1` matches both `Q1` and `1`;\n - input `1` matches both `1` and `Q1`;\n - case-insensitive input;\n - variant IDs such as `Q14a` remain usable.\n\nDo not accidentally replace this with the existing one-way\n`normalize_query_id()` helper unless the helper is extended to preserve\nthe bidirectional filter-set contract.\n\nReconcile, but do not necessarily replace, SQL adapter\n`_apply_query_subset()`: it resolves against actual query dict keys and\nis related rather than identical.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace duplicated bidirectional Q-prefix filter normalization with one helper", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w1", "w2", "w3"], "notes": "Update `benchbox/cli/orchestrator.py` and `benchbox/core/dryrun.py` to use\nthe shared execution-mode and query-resolution helpers rather than\nimporting from `benchbox.core.runner.dataframe_runner`.\n\nLeave `dataframe_runner.py` as a deprecated internal compatibility module\nif tests or public-contract docs require it. Its helper functions should\ndelegate to the new shared helpers so behavior cannot drift.\n\nUpdate docs/reference/backward-compatibility.md only if the lifecycle\nstatus or removal timing changes.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Remove production-path dependency on deprecated dataframe_runner helpers while preserving compatibility imports", "wid": "w4"}, {"evidence": null, "item_id": "execution-results-dataframe-routing-and-query-resolution", "needs": ["w4"], "notes": "Tests must prove:\n - lifecycle runner, adapter factory, CLI/orchestrator, and dry-run agree\n on DataFrame-vs-SQL mode for representative platforms including\n `polars`, `polars-df`, `pandas-df`, `datafusion-df`, `duckdb`, and a\n deployment-suffixed DataFrame platform;\n - dry-run query extraction and production mixin query resolution return\n the same ordered query IDs for TPC-H and TPC-DS at representative\n stream IDs;\n - TPC-DS variant fallback strict/allow modes remain unchanged;\n - Q-prefix query subset filtering still accepts both prefixed and bare\n inputs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add parity tests for DataFrame routing, dry-run query extraction, and production mixin query resolution", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add another wrapper helper in runner.py that recreates the same field map under a new name.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "change result JSON key names for driver metadata as part of helper consolidation.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "remove runner platform_info driver metadata population without replacing it in the shared helper", "instead": "(not specified)", "why": "existing exports may rely on those setdefault fields."}, {"dont": "turn all validation exceptions into `None`", "instead": "(not specified)", "why": "that would hide real failed validation attempts."}], "approach": "Centralize metadata propagation by strengthening the existing shared helper,\nthen reduce runner-local code to a wrapper. Treat validation applicability as\na small lifecycle contract: absent capability means no validation record;\nattempted validation failures remain failures.\n", "blocked_reason": null, "category": "Results", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-driver-metadata-and-validation-semantics", "id": 397, "reason": "This TODO fixes the confirmed not-applicable vs failed conflation; a typed validation model is larger result-schema work.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Redesign all validation stages as typed applicability/pass/fail objects"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-driver-metadata-and-validation-semantics", "id": 398, "reason": "Monitoring import-gate tests patch this helper; a compatibility wrapper is safer unless a focused audit proves no external/test reliance remains.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Removal gated on a focused reliance audit proving no external/test dependency on the patch point; audit not done.", "summary": "Remove `_enrich_driver_runtime_metadata()` public patch point immediately"}], "deps": ["architecture-contract-map-and-support-taxonomy", "result-schema-policy-and-explorer-input-gate"], "description": "Independent verification found two lifecycle/result consistency risks in\n`benchbox/core/runner/runner.py`.\n\nDriver metadata propagation exists in two places: the shared\n`benchbox/core/results/driver_metadata.py:apply_driver_metadata()` helper\nused by CLI/orchestrator paths, and runner-local\n`_enrich_driver_runtime_metadata()` plus `_apply_driver_meta_to_dicts()`.\nThe original critic overstated precedence divergence: both paths prefer\nadapter values over database config fallbacks. The real divergence is that\nthe runner also writes driver fields into `result.platform_info` with\n`setdefault()`, while the shared helper only writes result attributes and\n`execution_metadata`.\n\nPost-load validation also conflates not-applicable validation with failed\nvalidation. `_run_postload_validation()` returns `None` for `adapter is None`,\nbut returns a synthetic invalid `ValidationResult` when an adapter lacks\n`create_connection`/`close_connection`. Consumers already treat `None` as\n\"no validation record\" and aggregate `is_valid=False` as a failed stage.", "id": "execution-results-driver-metadata-and-validation-semantics", "preserves": ["Actual post-load validation exceptions still surface as failed validation, not as not-run.", "Adapter-sourced driver metadata remains higher precedence than database_config fallbacks.", "DataFrame post-load validation remains skipped unless a separate TODO changes DataFrame validation support.", "Driver metadata still serializes through schema-v2 platform/execution blocks as before.", "Existing platform_info keys are not overwritten by driver metadata propagation."], "prior_art": [{"concept": "benchbox/core/results/driver_metadata.py: shared metadata helper already used by CLI paths - extend this rather than keeping runner-local duplication.", "decision": "extend", "path": "benchbox/core/results/driver_metadata.py"}, {"concept": "benchbox/core/results/schema.py: driver metadata collection for export - verify exported fields remain stable.", "decision": "reuse", "path": "benchbox/core/results/schema.py"}, {"concept": "benchbox/core/runner/runner.py: current platform_info setdefault behavior - preserve while routing through shared helper.", "decision": "reuse", "path": "benchbox/core/runner/runner.py"}, {"concept": "tests/unit/cli/test_run_metadata_contract.py: existing shared-helper contract tests - expand for platform_info and runner parity.", "decision": "reuse", "path": "tests/unit/cli/test_run_metadata_contract.py"}, {"concept": "tests/unit/core/test_runner_helpers.py: existing post-load validation tests - update to the explicit applicability semantics.", "decision": "reuse", "path": "tests/unit/core/test_runner_helpers.py"}], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner/dataframe_runner.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/driver_metadata.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/runner/runner.py"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "docs/reference/result-formats.md"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_run_metadata_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_runner_helpers.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_runner_lifecycle.py"}], "state": "done", "title": "Execution/results hardening: unify driver metadata propagation and validation applicability semantics", "verifications": [{"command": "uv run -- python -m pytest tests/unit/cli/test_run_metadata_contract.py tests/unit/core/test_runner_helpers.py -q", "description": "Driver metadata contract tests pass.", "expected": "Metadata and validation helper tests pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/test_runner_lifecycle.py tests/unit/core/results/test_schema_timing_contract.py --tb=short", "description": "Focused lifecycle tests pass.", "expected": "Lifecycle result metadata and schema contract tests pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite catches path parity regressions.", "expected": "Fast tests pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-driver-metadata-and-validation-semantics", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": [], "notes": "Re-run source checks before editing and capture a compact summary with\nchecked SHA, command, and counts. Do not commit raw stdout.\n\nRequired checks:\n - `rg -n \"apply_driver_metadata|_enrich_driver_runtime_metadata|_apply_driver_meta_to_dicts|_CONFIG_FIELD_MAP|_EXEC_META_SETDEFAULT\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - `rg -n \"_run_postload_validation|postload_result|validation_records|_finalize_validation_metadata|ValidationResult\\\\(is_valid=False\" benchbox tests --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - Inspect `tests/unit/core/test_runner_helpers.py` for tests pinning current missing-method invalid behavior.\n\nDecision gate: if result-schema policy or public-contract docs currently\npromise a failed stage for validation-not-applicable, preserve that public\nbehavior and document why this TODO should not change it.\n\nCompleted 2026-05-22 at checked SHA\n`f997b0aebc354f69bf32ca6b431dd4bd7e177404`: driver metadata rg returned\n28 lines, post-load validation rg returned 55 lines. Public contracts\nreserve `summary.validation=\"passed\"` for actual validation evidence and\nsay missing validation records finalize as `not_run`; no contract requires\nunsupported connection validation to record a failed stage.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate driver metadata call sites and post-load validation consumers", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w0"], "notes": "Update `benchbox/core/results/driver_metadata.py` so the shared helper\ncan also populate `result.platform_info` using the same `setdefault()`\nbehavior currently present in runner-local `_apply_driver_meta_to_dicts()`.\n\nPreserve these semantics:\n - adapter values win over database config values;\n - config values fill only missing adapter values;\n - `driver_version_resolved` falls back to requested `driver_version`;\n - `driver_package` and `driver_version_requested` in execution metadata\n use `setdefault`;\n - `driver_auto_install_used` remains recorded even when false;\n - platform_info population never overwrites existing platform metadata.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend the shared driver metadata helper to preserve runner platform_info behavior", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w1"], "notes": "Replace the runner-local duplication in\n`_enrich_driver_runtime_metadata()` with calls to\n`apply_driver_metadata(...)`. Delete or shrink `_DRIVER_STR_FIELDS`,\n`_CONFIG_FIELD_MAP`, `_EXEC_META_SETDEFAULT`, and\n`_apply_driver_meta_to_dicts()` if they become unused.\n\nKeep `_enrich_driver_runtime_metadata()` as a small compatibility wrapper\nif tests or monitoring import gates patch it directly. The wrapper should\nbe boring: call the shared helper and return the result.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Route runner lifecycle metadata enrichment through the shared helper", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w0"], "notes": "Decide and implement one explicit semantic rule:\n - not applicable: return `None`, do not add a validation stage;\n - attempted and failed: return `ValidationResult(is_valid=False)`;\n - attempted and passed/warned: return the engine result.\n\nApply that rule to `_run_postload_validation()`. The likely change is\nreturning `None` when the adapter lacks `create_connection` or\n`close_connection`, while preserving synthetic failure for exceptions\nraised after validation is actually attempted.\n\nUpdate tests currently expecting invalid for missing adapter methods to\nassert `None` if the public-contract check in w0 allows the change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define validation applicability semantics and update post-load validation behavior", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-driver-metadata-and-validation-semantics", "needs": ["w2", "w3"], "notes": "Cover:\n - CLI/orchestrator path and runner lifecycle path produce the same\n top-level driver fields and `execution_metadata` fields;\n - runner path still backfills `platform_info` without overwriting\n existing platform_info values;\n - missing connection validation hooks produce no validation stage after\n the semantics change;\n - actual post-load validation exceptions still produce a failed stage.\n\nPrefer focused unit tests around `apply_driver_metadata()`,\n`_enrich_driver_runtime_metadata()`, `_run_postload_validation()`, and\n`_finalize_validation_metadata()` before broad lifecycle tests.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add metadata and validation regression tests for lifecycle and CLI parity", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add a catch-all dict that bypasses schema-v2 validation policy.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "delete `_benchmark_id_override` until result filename, loader, and builder behavior have an explicit replacement.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "move `native_comparison` or migration fields into `platform_extensions` in one step without loader, explorer, MCP, and public-submission tests.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "treat the dataclass field count as the user-facing problem", "instead": "(not specified)", "why": "exported result compatibility is the primary risk."}], "approach": "Treat `BenchmarkResults` cleanup as a compatibility design problem, not a\nmechanical field move. First document the extension policy, then inventory\ncurrent fields and consumers, then add contract tests, and only then make a\nminimal internal-only or backward-compatible change.\n", "blocked_reason": null, "category": "Results", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-result-model-extension-contract", "id": 399, "reason": "This is the risky change the TODO is designed to evaluate; do it only after policy, inventory, and consumer tests prove compatibility.", "resolution": "open", "resolved_item": null, "resolved_reason": null, "summary": "Move all existing platform-specific result fields into `platform_extensions`"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-result-model-extension-contract", "id": 400, "reason": "Result schema policy currently supports schema-v2 consumers; a major break requires a separate migration plan.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: A major result-schema version bump needs a separate migration plan that doesn't exist; current schema-v2 policy still holds.", "summary": "Bump result schema major version for a broad model cleanup"}], "deps": ["architecture-contract-map-and-support-taxonomy", "result-schema-policy-and-explorer-input-gate"], "description": "Independent verification partially confirmed that `BenchmarkResults` is doing\ntoo much, but also found that the original critic overstated the facts. The\ndataclass currently has 78 fields, not roughly 205. It has several platform\nmetadata dictionaries, `_benchmark_id_override`, and a platform-specific\n`native_comparison` field. The claimed `migration` top-level model field is\nnot a `BenchmarkResults` field; schema-v2 exports migration as a phase when\n`ExecutionPhases.migration` is present, while a separate pg_mooncake script\nemits a top-level `migration` block.\n\nThe suggested fix, moving platform-specific results into\n`platform_extensions: dict[str, Any]`, is therefore directionally useful but\nrisky. Result JSON is a product boundary consumed by runtime loaders, CLI\ncompare/export/show, MCP tools, public submissions, hosted result bundles,\nand the explorer pipeline. This TODO creates the migration contract and a\nnarrow extension policy before any field move.", "id": "execution-results-result-model-extension-contract", "preserves": ["Do not conflate internal dataclass cleanup with exported JSON schema cleanup.", "Existing schema-v2 result bundles remain loadable by `load_result_file()`.", "Explorer pipeline input gate behavior from result-schema-policy work is not weakened.", "No existing top-level exported key is removed without same-PR compatibility and docs updates.", "Public submission validation remains compatible with accepted schema-v2 bundles."], "prior_art": [{"concept": "_project/scripts/explorer_pipeline/transformer.py: hosted/explorer consumer - include in downstream compatibility checks.", "decision": "reuse", "path": "_project/scripts/explorer_pipeline/transformer.py"}, {"concept": "benchbox/core/results/loader.py: reconstruction contract for historical result files - test before moving fields.", "decision": "reuse", "path": "benchbox/core/results/loader.py"}, {"concept": "benchbox/core/results/schema.py: current schema-v2 payload builder - exported keys are the compatibility boundary.", "decision": "reuse", "path": "benchbox/core/results/schema.py"}, {"concept": "benchbox/core/results/schema_policy.py: consumer-specific schema acceptance - preserve these risk postures.", "decision": "reuse", "path": "benchbox/core/results/schema_policy.py"}, {"concept": "docs/reference/public-contracts.md: result JSON bundle contract tier - extend before moving exported fields.", "decision": "extend", "path": "docs/reference/public-contracts.md"}, {"concept": "scripts/benchmark_mooncake_migration.py: standalone top-level migration payload - decide whether to align or leave separate.", "decision": "reuse", "path": "scripts/benchmark_mooncake_migration.py"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/benchmark.py"}, {"kind": "only_modify", "path_glob": "benchbox/mcp/tools/results.py"}, {"kind": "only_modify", "path_glob": "docs/reference/public-contracts.md"}, {"kind": "only_modify", "path_glob": "docs/reference/result-formats.md"}, {"kind": "only_modify", "path_glob": "scripts/benchmark_mooncake_migration.py"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/commands/test_submit.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/"}], "state": "done", "title": "Execution/results hardening: define result-model extension contract before platform-specific field migration", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/results -q", "description": "Focused result schema/model tests pass.", "expected": "Result model, loader, schema, and normalizer tests pass.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline tests/unit/cli/commands/test_submit.py --tb=short", "description": "Explorer and submission consumers remain compatible.", "expected": "Explorer pipeline and submission command tests pass.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python scripts/validate_submission.py results-data/bundles/", "description": "Published bundle validator still accepts existing corpus.", "expected": "Existing public result bundles validate.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-result-model-extension-contract", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": [], "notes": "Re-run the current-tree evidence before proposing schema or model changes.\nCapture checked SHA, commands, and compact results in the implementation\nPR body or TODO notes; do not commit raw stdout.\n\nRequired checks:\n - `uv run -- python - <<'PY'` snippet using `dataclasses.fields(BenchmarkResults)` to print field count and platform/native/migration-related fields.\n - `rg -n \"native_comparison|\\\\[\\\\\\\"migration\\\\\\\"\\\\]|platform_info|platform_metadata|platform_raw|BenchmarkResults\\\\(|build_result_payload|reconstruct_benchmark_results|load_result_file|ResultExporter|PUBLIC_SUBMISSION_SCHEMA_POLICY\" benchbox _project scripts tests docs/reference --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'`.\n - Inspect `benchbox/core/results/schema.py`, `benchbox/core/results/loader.py`, `benchbox/core/results/exporter.py`, `benchbox/mcp/tools/benchmark.py`, `benchbox/mcp/tools/results.py`, and `_project/scripts/explorer_pipeline/`.\n\nDecision gate: do not move or remove any serialized result key in the\nimplementation PR unless the migration policy explicitly classifies the\nchange as backward-compatible or includes a same-PR migration/loader gate.\n\nCompleted 2026-05-22 at checked SHA\n`185006fde1a04b460e4c30d52d4035afeb1861eb`: dataclass field count is 78;\nplatform/native/migration-related fields are `platform`,\n`platform_deployment`, `platform_cloud`, `platform_compute`,\n`platform_storage`, `platform_raw_config`, `platform_raw_metadata`,\n`platform_info`, `platform_metadata`, and `native_comparison`;\n`BenchmarkResults` has no `migration` field. Consumer rg returned 2504\nbroad hits. Inspection confirmed `comparisons.native_duckdb`,\n`phases.migration`, platform metadata blocks, loader reconstruction, MCP\nresult loading, and explorer projection are the relevant compatibility\nboundaries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate BenchmarkResults field count, platform-specific fields, and schema consumers", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w0"], "notes": "Add a policy section to docs/reference/public-contracts.md or a focused\nresult-schema doc that answers:\n - where platform-specific structured data should live in the internal\n model;\n - where it should serialize in schema-v2 payloads;\n - when top-level schema keys are allowed;\n - when a new `platform_extensions` block is acceptable;\n - how loader, normalizer, public submission, and explorer consumers\n must handle unknown extension keys;\n - what evidence is required before moving an existing field.\n\nThe policy must distinguish internal dataclass fields from exported JSON\nkeys. Do not let a model cleanup imply a schema break.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write a result-model extension policy covering platform-specific data and schema-v2 compatibility", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w1"], "notes": "Build a concise inventory covering at least:\n - `native_comparison` and its `comparisons.native_duckdb` export;\n - `ExecutionPhases.migration` and `phases.migration` export;\n - the standalone pg_mooncake migration script's top-level `migration`\n payload;\n - `platform_info`, `platform_metadata`, `platform_raw_config`,\n `platform_raw_metadata`;\n - `platform_deployment`, `platform_cloud`, `platform_compute`,\n `platform_storage`;\n - `execution_context` and `_benchmark_id_override`.\n\nFor each row record current owner, exported key, loader reconstruction\nbehavior, downstream consumers, and recommended action:\n - keep;\n - alias;\n - move internally only;\n - migrate exported schema with compatibility;\n - reject as not worth it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Create an extension inventory and migration matrix for current platform-specific fields", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w2"], "notes": "Before changing model shape, add focused tests or fixtures that prove\ncurrent behavior for `comparisons.native_duckdb`, `phases.migration`, and\nplatform metadata blocks through:\n - `build_result_payload()`;\n - `reconstruct_benchmark_results()`;\n - public submission validation where applicable;\n - explorer pipeline transformation where applicable.\n\nIf the matrix chooses no field migration yet, this work still protects\nfuture refactors by making the current contract explicit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add consumer tests or fixtures that lock current extension behavior before migration", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-result-model-extension-contract", "needs": ["w3"], "notes": "Choose one narrow change from the w2 matrix that does not break exported\nresult JSON. Examples:\n - add an internal `platform_extensions` field while continuing to\n serialize existing keys unchanged;\n - route one new platform-specific result through an extension block\n without moving existing keys;\n - add loader passthrough for unknown extension blocks while preserving\n known fields.\n\nIf every field move is risky, record that as the outcome and stop after\npolicy, inventory, and tests. A \"no migration yet\" result is acceptable\nif it is evidence-backed.\n\nOutcome: no existing field was moved. The only code cleanup is accepting\nthe existing producer-emitted top-level `comparisons` key in schema-v2\nvalidation. Current extension locations are locked by tests and policy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement only the lowest-risk internal extension cleanup approved by the matrix", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change exported result keys such as `summary.tpc_metrics` or query result `test_type` while cleaning up internals.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "delete `benchbox/core/tpc_metrics.py` solely because static import grep is clean", "instead": "(not specified)", "why": "first audit whether its validation/reporting behavior is needed elsewhere."}, {"dont": "introduce a new generic TPC execution framework in this TODO", "instead": "(not specified)", "why": "the confirmed duplication is result conversion, not the entire driver lifecycle."}, {"dont": "replace explicit failure sentinels with empty query lists", "instead": "(not specified)", "why": "that can make failed power-test setup look like a successful run with zero queries."}], "approach": "Treat the legacy TPC metrics module as a retirement candidate, not as already\nsafe to delete. First prove the import graph and unique behavior. Then either\ndelete it or fold the useful surface into the canonical results helpers. In\nparallel, extract the small repeated power-result dict construction logic in\nthe base execution adapter without changing the surrounding TPC driver flow.\n", "blocked_reason": null, "category": "Results", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-21T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-tpc-metrics-and-power-result-normalization", "id": 401, "reason": "This TODO removes confirmed duplication in the power conversion path only; a broader driver rewrite needs separate perf and compliance risk review.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: The confirmed power-conversion duplication was already removed by this item. A full rewrite of ALL TPC-H/DS power+throughput drivers around a shared adapter is speculative on a compliance-critical path (the item flags it needs separate perf/compliance risk review); no remaining defect demonstrated.", "summary": "Rewrite all TPC-H/TPC-DS power and throughput execution drivers around one shared result adapter"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "execution-results-tpc-metrics-and-power-result-normalization", "id": 402, "reason": "Only do this if w1 proves the legacy reporter has product value; otherwise it is dead internal surface.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Conditional on w1 proving legacy reporter has product value; otherwise explicitly 'dead internal surface' \u2014 no evidence condition met.", "summary": "Promote rich metrics reporting to a user-facing CLI command"}], "deps": ["architecture-contract-map-and-support-taxonomy"], "description": "Independent verification of the execution/results architecture review found\ntwo adjacent duplication risks in the TPC metrics and power-test result path.\n\nFirst, `benchbox/core/tpc_metrics.py` is an 877-line legacy module with no\nproduction importers in the current tree. Production code imports\n`benchbox.core.results.metrics.TPCMetricsCalculator`, which is re-exported\nfrom `benchbox.core.results`. Only `tests/unit/core/test_tpc_metrics.py`\nimports the legacy module. The module is therefore a deletion candidate, but\nit contains richer validation/reporting classes that should be audited before\nremoval so useful behavior is not silently lost.\n\nSecond, `benchbox/platforms/base/execution.py` repeats TPC-H and TPC-DS\npower-test result-dict conversion in four blocks, plus four\n`power_test_error` sentinels. The duplication is smaller than the original\ncritic claimed, but it is still enough to make failure/result schema drift\nlikely when one TPC driver changes.", "id": "execution-results-tpc-metrics-and-power-result-normalization", "preserves": ["Do not weaken TPC compliance or row-count validation behavior as part of cleanup.", "No result JSON schema change from metrics-module retirement or conversion-helper extraction.", "Power-test failure sentinels still make all-query factory failures visible as FAILED runs.", "TPC-H and TPC-DS Power@Size calculations continue using the canonical `results.metrics.TPCMetricsCalculator` formulas."], "prior_art": [{"concept": "benchbox/core/results/builder.py: result-builder metric population - preserve as the production metric source.", "decision": "reuse", "path": "benchbox/core/results/builder.py"}, {"concept": "benchbox/core/results/metrics.py: canonical lightweight TPC metric formulas - extend this rather than keeping a second calculator.", "decision": "extend", "path": "benchbox/core/results/metrics.py"}, {"concept": "benchbox/platforms/base/execution.py: current TPC-H/TPC-DS power conversion blocks - consolidate locally first.", "decision": "reuse", "path": "benchbox/platforms/base/execution.py"}, {"concept": "docs/reference/public-contracts.md: contract tier map - consult before deleting any import path that may be public.", "decision": "reuse", "path": "docs/reference/public-contracts.md"}], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/runner/"}, {"kind": "do_not_modify", "path_glob": "benchbox/platforms/dataframe/"}, {"kind": "do_not_modify", "path_glob": "results-data/"}, {"kind": "only_modify", "path_glob": "_project/DONE/"}, {"kind": "only_modify", "path_glob": "_project/TODO/"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/builder.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/results/metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/tpc_metrics.py"}, {"kind": "only_modify", "path_glob": "benchbox/platforms/base/execution.py"}, {"kind": "only_modify", "path_glob": "docs/reference/backward-compatibility.md"}, {"kind": "only_modify", "path_glob": "tests/unit/core/results/test_metrics.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/test_tpc_metrics.py"}, {"kind": "only_modify", "path_glob": "tests/unit/platforms/test_base_adapter.py"}], "state": "done", "title": "Execution/results hardening: retire legacy TPC metrics and normalize power-result conversion", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/results/test_metrics.py tests/unit/platforms/test_base_adapter.py -q", "description": "Focused TPC metrics and base adapter tests pass.", "expected": "All focused metrics and base-adapter tests pass.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 1}, {"command": "rg -n \"^from benchbox\\.core\\.tpc_metrics import|^import benchbox\\.core\\.tpc_metrics|from benchbox\\.core import tpc_metrics|benchbox\\.core\\.tpc_metrics\" benchbox tests pyproject.toml --glob '!build/**' --glob '!docs/_build/**' --glob '!.venv/**'", "description": "Legacy TPC metrics import graph has an explicit terminal state.", "expected": "No matches if deletion path is chosen; otherwise only documented compatibility imports remain and the PR explains why deletion was rejected.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite catches result-shape regressions.", "expected": "Fast tests pass.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run --project _project/scripts -- python _project/scripts/todo_cli.py validate && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO graph remains valid after implementation.", "expected": "TODO validation and graph checks pass.", "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": [], "notes": "Revalidated against checked SHA\n`7a4262f8f6457e97b334e6132c9e58b2640c785f`. `wc -l` reported\n`benchbox/core/tpc_metrics.py` at 877 lines and\n`benchbox/core/results/metrics.py` at 211 lines. The legacy import grep\nfound only `tests/unit/core/test_tpc_metrics.py`; no production importer\nexists in the current tree. The canonical results metrics grep found\nproduction and test use of `benchbox.core.results.metrics` /\n`TPCMetricsCalculator`. The power conversion grep found the duplicated\nTPC-H/TPC-DS normal result blocks and `power_test_error` sentinels in\n`benchbox/platforms/base/execution.py`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Revalidate TPC metrics import graph, unique legacy surface, and power-result conversion count", "wid": "w0"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w0"], "notes": "Audit decision table:\n - Power@Size, Throughput@Size, and composite Qph formulas are covered\n by `benchbox/core/results/metrics.py`; focused tests now pin the\n legacy formula expectations on the canonical calculator.\n - `PowerMetrics`, `ThroughputMetrics`, `CompositeMetrics`,\n `MetricsValidator`, and `calculate_full_metrics()` are unimported\n legacy object/reporting surfaces, not part of current schema-v2\n result construction.\n - `StatisticalAnalyzer` and `MetricsReporter` may be useful product\n ideas, but they are uncalled internal reporting helpers and should be\n promoted only through a separate CLI/reporting TODO.\nNo unique production-useful behavior required migration before deleting\nthe legacy module.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit legacy tpc_metrics.py for behavior not present in results.metrics", "wid": "w1"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w1"], "notes": "Deleted `benchbox/core/tpc_metrics.py` and\n`tests/unit/core/test_tpc_metrics.py`. No public/backward-compat docs\nwere changed because the revalidated import graph did not show an\nintentionally supported import path. Removed the stale mutation-tool\nexclusion for the deleted module from `pyproject.toml` as a narrow scope\nexception found during self-review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Retire or fold the legacy TPC metrics module through an explicit compatibility decision", "wid": "w2"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w0"], "notes": "Added `_power_query_result()` and `_power_test_error_result()` in\n`benchbox/platforms/base/execution.py` and replaced the duplicated\nTPC-H/TPC-DS power conversion blocks and sentinels. Throughput,\nmaintenance, and generic power conversion paths were left unchanged.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract power-test result conversion and sentinel construction helpers", "wid": "w3"}, {"evidence": null, "item_id": "execution-results-tpc-metrics-and-power-result-normalization", "needs": ["w2", "w3"], "notes": "Added canonical formula coverage in\n`tests/unit/core/results/test_metrics.py` and conversion-helper coverage\nin `tests/unit/platforms/test_base_adapter.py`. The final import grep is\nexpected to terminate with no matches for `benchbox.core.tpc_metrics`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add focused regression tests for canonical TPC metrics and power-result conversion", "wid": "w4"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-01-19T00:00:00Z", "deferrals": [], "deps": [], "description": "The MCP server currently exposes only 11 tools while the CLI has ~34 commands.\nThis creates a significant gap in what AI agents can accomplish via MCP vs CLI.\n\nCurrent MCP coverage:\n- Discovery: 4 tools (list_platforms, list_benchmarks, get_benchmark_info, system_profile)\n- Execution: 4 tools (run_benchmark, dry_run, validate_config, get_query_details)\n- Results: 3 tools (list_recent_runs, get_results, compare_results)\n\nMajor gaps identified:\n- Data preparation (datagen, convert)\n- Query plan analysis (show-plan, plan-history)\n- Dependency checking (check-deps)\n- Performance analytics (report trends, regressions, rankings)\n- Result aggregation and export\n\nThis TODO adds 7+ new MCP tools and 2 new MCP prompts to close the most important gaps.", "id": "expand-mcp-tool-surface", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Expand MCP tool surface area to match CLI capabilities", "verifications": [], "work": [], "worktree": "mcp-integration"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2025-11-26T00:00:00Z", "deferrals": [], "deps": [], "description": "Fingerprint fast-paths currently ignore core logical structure (join conditions,\ngrouping, projections, limits/offsets), so materially different plans hash the\nsame and are marked identical. Stored fingerprints are also trusted on ingest,\nallowing stale/tampered hashes to bypass comparison. Tighten the structural\nsignature, recompute/verify fingerprints on load and before comparison, and\nupdate regression/summary flows to refuse hashes that don\u2019t match the current\ntree. Ensure plan history and CLI outputs surface verification failures cleanly.", "id": "expand-plan-fingerprint-verification", "preserves": [], "prior_art": [], "priority": "high", "scope": [], "state": "done", "title": "Expand query plan fingerprint coverage and verification", "verifications": [], "work": [], "worktree": "query-plan-remediation"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": null, "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-19T12:34:13Z", "deferrals": [], "deps": [], "description": "Promoted from deferral #154 of ensure-read-primitives-variant-comparability.\nDeferred: Expand runtime parity coverage for retained read_primitives variants not covered by the first local parity suite\nReason deferred: The first pass targets the highest-risk families. Follow-up coverage should include array_unnest (clickhouse, bigquery), array_agg_simple (clickhouse), struct_construction (clickhouse), list_transform (clickhouse), and the duckdb/clickhouse paths of map_construction so retained variants do not drift silently.", "id": "expand-runtime-parity-coverage-for-retained-read-primitives", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Expand runtime parity coverage for retained read_primitives variants not covered by the first local parity suite", "verifications": [], "work": [], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "bypass `benchbox submit` and copy bundles directly into `results-data/bundles/`", "why": "the production flow (submit + manifest + CI) needs to be exercised on every cohort, not just community ones."}, {"INSTEAD": "(not specified)", "dont": "hand-edit `corpus-inventory.json`", "why": "the generator from `automate-corpus-inventory-generation` is now authoritative."}, {"INSTEAD": "(not specified)", "dont": "skip platforms where a few queries fail and ship partial-coverage bundles", "why": "the cohort's value is in apples-to-apples comparison; partial coverage misleads."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["automate-corpus-inventory-generation", "verify-ci-validation-workflow-end-to-end"], "description": "The Phase 1 launch corpus is 6 bundles, all at scale factor 0.01. This is\nenough to prove the explorer renders, but too small to look credible to a\nvisitor evaluating real-world performance. The strategy doc commits to\n\"depth per benchmark\" - \u22653 comparable platforms per cohort - and SF 0.1\nis the next rung that exercises the platforms beyond toy timings while\nstaying within reasonable runtime + storage budgets for CI / local runs.\n\nThis item generates SF 0.1 cohorts for TPC-H and SSB on the same three\nplatforms each cohort already covers at SF 0.01 (TPC-H: DuckDB, DataFusion,\nPolars; SSB: DuckDB, DataFusion, SQLite), uses the production submit\u2192PR\nflow to land them on `published-results`, and confirms the explorer\nrenders the new cohort selectors.", "id": "expand-seed-corpus-to-sf-0-1", "preserves": ["Cohort isolation - SF 0.01 and SF 0.1 must remain separate cohorts in the explorer.", "Existing 6 SF 0.01 bundles untouched.", "Maintainer-run trust label on all generated bundles (no fake community labels)."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/notes/"}, {"kind": "only_modify", "path_glob": "results-data/bundles/"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json"}], "state": "done", "title": "Expand seed corpus to SF 0.1 cohorts for TPC-H and SSB", "verifications": [{"command": "uv run -- python -c \"import json; d=json.load(open('results-data/corpus-inventory.json')); print(len(d['bundles']))\"", "description": "Inventory contains 12 bundles total (6 SF 0.01 + 6 SF 0.1)", "expected": "12", "item_id": "expand-seed-corpus-to-sf-0-1", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python results-data/validate_corpus.py", "description": "Each new bundle validates against schema-v2", "expected": "all bundles valid", "item_id": "expand-seed-corpus-to-sf-0-1", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": [], "notes": "For each platform run a clean tuned run, using full TPC-H query set:\n benchbox run --platform duckdb --benchmark tpch --scale 0.1 --tuning tuned\n benchbox run --platform datafusion --benchmark tpch --scale 0.1 --tuning tuned\n benchbox run --platform polars-df --benchmark tpch --scale 0.1 --tuning tuned\nCapture environment metadata (uname, CPU, RAM) in the bundle. If a\nquery fails on a given platform, document the failure in\n`_project/notes/` and leave the bundle out (don't ship a partial\ncohort that misleads the compare view).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generate TPC-H SF 0.1 results on DuckDB, DataFusion, Polars", "wid": "w1"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": [], "notes": "Same pattern as w1. SQLite at SF 0.1 may be slow; allow extended\ntimeouts but do not silently shorten the query set:\n benchbox run --platform duckdb --benchmark ssb --scale 0.1 --tuning tuned\n benchbox run --platform datafusion --benchmark ssb --scale 0.1 --tuning tuned\n benchbox run --platform sqlite --benchmark ssb --scale 0.1 --tuning tuned\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generate SSB SF 0.1 results on DuckDB, DataFusion, SQLite", "wid": "w2"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": ["w1", "w2"], "notes": "For each bundle:\n benchbox submit --last --output ./submission---sf0.1/\nVerify the manifest hash and the labelled trust label is `maintainer-run`\n(these are not community submissions).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Package each bundle with `benchbox submit` and stage the PR", "wid": "w3"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": ["w3"], "notes": "Single PR is preferable to 6 PRs because:\n - the cohort is meaningful only when complete,\n - the explorer rebuild only fires once per merge,\n - reviewers see all SF 0.1 results together.\nConfirm validate-submission CI passes; address any failures.\nConfirm corpus-inventory.json regenerates cleanly (depends on\nautomate-corpus-inventory-generation). Merge after review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Open a single submission PR against published-results with all 6 bundles", "wid": "w4"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": ["w4"], "notes": "After explorer redeploy:\n - TPC-H scale-factor selector lists 0.01 and 0.1\n - SSB scale-factor selector lists 0.01 and 0.1\n - Compare view at /results/compare?ids=<3 SF 0.1 ids> renders\n - Cohort validation does NOT block within-SF comparisons\n - Cohort validation DOES block mixed-SF comparisons\nCapture one screenshot per benchmark of the SF 0.1 compare view.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify the explorer surfaces the new SF 0.1 cohort", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "bypass `benchbox submit` and copy bundles directly into `results-data/bundles/`", "instead": "(not specified)", "why": "the production flow (submit + manifest + CI) needs to be exercised on every cohort, not just community ones."}, {"dont": "hand-edit `corpus-inventory.json`", "instead": "(not specified)", "why": "the generator from `automate-corpus-inventory-generation` is now authoritative."}, {"dont": "skip platforms where a few queries fail and ship partial-coverage bundles", "instead": "(not specified)", "why": "the cohort's value is in apples-to-apples comparison; partial coverage misleads."}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-12T00:00:00Z", "completed_pr": null, "created_at": "2026-04-12T00:00:00Z", "deferrals": [], "deps": ["automate-corpus-inventory-generation", "verify-ci-validation-workflow-end-to-end"], "description": "The Phase 1 launch corpus is 6 bundles, all at scale factor 0.01. This is\nenough to prove the explorer renders, but too small to look credible to a\nvisitor evaluating real-world performance. The strategy doc commits to\n\"depth per benchmark\" - \u22653 comparable platforms per cohort - and SF 0.1\nis the next rung that exercises the platforms beyond toy timings while\nstaying within reasonable runtime + storage budgets for CI / local runs.\n\nThis item generates SF 0.1 cohorts for TPC-H and SSB on the same three\nplatforms each cohort already covers at SF 0.01 (TPC-H: DuckDB, DataFusion,\nPolars; SSB: DuckDB, DataFusion, SQLite), uses the production submit\u2192PR\nflow to land them on `published-results`, and confirms the explorer\nrenders the new cohort selectors.", "id": "expand-seed-corpus-to-sf-0-1", "preserves": ["Cohort isolation - SF 0.01 and SF 0.1 must remain separate cohorts in the explorer.", "Existing 6 SF 0.01 bundles untouched.", "Maintainer-run trust label on all generated bundles (no fake community labels)."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "_project/notes/"}, {"kind": "only_modify", "path_glob": "results-data/bundles/"}, {"kind": "only_modify", "path_glob": "results-data/corpus-inventory.json"}], "state": "done", "title": "Expand seed corpus to SF 0.1 cohorts for TPC-H and SSB", "verifications": [{"command": "uv run -- python -c \"import json; d=json.load(open('results-data/corpus-inventory.json')); print(len(d['bundles']))\"", "description": "Inventory contains 12 bundles total (6 SF 0.01 + 6 SF 0.1)", "expected": "12", "item_id": "expand-seed-corpus-to-sf-0-1", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python results-data/validate_corpus.py", "description": "Each new bundle validates against schema-v2", "expected": "all bundles valid", "item_id": "expand-seed-corpus-to-sf-0-1", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": [], "notes": "For each platform run a clean tuned run, using full TPC-H query set:\n benchbox run --platform duckdb --benchmark tpch --scale 0.1 --tuning tuned\n benchbox run --platform datafusion --benchmark tpch --scale 0.1 --tuning tuned\n benchbox run --platform polars-df --benchmark tpch --scale 0.1 --tuning tuned\nCapture environment metadata (uname, CPU, RAM) in the bundle. If a\nquery fails on a given platform, document the failure in\n`_project/notes/` and leave the bundle out (don't ship a partial\ncohort that misleads the compare view).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generate TPC-H SF 0.1 results on DuckDB, DataFusion, Polars", "wid": "w1"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": [], "notes": "Same pattern as w1. SQLite at SF 0.1 may be slow; allow extended\ntimeouts but do not silently shorten the query set:\n benchbox run --platform duckdb --benchmark ssb --scale 0.1 --tuning tuned\n benchbox run --platform datafusion --benchmark ssb --scale 0.1 --tuning tuned\n benchbox run --platform sqlite --benchmark ssb --scale 0.1 --tuning tuned\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Generate SSB SF 0.1 results on DuckDB, DataFusion, SQLite", "wid": "w2"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": ["w1", "w2"], "notes": "For each bundle:\n benchbox submit --last --output ./submission---sf0.1/\nVerify the manifest hash and the labelled trust label is `maintainer-run`\n(these are not community submissions).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Package each bundle with `benchbox submit` and stage the PR", "wid": "w3"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": ["w3"], "notes": "Single PR is preferable to 6 PRs because:\n - the cohort is meaningful only when complete,\n - the explorer rebuild only fires once per merge,\n - reviewers see all SF 0.1 results together.\nConfirm validate-submission CI passes; address any failures.\nConfirm corpus-inventory.json regenerates cleanly (depends on\nautomate-corpus-inventory-generation). Merge after review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Open a single submission PR against published-results with all 6 bundles", "wid": "w4"}, {"evidence": null, "item_id": "expand-seed-corpus-to-sf-0-1", "needs": ["w4"], "notes": "After explorer redeploy:\n - TPC-H scale-factor selector lists 0.01 and 0.1\n - SSB scale-factor selector lists 0.01 and 0.1\n - Compare view at /results/compare?ids=<3 SF 0.1 ids> renders\n - Cohort validation does NOT block within-SF comparisons\n - Cohort validation DOES block mixed-SF comparisons\nCapture one screenshot per benchmark of the SF 0.1 compare view.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Verify the explorer surfaces the new SF 0.1 cohort", "wid": "w5"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-01-16T00:00:00Z", "completed_pr": null, "created_at": "2026-01-16T00:00:00Z", "deferrals": [], "deps": [], "description": "The current CLI handling of `--tuning` argument uses implicit logic that can confuse users.\nWhen users specify `--tuning tuned`, the system silently attempts to load a template from\n`examples/tunings/{platform}/{benchmark}_tuned.yaml`. If not found, it falls back to basic\nconstraints without warning. Users have no visibility into which tuning file was loaded\nor what settings are being applied.\n\nKey issues identified:\n1. Silent template loading - users don't know what template is used\n2. Implicit fallback chain - silent fallback to basic config if template missing\n3. No discovery mechanism - users must manually find tuning files\n4. Path resolution ambiguity - custom paths resolved from cwd only\n5. Error messaging - invalid path treated same as invalid keyword\n6. \"auto\" mode unclear - documentation doesn't explain how it differs from \"tuned\"\n\nThis TODO addresses making tuning file resolution explicit and transparent so users\nalways know exactly what tuning configuration is being applied and where it came from.", "id": "explicit-tuning-file-resolution", "preserves": [], "prior_art": [], "priority": "medium-high", "scope": [], "state": "done", "title": "Make tuning file resolution explicit and transparent in CLI", "verifications": [], "work": [], "worktree": "ux-improvements"} -{"anti_patterns": [{"INSTEAD": "default to fastest", "dont": "default to the slowest result as baseline", "why": "speedup is conventionally measured relative to the alternative, not the worst option"}, {"INSTEAD": "inline SVG is sufficient", "dont": "pull in a charting library (D3, Chart.js, Recharts)", "why": "bundle size and dependency overhead aren't justified for 2 chart types"}, {"INSTEAD": "log scale is required for legibility", "dont": "use linear scale for the speedup chart", "why": "OLAP deltas span orders of magnitude"}], "approach": "Both charts are inline SVG components (consistent with existing\nQueryTimingChart pattern). No external charting library needed for\nthese two chart types - the math is simple and SVG gives full control\nover log scale rendering.\n\nLog scale: manually compute SVG x-positions from log2(speedup).\nRender grid lines at 0.25x, 0.5x, 1x, 2x, 4x. Label them.\n\nDiverging bar: sort queries by abs(delta_pct) descending so the most\nimpactful queries appear at top. Include a summary line: \"N queries\nfaster, M queries slower, median delta: \u00b1X%\".\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-align-duration-metric", "explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found that the explorer compare view\nships only 2 inline SVG components (grouped bar chart + simple ratio\ntable) while the CLI offers 15 chart types. The most strategically\nimportant gap for the \"dynamic comparison tool\" claim is the absence\nof charts that make per-query performance deltas legible at a glance:\n\n 1. Normalized speedup chart (log scale, baseline-relative) - answers\n \"how much faster/slower is platform B vs platform A per query?\"\n The current grouped bar chart doesn't normalize, so a 1ms vs 2ms\n delta looks the same as a 100ms vs 200ms delta.\n\n 2. Diverging bar chart (per-query improvement/regression) - answers\n \"which queries got faster and which got slower?\" A positive/negative\n bar per query makes regressions immediately obvious. The CLI's\n `diverging_bar` chart type is the primary tool for this.\n\nThese two charts close the most visible CLI parity gap and directly\nsupport the compare view's core UX claim.\n\nA phase breakdown (stacked load vs query time) and percentile ladder\nare valuable but deferred - they require phase timing fields that are\nnot yet in the manifest.", "id": "explorer-add-comparison-charts", "preserves": ["Compare view URL format - chart selection is UI state, not URL state (no new query params needed)", "Existing grouped bar chart - it is the default for 3+ result comparisons and remains accessible via the chart selector", "Null-safety: if geomean_ms is missing, disable normalized speedup with a clear message"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (no pipeline changes needed)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/DivergingBarChart.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/NormalizedSpeedupChart.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}], "state": "done", "title": "Add normalized speedup and diverging bar charts to explorer compare view", "verifications": [{"command": "ls results-explorer/src/components/NormalizedSpeedupChart.tsx results-explorer/src/components/DivergingBarChart.tsx", "description": "Both new chart components exist", "expected": "both files present", "item_id": "explorer-add-comparison-charts", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'NormalizedSpeedupChart\\|DivergingBarChart\\|chart.*selector\\|chartType' results-explorer/src/pages/Compare.tsx", "description": "Chart selector renders in Compare", "expected": "both chart components referenced", "item_id": "explorer-add-comparison-charts", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-add-comparison-charts", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": [], "notes": "Log scale is important because OLAP query deltas span orders of\nmagnitude. A linear scale makes small queries illegible when one\nlarge query dominates.\n\nBar color: green (speedup > 1), red (speedup < 1), neutral (= 1).\nLog scale axis labels: 0.1x, 0.25x, 0.5x, 1x, 2x, 4x, 10x.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement NormalizedSpeedupChart: speedup = baseline_ms / this_ms per query, log-scale horizontal bars, green/red for faster/slower. Baseline = first selected result.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": [], "notes": "Convention: red bars = regression (slower than baseline), green bars =\nimprovement (faster). Magnitude-sorted by abs(delta) so the biggest\nchanges are at the top. Match CLI diverging_bar semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DivergingBarChart: delta_ms = this_ms - baseline_ms per query, horizontal diverging bars centered at 0, % change labels, sorted by abs(delta).", "wid": "w2"}, {"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add chart selector to Compare.tsx: toggle between Grouped Bar, Normalized Speedup, Diverging Bar. Default Normalized Speedup for 2-result; Grouped Bar for 3+.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add baseline selector for normalized speedup chart: when comparing 3+ results, a dropdown lets the user pick which result is the baseline. Default to the fastest (lowest geomean_ms) result.", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "action controls belong elsewhere", "dont": "add interactive filtering or controls to the disclosure panel", "why": "this is read-only provenance"}, {"INSTEAD": "disclosure is for the methodologically curious", "dont": "expand the panel by default", "why": "most visitors just want the numbers"}, {"INSTEAD": "always include the definition", "dont": "omit the metric definition", "why": "'Geomean query time' is not obvious to a first-time visitor"}], "approach": "Static disclosure - no new data fetching required. All the displayed\nfields come from the detail result object already loaded by ResultDetail.\nThe panel is a simple definition list rendered from props.\n\nCollapsed by default with a clear \"How this was measured \u2193\" label.\nThe label should be visible on first load so the panel is discoverable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found that a visitor to a result detail\npage cannot determine:\n - What the \"Duration\" number means (sum? mean? median?)\n - Whether tuning was applied (currently just a boolean)\n - Whether warmup queries are included in the timing\n - What execution mode was used (SQL vs DataFrame)\n - What hardware the result was produced on beyond OS/arch/CPU count\n\nThis matters because BenchBox's differentiator over ClickBench is\n\"rich per-query detail - execution plans, tuning configurations,\nvalidation status\". If a visitor can't interpret the numbers, that\nclaim is hollow. The absence of methodology disclosure also makes it\nimpossible to assess whether two results are meaningfully comparable.\n\nThis TODO adds a \"How this was measured\" disclosure section to the\nResultDetail page that surfaces the aggregation method, execution mode,\ntuning state, test type, and key environment parameters.", "id": "explorer-add-methodology-disclosure", "preserves": ["Collapsed by default - visitors who don't care are not burdened", "Existing ResultDetail layout above the disclosure panel - disclosure is additive", "Graceful rendering for bundles that lack execution_mode / tuning_mode (null-safe)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/components/MethodologyDisclosure.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}], "state": "done", "title": "Add methodology disclosure panel to result detail pages", "verifications": [{"command": "grep -n 'MethodologyDisclosure' results-explorer/src/pages/ResultDetail.tsx", "description": "MethodologyDisclosure component renders in ResultDetail", "expected": "imported and used", "item_id": "explorer-add-methodology-disclosure", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-add-methodology-disclosure", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'execution_mode.*null\\|null.*execution_mode\\|??' results-explorer/src/components/MethodologyDisclosure.tsx", "description": "Component renders for null execution_mode (old bundle compatibility)", "expected": "null-safe access present", "item_id": "explorer-add-methodology-disclosure", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": [], "notes": "Metric definition copy: \"Primary metric is geometric mean of per-query\nexecution times (measurement runs only). Wall-clock total is shown\nseparately. Lower geomean = faster.\"\n\nTest type: \"Power test (single-stream sequential)\" or \"Throughput test\n(N concurrent streams)\".\n\nQuery scope: \"Full benchmark (22 queries)\" or \"Query subset: Q1, Q6, Q17\".\n\nEnvironment: already surfaced (OS/arch/CPU/memory/Python). Add\nexecution_mode and tuning_mode here too.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the disclosure panel content: list each disclosed item (metric definition, execution mode, tuning mode, test type, query scope, environment summary) and write the copy for each.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement MethodologyDisclosure component in results-explorer/src/components/. Collapsible panel, collapsed by default. Accepts detail result props; renders the disclosed items as a definition list.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate MethodologyDisclosure into ResultDetail.tsx below the query timing table. Ensure it renders for all bundles including those predating the extended manifest (graceful null handling).", "wid": "w3"}, {"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add comparability summary row to Compare.tsx header: execution_mode + tuning_mode + test_type per result. Surfaces differences without navigating to detail pages.", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "lazy-fetch on expand only", "dont": "fetch the .tuning.json sidecar eagerly", "why": "most visitors won't expand the panel"}, {"INSTEAD": "warn visibly but allow", "dont": "hard-block cross-tuning comparisons", "why": "intentional tuned vs notuning comparisons are valid and valuable"}, {"INSTEAD": "show tuning_mode and tuning summary instead", "dont": "show tuning_hash to end users", "why": "it is an internal grouping key"}], "approach": "Tuning mode label is simple: read tuning_mode from manifest, render a\nstyled badge (Tuned=green, No Tuning=gray, Auto=blue). Mirror the\nTrustBadge component pattern.\n\nTuning config panel on detail page: lazy-fetch the .tuning.json sidecar\nURL (already derivable from bundle_path convention). Render as a\ncollapsible key-value table. Only fetch on expand - not on page load.\n\nCross-tuning comparison: the strategy doc says tuning_mode is a\ncomparability-breaking dimension that should warn, not hard-block. The\nwarning must be visually prominent (yellow banner) but must not prevent\nthe comparison from rendering - a user may intentionally compare tuned\nvs notuning to see the delta.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found that the explorer records only\n`has_tuning: boolean` despite tuning config being one of BenchBox's\nprimary differentiators. A visitor comparing DuckDB \"tuned\" vs \"notuning\"\nruns has no way to know they differ on this axis, let alone see what\ntuning was applied.\n\nThis TODO:\n 1. Surfaces tuning_mode (\"tuned\" / \"notuning\" / \"auto\") as a visible\n label on result detail pages and browse tables\n 2. Loads and displays the tuning config summary from the .tuning.json\n sidecar (if present) on the detail page\n 3. Enables the Compare view to show a tuning-mode label per result so\n cross-tuning comparisons are legible (not blocked - just labeled)\n\nPrerequisite: explorer-extend-manifest-and-pipeline must land first\n(tuning_mode and tuning_hash must be in the manifest).", "id": "explorer-add-tuning-config-visibility", "preserves": ["Existing compare view behavior when tuning is the same across both results (no regression)", "Results without a .tuning.json sidecar render cleanly - tuning panel shows 'No tuning config recorded' not an error", "TrustBadge layout unchanged - TuningBadge is a separate element"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (covered by prerequisite TODO)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/TuningBadge.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/"}], "state": "done", "title": "Surface tuning config in explorer detail pages and enable cross-tuning comparison", "verifications": [{"command": "grep -n 'TuningBadge\\|tuning_mode' results-explorer/src/pages/BenchmarkIndex.tsx", "description": "Tuning badge renders in browse tables", "expected": "TuningBadge imported and used", "item_id": "explorer-add-tuning-config-visibility", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'tuning_mode\\|Different tuning' results-explorer/src/pages/Compare.tsx", "description": "Cross-tuning warning renders in Compare", "expected": "warning logic present", "item_id": "explorer-add-tuning-config-visibility", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-add-tuning-config-visibility", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add tuning_mode label to BenchmarkIndex and PlatformIndex browse tables: small badge (styled like trust_label badge) showing 'Tuned' / 'No Tuning' / 'Auto'. Use tuning_mode from manifest.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add tuning section to ResultDetail.tsx: show tuning_mode prominently; collapsible panel lazy-fetches .tuning.json sidecar and renders key-value config.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update Compare.tsx: add tuning_mode label per result in the comparison header. If the two results have different tuning_mode values, show a visible 'Different tuning config' warning badge.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add tuning filter to DuckDB-WASM browse queries: allow filtering by tuning_mode in Browse pages (All / Tuned / No Tuning). Uses tuning_mode column added by pipeline extension.", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "read manifest.geomean_ms directly", "dont": "recompute geomean in the browser", "why": "it is precomputed in the manifest"}, {"INSTEAD": "demote it, don't delete it", "dont": "remove total_duration_s from the UI", "why": "it is useful context"}, {"INSTEAD": "always label it", "dont": "silently show geomean as 'Duration'", "why": "that hides the metric choice from the reader"}], "approach": "The geomean_ms field will exist in the manifest after explorer-extend-manifest-\nand-pipeline lands. This TODO is a UI-layer change: swap the field used for\nsorting, update column labels, and add a metric definition. No new data\nfetching or computation needed.\n\nTooltip copy: \"Geometric mean of per-query execution times (measurement\nruns only). More comparable than wall-clock total when query counts differ.\"\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found a metric discrepancy: the CLI uses\ngeometric mean of per-query times as the primary comparison metric; the\nexplorer uses total_duration_s (wall-clock sum). These produce different\nnumbers for the same result, and two results with different query counts\n(e.g., full TPC-H vs a query subset) produce incomparable totals.\n\nThis TODO:\n 1. Makes geomean_ms the primary sort/compare axis in the Compare view,\n Browse tables, and any cohort summary\n 2. Relabels total_duration_s wherever it appears so readers know what\n they are looking at\n 3. Adds a clear metric definition in the UI (tooltip or footnote)\n\nPrerequisite: explorer-extend-manifest-and-pipeline must land first\n(geomean_ms must exist in the manifest before this TODO can consume it).", "id": "explorer-align-duration-metric", "preserves": ["All data loading logic - this is display/labeling changes only", "Existing URL/query-param format for compare view", "Null-safety: if geomean_ms is null (e.g., old bundle), fall back to total_duration_s display with a 'N/A' geomean", "total_duration_s remains visible - it is honest wall-clock data; just secondary"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (pipeline changes are in the prerequisite TODO)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/components/ (no component changes needed)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/"}], "state": "done", "title": "Align explorer duration metric to geomean; retire wall-clock as primary comparison axis", "verifications": [{"command": "grep -n 'geomean_ms' results-explorer/src/pages/Compare.tsx", "description": "Compare view sorts by geomean_ms", "expected": "at least one sort reference to geomean_ms", "item_id": "explorer-align-duration-metric", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'Wall-clock\\|total_duration' results-explorer/src/pages/Compare.tsx", "description": "total_duration_s is still visible but labeled", "expected": "present with descriptive label", "item_id": "explorer-align-duration-metric", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-align-duration-metric", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-align-duration-metric", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update Compare.tsx: use geomean_ms as primary sort/compare axis. Keep total_duration_s as secondary column labeled 'Wall-clock total'.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-align-duration-metric", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update BenchmarkIndex.tsx and PlatformIndex.tsx browse tables: sort default by geomean_ms ascending. Relabel 'Duration' column to 'Geomean (ms)' with a tooltip explaining the metric.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-align-duration-metric", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update ResultDetail.tsx summary section: show geomean_ms as 'Geomean query time' prominently; move total_duration_s to secondary metadata row with label 'Total wall-clock'.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-align-duration-metric", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add metric definition tooltip or footnote: 'Geometric mean of per-query execution times (measurement runs only). Lower is faster. Comparable across runs with different query counts.'", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "show both, primary prominent, secondary muted", "dont": "drop the secondary metric from the UI", "why": "some users will want to cross-reference"}, {"INSTEAD": "serialize the choice into the summary artifact", "dont": "hard-code the benchmark \u2192 metric mapping in TypeScript", "why": "it creates two sources of truth"}, {"INSTEAD": "read the pre-computed value and null if absent", "dont": "recompute power_score in the pipeline", "why": "the CLI already emits it per-bundle"}], "approach": "The cleanest path is a single registry lookup in the pipeline.\nPipeline reads summary.benchmark, looks up the metric config, attaches\nit to the emitted BenchmarkSummary. The frontend is purely declarative\nfrom that point - no family logic in TypeScript.\n\nRegistry entries are strings keyed on canonical benchmark family names\n(tpch, tpcds, star_schema, clickbench, nyctaxi, tsbs-devops, h2odb,\ndatavault, ...). Unknown families fall back to geomean_ms asc.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-emit-benchmark-summary-artifact"], "description": "The strategy review (2026-04-15) identified that defaulting to geometric\nmean as the single ranking metric across all benchmarks is wrong:\n\n- TPC-H and TPC-DS define **power_score** (= 3600 / geomean \u00d7 scale\n adjustment) as the standard metric. Users arriving from published\n TPC results expect to see that number.\n- ClickBench and SSB have no published aggregate; geomean is a\n reasonable default there.\n- Custom or ad-hoc benchmarks should expose both so users can pick.\n\nThis TODO introduces a per-benchmark-family ranking metric registry and\nthreads it through the pipeline and UI. The matrix leaderboard uses\n`power_score` for TPC-H/TPC-DS and `geomean_ms` for everything else;\nboth are shown in each row, but the primary sort is the canonical one.\n\nScope excludes computing power_score (the CLI already does this) - this\nTODO only ensures the explorer surfaces and ranks by the right metric.", "id": "explorer-align-ranking-metric-with-tpc-standards", "preserves": ["CLI-computed power_score values - do not recompute in the pipeline; read from bundle metadata", "Existing `geomean_ms` field on platform rows - remains present as secondary metric for all benchmarks", "The 'Canonical Duration Metric' decision from 2026-04-14 - refines, does not overturn"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/benchbox-results-platform-strategy.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Select ranking metric per benchmark family (power_score for TPC, geomean elsewhere)", "verifications": [{"command": "grep -n 'RANKING_METRIC_BY_FAMILY' benchbox/core/explorer/models.py", "description": "Registry is defined and exported", "expected": "constant defined", "item_id": "explorer-align-ranking-metric-with-tpc-standards", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import gzip, json; d = json.loads(gzip.open(\"results-explorer/public/data/benchmarks/tpch_sf0.01_power.json.gz\").read()); assert d[\"ranking\"][\"primary_metric\"] == \"power_score\"'", "description": "tpch artifact declares power_score as primary metric", "expected": "no assertion failure", "item_id": "explorer-align-ranking-metric-with-tpc-standards", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c 'import gzip, json; d = json.loads(gzip.open(\"results-explorer/public/data/benchmarks/star_schema_sf0.01_power.json.gz\").read()); assert d[\"ranking\"][\"primary_metric\"] == \"geomean_ms\"'", "description": "star_schema artifact declares geomean_ms as primary metric", "expected": "no assertion failure", "item_id": "explorer-align-ranking-metric-with-tpc-standards", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": [], "notes": "Map benchmark family \u2192 {primary: str, secondary: str, primary_order:\n'asc'|'desc'}. Initial entries:\n - tpch \u2192 power_score (desc)\n - tpcds \u2192 power_score (desc)\n - star_schema \u2192 geomean_ms (asc)\n - clickbench \u2192 geomean_ms (asc)\nFallback for unknown families: geomean_ms (asc).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add RANKING_METRIC_BY_FAMILY registry in benchbox/core/explorer/models.py", "wid": "w1"}, {"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": ["w1"], "notes": "Shape: {primary_metric, secondary_metric, primary_order}. Lives\nalongside platform rows so the frontend does not need to know the\nregistry.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend BenchmarkSummary serialization with a top-level `ranking` field", "wid": "w2"}, {"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": ["w2"], "notes": "Display primary metric in a dedicated column with its unit;\ndisplay secondary metric below it in muted text. Power score has\nno unit suffix (composite); geomean is \"Xms\". Column header is\ndynamic: \"Power Score\" vs \"Geomean\" vs family-specific label.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In QueryHeatmap.tsx, read summary.ranking and sort/display by primary_metric", "wid": "w3"}, {"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cross-reference RANKING_METRIC_BY_FAMILY in the strategy doc's 'Canonical Duration Metric' section", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change sort behavior in w1 for any currently-registered benchmark family", "why": "only the mechanism changes"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-emit-benchmark-summary-artifact. The two Required issues\n(missing percentile_stats test, missing sort-order test) were fixed\nimmediately post-review. This item tracks the remaining nit/consider findings.", "id": "explorer-benchmark-summary-review-followups", "preserves": ["_natural_sort_key output for all current query IDs - empty-string filter is behavior-equivalent in w2", "_platform_row_sort_key behavior for all current benchmark families - only extensibility changes in w1"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from benchmark-summary artifact code review", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_benchmark_summary.py -v -n 0", "description": "Benchmark summary tests still pass", "expected": "16 passed", "item_id": "explorer-benchmark-summary-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Full fast test suite green", "expected": "all green", "item_id": "explorer-benchmark-summary-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:78-87 - _platform_row_sort_key hardcodes sort direction: negate\npower_score (higher-is-better), use geomean_ms as-is (lower-is-better).\nThis is correct for all current benchmark families but duplicates logic\nalready encoded in RankingConfig.primary_order.\n\nIf a future family added a metric where the direction didn't match these\nassumptions, the sort would silently be wrong.\n\nFix: pass `benchmark` name to _platform_row_sort_key and look up\nget_ranking_config(benchmark).primary_order to derive the negation flag:\n cfg = get_ranking_config(benchmark)\n primary_val = getattr(row, cfg.primary_metric)\n if primary_val is None:\n return (ineligible, float(\"inf\"))\n metric = -primary_val if cfg.primary_order == \"desc\" else primary_val\n return (ineligible, metric)\n\nNote: _build_benchmark_summaries already has the benchmark name in scope\nand already calls get_ranking_config(benchmark) at line 142.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consult RankingConfig.primary_order in _platform_row_sort_key", "wid": "w1"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:54-56 - re.split(r\"(\\d+)\", s) produces empty strings between\nnumber groups (e.g., \"Q10\" \u2192 [\"\", \"Q\", \"10\", \"\"]). Python's sort handles\nempty strings correctly (they sort first), but the intent is clearer with:\n return [int(c) if c.isdigit() else c.lower() for c in re.split(r\"(\\d+)\", s) if c]\n\nLow-risk: behavior is identical for all current query IDs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Filter empty strings from _natural_sort_key output", "wid": "w2"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "transformer.py:272 - filter `dt.display_ms is not None and dt.display_ms > 0`\nexcludes zero-millisecond measurements. For extremely fast queries (sub-ms\nrounded to 0), this would silently exclude them from percentile stats.\n\nFix: add a comment explaining the reason - presumably 0ms indicates a\ntiming error or warmup artifact, not a genuine instant result. If the\nintent is to exclude non-positive values, document it. If 0ms is valid,\nchange to `>= 0`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document why percentile filter uses display_ms > 0 not >= 0", "wid": "w3"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:381,395 - Path traversal check uses string prefix:\n if not str(detail_path).startswith(str(details_dir.resolve()) + \"/\"):\n\nPython 3.9+ has Path.is_relative_to() which is more readable and avoids\nthe string manipulation:\n if not detail_path.is_relative_to(details_dir.resolve()):\n\nFunctionally identical; Python \u22653.9 is already required by the project.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Use Path.is_relative_to() for path-traversal guard", "wid": "w4"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:165-256 - _build_meta_leaderboard is ~90 lines with nested\nfunctions and multiple accumulation passes. Consider extracting:\n _rank_platforms_in_cohort(summary) \u2192 list[dict]\n _compute_average_ranks(cohort_rows) \u2192 dict[str, float]\n\nNot urgent; logic is correct and well-commented. Low value until the\nfunction needs modification.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor _build_meta_leaderboard into smaller helpers", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "add new reductions under `benchbox/core/explorer_pipeline/` and import existing utilities read-only from `benchbox/core/results/` as needed", "dont": "add new reduction code under `benchbox/core/results/`", "why": "this TODO's scope_limit keeps that tree read-only"}, {"INSTEAD": "migrate every metric-bearing surface", "dont": "allow Home, PlatformIndex, BenchmarkIndex, Compare, or ResultDetail to keep metric reads outside DuckDB", "why": "that violates the core architecture goal"}, {"INSTEAD": "key `cohort_metadata` on `(cohort_key, result_id)`, carry `tuning_mode`/`trust_label`/`short_id` alongside, and let the UI decide any display-time collapsing", "dont": "dedupe `cohort_metadata` rows by `(cohort_key, platform_id)` or hide publishable variants of the same platform in any canonical table", "why": "users must be able to compare across tuning levels and trust levels from the DuckDB store, and the pipeline is the wrong layer to decide which variant 'represents' a platform"}, {"INSTEAD": "W2 is additive, W4 deletes each emitter in the same slice as its last consumer, W7 does the final sweep", "dont": "delete metric-JSON emitters in W2", "why": "Home/PlatformIndex/BenchmarkIndex/Compare/ResultDetail still read from them until W4"}, {"INSTEAD": "move all interactive metric reads behind one DuckDB contract", "dont": "keep separate canonical logic in JSON loaders and DuckDB helpers", "why": "that recreates the current drift risk"}, {"INSTEAD": "enforce via the G-8 second assertion that no `src/pages/` or `src/components/` file fetches `/results/data/bundles/`", "dont": "let `public/data/bundles/*.json` drift into a rendered metric path", "why": "bundles are committed JSON source inputs and a download-only affordance, not a runtime read source"}, {"INSTEAD": "keep any surviving JSON bootstrap-only and enforce via a key-whitelist contract test", "dont": "let bootstrap convenience justify metric-bearing `manifest.json` or `results_schema.json` sidecars", "why": "that silently recreates split-brain storage"}, {"INSTEAD": "optimize for the correct canonical model, not compatibility theater", "dont": "preserve beta-era JSON metric contracts just because they already exist", "why": "the explorer has not shipped and the current contract is wrong"}, {"INSTEAD": "persist canonical values in DuckDB and read them directly", "dont": "recompute display reductions, rankings, compliance classes, speedup ratios, or fastest/slowest spreads in TypeScript", "why": "the canonical Python reference computation in `benchbox/core/explorer_pipeline/` owns those semantics"}, {"INSTEAD": "compare ranks on write and keep the best (or rely on best-first iteration order with `setdefault`)", "dont": "rely on dict last-write-wins inside `_build_meta_leaderboard` `platform_agg` when the same `platform_id` ranks more than once in the same cohort", "why": "the second write clobbers the better rank with a worse one and poisons `avg_rank` for the Home meta-leaderboard"}, {"INSTEAD": "prove final-value parity against the canonical Python reference computation instead", "dont": "require derived user-visible metrics to equal verbatim JSON source fields", "why": "many visible values are canonical reductions or rollups"}, {"INSTEAD": "lock the rules first and record them", "dont": "start implementation before the research gate closes the schema and cutover details", "why": "this is a contract refactor across pipeline and UI"}, {"INSTEAD": "W3 keeps the fallback in place; W4 lands the hardening in the same slice that deletes `manifest.ts`", "dont": "tighten `db.ts` to reject on attach failure until the last manifest.ts consumer migrates", "why": "pages still on the JSON path rely on the soft failure path"}, {"INSTEAD": "define parity as `DuckDB read-path value == canonical Python reference computation in `benchbox/core/explorer_pipeline/` (with read-only imports from `benchbox/core/results/`)`, within the per-metric tolerance declared in `visible_metrics.yaml`", "dont": "treat pre-calculated CLI JSON artifacts, current TS reduction libs, or any bridge artifact as the parity oracle", "why": "doing so freezes today's drift into tomorrow's contract"}, {"INSTEAD": "leave write-path contracts untouched", "dont": "widen scope into CLI, hosted API, or submission-path changes", "why": "this TODO is about the static explorer read model only"}], "approach": "Follow a research-first but opinionated cutover, not an additive\ncompatibility migration.\n\nFirst, inventory every runtime metric fetch and every pipeline artifact that\ncurrently feeds it. Then define the DuckDB schema and materializations that\nreplace those paths. After the research gate, wire the explorer pipeline\ndirectly from committed JSON objects into DuckDB, move all metric surfaces\nonto typed DuckDB queries, and remove metric-bearing JSON outputs and\nloaders.\n\nTemporary parity checks are allowed as verification scaffolding, but JSON is\nnot a second product contract. If a surface shows metrics to the user, those\nmetrics must exist in DuckDB first.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-18T00:00:00Z", "completed_pr": null, "created_at": "2026-04-18T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-canonical-browser-duckdb-read-model", "id": 403, "reason": "This TODO is static-first only; the read path remains browser-side and read-only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicit scope decision: explorer stays static/browser-side; introducing a hosted/authenticated API is a declined direction, not deferred work.", "summary": "Introduce any hosted API, authenticated query service, or server-side execution path"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-canonical-browser-duckdb-read-model", "id": 404, "reason": "The adopted product strategy keeps benchmark landing pages matrix-first and the query workbench secondary.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Adopted product strategy explicitly keeps matrix-first landing pages; making /results/query the default landing page was rejected.", "summary": "Make `/results/query` the default landing page for the explorer"}], "deps": [], "description": "Rework the results explorer so benchmark landing pages stay matrix-first\nwhile DuckDB-WASM becomes the single runtime store for every user-visible\nmetric. Today the implementation is split-brain: `Query.tsx` already runs\nagainst DuckDB, but `Home.tsx`, `PlatformIndex.tsx`, `BenchmarkIndex.tsx`,\n`Compare.tsx`, and `ResultDetail.tsx` still fetch metric-bearing JSON\nthrough `results-explorer/src/manifest.ts`.\n\nThe target contract is stricter than \"prefer DuckDB when convenient\":\ncommitted JSON result objects are immutable source inputs; the explorer\npipeline projects them directly into `results.duckdb`; and every\nuser-visible metric, aggregate, rank, compare cell, detail timing, and\ncohort summary is read from DuckDB. Any JSON that survives after this work\nis bootstrap-only navigation metadata and must not carry user-visible\nmetrics.\n\nThis item still needs a research gate before implementation, but that gate\nis for schema and cutover precision, not to re-open whether JSON remains a\nfirst-class metric surface. The end state is DuckDB-only for explorer\nmetrics.\n\nVerified current-state anchors:\n - `results-explorer/src/db.ts` attaches `/results/data/results.duckdb`\n with `ATTACH ... AS bench (READ_ONLY)`.\n - `results-explorer/src/pages/Query.tsx` already queries `bench.results`\n but still bootstraps schema from `results_schema.json`.\n - `results-explorer/src/manifest.ts` fetches `manifest.json`,\n `benchmarks/*.json.gz`, `details/*.json`, `compare/*.json`,\n `meta_leaderboard.json`, and `short_ids.json`, making JSON the\n dominant runtime read path today.\n - `benchbox/core/explorer_pipeline/duckdb_builder.py` currently emits only\n a thin `results` table plus `results_schema.json`, so DuckDB cannot yet\n serve Home, platform, benchmark, compare, and detail surfaces alone.", "id": "explorer-canonical-browser-duckdb-read-model", "preserves": ["Any temporary bridge artifact used during W2-W4 page-by-page migration is regenerated from the same pipeline pass that writes DuckDB, never hand-patched, and cannot define, override, or outlive user-visible metrics", "Benchmark landing pages remain matrix-first default routes; `/results/query` stays a secondary analysis surface rather than the primary explorer entrypoint", "Committed JSON result objects remain immutable source inputs; the browser does not fetch them directly for metric rendering", "DuckDB is always attached read-only in the browser; no implementation step may weaken `ATTACH ... (READ_ONLY)` or permit persistent mutation from the SQL workbench", "Every result trusted enough to publish appears in the DuckDB published-results dataset with its identity preserved (`result_id`, `tuning_mode`, `trust_label`); the pipeline never silently collapses, dedupes, or hides publishable variants - users must be able to compare across tuning levels AND trust levels from the same store", "Every user-visible metric, rank, aggregate, and detail timing shown in the explorer originates from DuckDB rather than JSON sidecars or TypeScript-only recomputation", "Final-value parity for derived metrics is defined against the canonical Python reference computation in `benchbox/core/explorer_pipeline/` (with read-only imports from `benchbox/core/results/`), NOT against pre-calculated CLI JSON, current TS reduction libs, or bridge artifacts", "Raw JSON bundles under `public/data/bundles/` are a download-only affordance; no `src/pages/` or `src/components/` file fetches from `/results/data/bundles/` at runtime", "Shareable compare flows remain available after cutover, but route convenience is subordinate to the DuckDB-only metric contract", "The explorer remains static-hosted and browser-executed; no server API, auth dependency, or hosted query runtime is introduced by this TODO"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "docs/reference/hosted-results-contract.md"}, {"kind": "only_modify", "path_glob": "_project/planning/visible_metrics.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "docs/development/benchbox-results-platform-strategy.md"}, {"kind": "only_modify", "path_glob": "docs/development/browser-duckdb-schema.sql"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/"}, {"kind": "only_modify", "path_glob": "results-explorer/playwright.config.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}], "state": "done", "title": "Make DuckDB-WASM the sole browser store for all user-visible explorer metrics", "verifications": [{"command": "rg -n \"DuckDB-only metric contract|direct ingest contract|bootstrap-only|metric-bearing JSON\" _project/TODO/main/planning/explorer-canonical-browser-duckdb-read-model.yaml docs/development || true", "description": "Research gate records the DuckDB-only metric contract before code work begins", "expected": "The contract and cutover rules are recorded before implementation starts", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/TODO/main/planning/explorer-canonical-browser-duckdb-read-model.yaml && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO metadata and graph validate cleanly", "expected": "TODO file validates; dependency graph check passes", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -q", "description": "Explorer pipeline tests remain green after DuckDB schema expansion", "expected": "all tests pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck && npm test && npm run build", "description": "Explorer frontend typecheck, tests, and production build remain green", "expected": "all green", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -c \"import duckdb; con=duckdb.connect('results-explorer/public/data/results.duckdb', read_only=True); names={r[0] for r in con.execute('PRAGMA show_tables').fetchall()}; required={'results','query_display_timings','query_executions','result_detail_metrics','benchmark_matrix_cells','benchmark_rankings','platform_index_rows','cohort_metadata','meta_leaderboard','short_ids'}; missing=required-names; assert not missing, missing\"", "description": "Canonical DuckDB base tables exist for the browser metric contract", "expected": "no assertion failure", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 5}, {"command": "if rg -n \"/results/data/(benchmarks/|details/|compare/|meta_leaderboard\\.json|results_schema\\.json|short_ids\\.json)\" results-explorer/src; then exit 1; else echo CLEAN; fi", "description": "No runtime metric JSON fetches remain in the explorer source tree", "expected": "CLEAN", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 6}, {"command": "if rg -n \"/results/data/bundles/\" results-explorer/src/pages results-explorer/src/components; then exit 1; else echo CLEAN; fi", "description": "No page or component fetches the raw-bundle download path", "expected": "CLEAN", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 7}, {"command": "bash -c 'for f in meta_leaderboard.json short_ids.json results_schema.json; do if [ -e \"results-explorer/public/data/$f\" ]; then echo FAIL_$f; exit 1; fi; done; for d in details compare benchmarks; do if [ -d \"results-explorer/public/data/$d\" ]; then echo FAIL_$d; exit 1; fi; done; echo CLEAN'", "description": "Pipeline output contains no legacy metric-bearing artifacts (bundles/ is allowed as download-only)", "expected": "CLEAN", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 8}, {"command": "bash -c 'f=results-explorer/public/data/manifest.json; if [ ! -e \"$f\" ]; then echo ABSENT; exit 0; fi; uv run -- python -c \"import json,sys; keys=set(json.load(open(\\\"$f\\\")).keys()); allowed={\\\"routes\\\",\\\"navigation\\\",\\\"build_meta\\\",\\\"schema_version\\\"}; extra=keys-allowed; sys.exit(0 if not extra else (print(\\\"EXTRA_KEYS:\\\",sorted(extra)) or 1))\"'", "description": "Surviving manifest.json (if any) carries only whitelisted non-metric keys", "expected": "ABSENT or no extra keys", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 9}, {"command": "git diff --name-only main...HEAD -- benchbox/core/results/ | wc -l | tr -d ' '", "description": "No reduction code was added under `benchbox/core/results/` by this TODO", "expected": "0", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 10}, {"command": "cd results-explorer && npm test -- duckdbMetricParity", "description": "Cross-surface parity holds for canonical display values", "expected": "home, benchmark, compare, detail, and query parity tests pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 11}, {"command": "cd results-explorer && npm test -- Query", "description": "Read-only SQL enforcement still holds", "expected": "DDL/DML attempts fail; read-only SELECT flows still pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 12}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast Python suite stays green after the migration", "expected": "all tests pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 13}], "work": [{"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": [], "notes": "Deliverables:\n 1. A concrete call-site inventory for Home, PlatformIndex,\n BenchmarkIndex, Compare, ResultDetail, and Query showing the\n current data source and the target DuckDB table/view for each\n user-visible metric.\n 2. A schema decision record mapping every user-visible explorer metric\n to DuckDB tables/views, plus the minimal bootstrap JSON that may\n remain and proof that it carries no metrics.\n 3. A direct ingest contract documenting `committed JSON -> explorer\n pipeline -> DuckDB` and explicitly rejecting metric-bearing JSON as\n a steady-state runtime contract.\n 4. A cutover/deletion order showing which JSON readers and JSON\n artifacts are removed or demoted once DuckDB parity is proven.\n 5. A committed `visible_metrics.yaml` fixture enumerating every\n user-visible metric column across the ten canonical tables and\n classifying each as `raw_copy` (source-fidelity contract against\n the committed bundle field) or `derived` (final-value parity\n contract against the canonical Python reference computation).\n This file is the registry G-2 consumes and is required before W2\n starts.\n 6. A definition-of-terms block pinning the meaning of the parity\n vocabulary used across this TODO:\n - \"canonical Python reference computation\" = the pipeline's own\n Python implementation inside\n `benchbox/core/explorer_pipeline/` (plus read-only imports\n from `benchbox/core/results/`). TypeScript is never a\n reference oracle.\n - \"final-value parity\" = the rendered user-visible value\n produced by the DuckDB-backed read path equals the canonical\n Python reference computation for the same inputs, within a\n documented tolerance per metric. Parity is NOT defined\n against whatever the CLI, a pre-calculated JSON artifact, or\n the current TS reduction libraries happen to emit today.\n - \"source-fidelity parity\" = raw fields copied into DuckDB\n equal the corresponding field in the committed JSON bundle\n verbatim.\n 7. A compute-ownership note answering: for every derived metric in\n `visible_metrics.yaml`, does the canonical Python reference\n computation already exist under\n `benchbox/core/explorer_pipeline/` or `benchbox/core/results/`,\n or must a new module be added under\n `benchbox/core/explorer_pipeline/` (the only write-legal home for\n new reduction code under this TODO's scope_limit)? If a derived\n metric is TS-only today (e.g. currently lives in\n `queryReduction.ts`, `chartMath.ts`, `ranking.ts`, or\n `compliance.ts`), record it here and plan to port the logic into\n `benchbox/core/explorer_pipeline/` during W2.\n\nQuantified research gates (each must be green or explicitly risk-accepted\nbefore w2 starts; record findings in the schema decision record above):\n\n - RG-1 Cold-start init budget: measure time from page load to first\n successful SQL result against `results.duckdb` (cold cache, mid-tier\n laptop, Safari/Chrome/Firefox). Pass: p50 <1500ms, p95 <3500ms at\n current corpus; p95 <5000ms at 10x corpus.\n - RG-2 HTTP range-reads: verify `DuckDBDataProtocol.HTTP` with\n READ_ONLY ATTACH issues range requests and that benchbox.dev /\n GH Pages returns `Accept-Ranges: bytes` with 206 responses for a\n single-row `WHERE result_id = ?` query. Pass: the sum of HTTP\n Content-Length bytes transferred for a cold single-row lookup is\n \u226410% of the total `results.duckdb` file size, measured on the\n current corpus.\n - RG-3 DB size budget: build DB at 1x, 5x, 20x, 100x synthetic corpus\n (duplicated bundles with mutated ids). Record the `results.duckdb`\n file size on disk at each scale and publish the 1x baseline in the\n schema decision record. Pass: on-disk `results.duckdb` size grows\n sub-linearly in bundle count AND stays \u226420 MB at 100x. If the\n measured 1x baseline makes 20 MB at 100x structurally impossible\n (e.g. baseline already >1 MB per bundle), record a revised numeric\n ceiling in the decision record and escalate to RG-5 evaluation.\n - RG-4 COEP/COOP hosting: confirm either the `mvp` single-threaded\n DuckDB-WASM bundle meets RG-1 budgets, OR COEP/COOP headers can be\n served by the deploy target. Pass: a chosen WASM bundle works with\n the current hosting configuration.\n - RG-5 Single DB vs partitioned: only if RG-3 fails. Prototype\n multi-DB `ATTACH` from WASM. Default is single DB per the \"one and\n only\" rule; partitioning is a fallback, not a preference.\n - RG-6 View materialization budget: benchmark each canonical surface\n query cold against 100x corpus. Pass: <250ms p95 as a plain VIEW,\n otherwise promote to materialized `CREATE TABLE AS`.\n - RG-7 Attach-failure UX: every page renders a diagnostic ErrorMessage\n when the DB fails to attach (0-byte, 404, CORS). Pass: unit-tested\n per page, no silent JSON fallback.\n - RG-8 toJSON fidelity: round-trip every column type through\n `queryRows` in a Vitest matrix. Pass: no precision loss on DOUBLE /\n BIGINT / DECIMAL; date handling is explicit.\n - RG-9 Read-only fuzz: adversarial SQL (CREATE TABLE, ATTACH, COPY TO,\n INSTALL, LOAD, HTTPFS) against the READ_ONLY attached DB in WASM.\n Pass: all fail harmlessly with no side effects.\n\nResearch gate exit criterion: all RGs green or documented accepted-risk\nnote reviewed with the user, and a frozen `schema.sql` fixture committed\nalongside the decision record.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research gate: inventory every metric-bearing JSON read, define the DuckDB-only contract, and lock the cutover rules before implementation", "wid": "w1"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w1"], "notes": "Minimum DuckDB-resident contracts (tables or materialized views):\n - `results`\n - `query_display_timings`\n - `query_executions`\n - `result_detail_metrics`\n - `benchmark_matrix_cells`\n - `benchmark_rankings`\n - `platform_index_rows`\n - `cohort_metadata`\n - `meta_leaderboard`\n - `short_ids`\nFinal names can change in w1, but every surface above must have an\nexplicit DuckDB contract before frontend migration begins.\n\nPipeline implementation rules:\n - `duckdb_builder.py` rewrites the 19-column stub (currently lines\n 25-131) into a full-schema builder. Uses `CREATE OR REPLACE` on each\n build; the corpus is the only source of truth, so there are no\n migrations.\n - `pipeline.py` `run()` becomes: enumerate `bundles/*.json`, validate\n via `SchemaV2Validator`, compute derived values in memory (display\n timings, short_id, is_ranking_eligible, compliance_class, matrix\n cells, rankings, meta-leaderboard rows, cohort metadata), bulk\n insert in one transaction, copy raw bundles to\n `public/data/bundles/` for the raw-bundle download affordance ONLY.\n - All derived-value reductions consumed by the pipeline live under\n `benchbox/core/explorer_pipeline/` or are imported read-only from\n `benchbox/core/results/`. This is the canonical Python reference\n computation for G-2 parity. If a reduction is currently TS-only\n (e.g. in `queryReduction.ts`, `chartMath.ts`, `ranking.ts`,\n `compliance.ts`), port the algorithm into\n `benchbox/core/explorer_pipeline/` in this work item; the TS copy\n becomes dead code to be deleted in W4 once no page imports it.\n - W2 is ADDITIVE on the pipeline side. The pipeline starts emitting\n the ten canonical tables and keeps all existing metric-bearing\n JSON emitters running as bridge artifacts until the consuming\n page migrates in W4. No metric-JSON emitter is deleted at W2\n exit. This is the only reading that is consistent with \"do not\n remove the artifact still backing a live page until that page\n migrates\" - W4 removes each emitter in the same slice as its\n last consumer, and W7 does the final sweep.\n - Bridge artifacts are scaffolding, never a second accepted metric\n contract. During W2-W4, if a bridge artifact and DuckDB disagree,\n DuckDB is canonical by rule and the bridge artifact must be\n regenerated from the same pipeline pass, not hand-patched.\n - If `manifest.json` survives past W7, it is strictly non-metric\n route/navigation bootstrap derived from DuckDB-owned identities\n only. A key-whitelist contract test (see G-8) enforces this.\n - No TypeScript-side recomputation of display reductions, geomean,\n rank, compliance class, speedup ratios, or any other user-visible\n metric is permitted downstream. TS is allowed to do presentation\n formatting (locale formatting, unit suffixes, truncation) on\n values already produced by the canonical Python reference\n computation and read from DuckDB.\n\nExit:\n - All ten canonical tables/materializations exist with their W1-\n defined schemas (G-1).\n - Source-fidelity tests pass for every `raw_copy` metric in\n `visible_metrics.yaml` (raw bundle field == DuckDB column, byte/\n value exact).\n - Final-value parity tests pass for every `derived` metric in\n `visible_metrics.yaml`: for the same inputs, the DuckDB-backed\n read path and the canonical Python reference computation produce\n the same user-visible value within the per-metric tolerance\n declared in the fixture. Parity is NOT defined against\n pre-calculated CLI JSON artifacts or current TS reductions.\n - Schema contract test passes; short-id uniqueness test passes;\n no-computed-metric-view contract test passes.\n - All existing metric-bearing JSON emitters still run as bridge\n artifacts; `public/data/` still contains the current shape.\n Removal is deliberately deferred to W4/W7.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor the explorer pipeline so DuckDB is built directly from committed JSON artifacts and materializes every user-visible metric surface", "wid": "w2"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w8"], "notes": "Consolidate TS-side data access around `results-explorer/src/db.ts`\nplus typed query helpers. Retire `results-explorer/src/manifest.ts` as\na metric source. If manifest/bootstrap JSON survives, it is route and\nnavigation metadata only. `Query.tsx` is fully migrated in this work\nitem (it already queries DuckDB - W3 removes its last JSON\ndependency, `results_schema.json`) and is NOT reopened in W4. W4\nhandles Home, PlatformIndex, BenchmarkIndex, ResultDetail, and\nCompare. Query's schema discovery uses DuckDB introspection\n(`duckdb_tables`, `duckdb_columns`, `DESCRIBE`) or a DuckDB-resident\n`schema_metadata` table if introspection lands in the RG-1 hot path.\nThe `db.ts:1-17` header comment (which still calls DuckDB a \"Phase 2\"\nfeature) is rewritten in this work item to match the new contract.\n\nNew modules:\n - `results-explorer/src/lib/duckdbQueries.ts` - typed helpers per\n canonical surface: `getMetaLeaderboard`, `getBenchmarkMatrixCells`,\n `getBenchmarkRanking`, `getPlatformIndexRows`,\n `getResultDetailMetrics`, `getQueryDisplayTimings`, `getCohort`,\n `resolveShortId`, `listResults`.\n - `results-explorer/src/lib/duckdbSchema.ts` - introspection-based\n schema discovery for the Query workbench column picker.\n\n`db.ts` hardening sequencing:\n - In W3 the silent JSON fallback at `db.ts:86-89` STAYS in place.\n Removing it while Home/PlatformIndex/BenchmarkIndex/ResultDetail/\n Compare still import from `manifest.ts` would turn an attach\n failure into a hard error for pages that don't yet use DuckDB.\n - What W3 adds now: a `DbProvider` context exposing `{ db, schema,\n state }` and a `useQuery(sql, params, deps)` hook with\n `{ loading, error, data }` so pages do not each guard on init.\n These are opt-in; manifest-backed pages continue to work.\n - The attach-failure hardening - reject `getDb()` on attach\n failure, delete the silent JSON fallback, and wire per-page\n ErrorMessage UI (RG-7) - moves into W4 and lands in the SAME\n slice as the final manifest.ts consumer's migration. This\n guarantees no mid-migration tab state where some pages still\n depend on a soft attach failure.\n\nExit:\n - Vitest coverage on the data-layer helpers.\n - In-browser cold-start measurement confirms RG-1 budgets.\n - `Query.tsx` no longer depends on `results_schema.json`.\n - `duckdbQueries.ts` + `duckdbSchema.ts` land with typed tests.\n - The shared DuckDB data layer is ready for page-by-page migration\n in W4.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build the shared frontend DuckDB access layer and cut `Query.tsx` off from `results_schema.json`; defer `db.ts` attach-failure hardening until W4", "wid": "w3"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w3"], "notes": "Every user-visible number on these surfaces must originate from DuckDB\nqueries or DuckDB-backed helpers. Benchmark landing pages remain the\ndefault graph surface. `/results/query` remains secondary in product\npositioning, but the same DuckDB contract must back both flows. Query\nwas fully migrated in W3 and is not reopened here.\n\nSuggested migration order (one page per slice, commit per page):\nHome -> PlatformIndex -> BenchmarkIndex -> ResultDetail -> Compare.\n\nPer-slice rule: when a page migrates to DuckDB, the SAME commit\n (a) deletes that page's imports from `manifest.ts`,\n (b) deletes the metric-JSON emitter in the pipeline that backed\n that page (e.g. `meta_leaderboard.json` with Home,\n `benchmarks/*.json.gz` with BenchmarkIndex, `details/*.json`\n with ResultDetail, `compare/*.json` + `short_ids.json` with\n Compare),\n (c) deletes the TS reduction module(s) whose last caller\n disappeared (typically `queryReduction.ts`, `chartMath.ts`,\n `ranking.ts`, `compliance.ts` on various slices), since the\n canonical Python reference computation was ported into\n `benchbox/core/explorer_pipeline/` in W2.\n\nPer-page rule: never recompute a user-visible metric in TypeScript\n(display_ms, geomean_ms, rank, speedup, compliance class, speedup\nratios, fastest/slowest spreads). Read materialized columns from the\ncanonical tables. Presentation formatting (locale, unit suffixes,\ntruncation) remains TS-side.\n\nDelete `results-explorer/src/manifest.ts` in the final slice once the\nlast consumer is gone. In that same final slice, land the `db.ts`\nattach-failure hardening deferred from W3:\n - Reject `getDb()` on attach failure instead of swallowing the\n error.\n - Delete the silent JSON fallback at `db.ts:86-89` and the stale\n `db.ts:1-17` header comment's \"Phase 2 / fallback\" framing.\n - Wire the per-page ErrorMessage UI required by RG-7.\nThis co-scheduling is intentional: the fallback must not outlive the\nlast page that relied on it, and must not be removed before then.\n\nPrune transport-only JSON artifact interfaces from\n`results-explorer/src/types.ts` once their last caller disappears. Do\nnot delete render-layer domain types solely because the transport\nchanged; keep, move, or rename them only when DuckDB row models make\nthe old shape redundant.\n\nExit: chart-parity and accessibility tests green on every migrated\npage; no page imports `manifest.ts`; `db.ts` rejects on attach\nfailure with no JSON fallback; TS reduction libs whose callers are\ngone are deleted (not just dead-coded); any removed TS types are\nconfirmed unused rather than deleted by naming policy alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate remaining pages to DuckDB, delete each bridge artifact in the same slice, and land `db.ts` hardening with the last manifest consumer", "wid": "w4"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w4"], "notes": "Starter queries must cover common analyst questions across results,\nper-query timings, cohort comparisons, trust/tuning breakdowns, and\ndetail-level drill-down. Keep the SQL surface read-only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand the query workbench into the canonical analyst surface with starter queries, cohort-aware templates, and downloads against the same DuckDB contract", "wid": "w5"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w4", "w5"], "notes": "Required assertions:\n - home stats and leaderboard rows == DuckDB query results\n - benchmark matrix cell == compare cell == detail timing == SQL query\n result where applicable\n - no runtime metric fetches remain for `benchmarks/*.json.gz`,\n `details/*.json`, `compare/*.json`, `meta_leaderboard.json`,\n `short_ids.json`, or `results_schema.json`\n - read-only DuckDB enforcement still blocks DDL/DML from the\n workbench\n\nExit-gate inventory (all runnable locally - CI wiring is out of scope\nper `do_not_modify: [\".github/workflows/\"]` and is filed as a\nfollow-up TODO):\n\n - G-1 Schema contract:\n `tests/unit/core/explorer_pipeline/test_duckdb_browser_contract.py`\n asserts all ten canonical tables exist with expected columns.\n - G-2 Metric parity (driven by `visible_metrics.yaml` from W1):\n for every metric row in the fixture, the test dispatches on the\n `kind` field and compares the DuckDB read-path value against the\n contract for that kind:\n - `raw_copy`: assert `DuckDB column == committed bundle field`\n verbatim for a sampled corpus.\n - `derived`: assert `DuckDB-backed read path value ==\n canonical Python reference computation` for the same\n inputs, within the `tolerance` declared per metric. The\n canonical Python reference computation lives under\n `benchbox/core/explorer_pipeline/` (or is imported read-only\n from `benchbox/core/results/`); it is NOT the CLI's\n pre-calculated JSON, NOT the current TS reduction libraries,\n and NOT any bridge artifact. Final-rendered-value parity is\n the contract.\n - G-3 Cross-surface parity:\n `results-explorer/src/__tests__/duckdbMetricParity.test.ts` asserts\n Home stats == `meta_leaderboard`, benchmark matrix cell ==\n compare cell == detail timing == direct SQL result.\n - G-4 Chart parity (React): extend existing parity harness to feed\n DuckDB-shaped rows through every component; DOM must match the\n ASCII baseline.\n - G-5 Performance budgets (local): Node + `@duckdb/duckdb-wasm`\n harness loads a 100x synthetic fixture; asserts init p95 <3500ms,\n per-query p95 <250ms (or RG-6 budget), DB file size \u2264 RG-3 ceiling.\n - G-6 Accessibility: existing axe-core suite re-run against\n DuckDB-fed pages.\n - G-7 Headless E2E: Playwright (added under this TODO as new\n tooling; listed in `files_affected.added`) loads `/results/`,\n waits for DB ready, navigates, asserts a DuckDB-sourced row\n renders, opens the workbench, runs `SELECT COUNT(*) FROM\n results`, and asserts attach-failure UX by loading a broken\n fixture.\n - G-8 Legacy-artifact guard: post-build assertion fails if any of\n `details/`, `compare/`, `meta_leaderboard.json`, `short_ids.json`,\n `results_schema.json`, or `benchmarks/*.gz` appear under\n `public/data/`. Bundles handling: `public/data/bundles/` MAY\n remain as the raw-bundle download affordance, but a second\n assertion greps the explorer source tree and fails if any\n `.tsx`/`.ts` file under `results-explorer/src/pages/` or\n `results-explorer/src/components/` fetches from\n `/results/data/bundles/` - bundles are download-only and must\n never back a rendered metric. If `manifest.json` survives, a\n dedicated contract test loads it and asserts its top-level keys\n are a subset of an explicit whitelist recorded in the schema\n decision record (W1 deliverable 2), such as `{routes,\n navigation, build_meta}`. Any metric-looking key\n (`benchmark_summary`, `result_count`, `platform_version`,\n `power_score`, `geomean_ms`, etc.) appearing in that file fails\n the gate.\n - G-9 Read-only fuzz: Vitest runs ~50 adversarial SQL strings\n (CREATE TABLE, ATTACH, COPY TO, INSTALL, LOAD, HTTPFS) against the\n READ_ONLY-attached DB; all fail harmlessly. The test is agnostic\n about HOW each fails - whether because the extension is compiled\n out of the chosen DuckDB-WASM bundle or because READ_ONLY attach\n rejects it at runtime. Open question #6 is resolved by whichever\n mechanism satisfies G-9 in the bundle chosen under RG-4.\n - G-10 Short-ID uniqueness: Python test rebuilds DB against a 100x\n corpus and asserts `SELECT short_id, count(*) FROM short_ids\n GROUP BY 1 HAVING count(*) > 1` is empty.\n - G-11 No-computed-metric views: contract test enumerates every\n view in `main` via `information_schema.views` /\n `duckdb_columns`, parses each view's SQL with\n `sqlglot.parse_one(..., read='duckdb')`, and asserts that every\n projected column in the outermost SELECT is either a bare\n `Column` reference, a `Column` aliased via `AS`, or a\n whitelisted passthrough (`CAST`, `COALESCE(col, literal)`).\n Views may NOT introduce arithmetic, aggregation, CASE\n expressions, `IIF`, window functions, or function calls beyond\n the whitelist - any such projection means a user-visible metric\n is being produced by the view rather than persisted in a base\n table and fails the gate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add exit gates proving no metric-bearing JSON runtime reads remain and all explorer surfaces agree on DuckDB values", "wid": "w6"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w6"], "notes": "Remove benchmark/detail/compare/meta-leaderboard JSON outputs once the\nDuckDB-backed contract passes parity tests. If `manifest.json` or any\nroute helper remains, strip it to non-metric navigation or identity\ndata only and document that rule explicitly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Delete metric-bearing JSON artifacts and stale compatibility framing once DuckDB-backed parity is proven", "wid": "w7"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w2"], "notes": "Surfaced during W2 contract tests: the same platform can legitimately\nappear in a cohort more than once (different `tuning_mode`, different\n`trust_label`, different run). The originating schema pinned\n`cohort_metadata` PRIMARY KEY to `(cohort_key, platform_id)` and the\nW2 builder silently deduplicated by keeping only the best-ranked row,\ndropping the other publishable variant from the DuckDB store.\n\nThat violates the project principle: ALL results we trust enough to\npublish must be available in the DuckDB published-results dataset so\nusers can compare across tuning levels AND trust levels from the same\nstore. The table is the per-cohort published-result set; the UI\ndecides any display-time grouping.\n\nEmpirical evidence recorded with this TODO (`/tmp/reproduce_cohort_duplicates.py`-style\ntwo-bundle reproducer, same platform `duckdb`, tuning_mode\n`tuned` vs `notuning`, both in `tpch-sf0.1-power`):\n - `meta_leaderboard.json` `cohorts[0].platforms` kept BOTH variants\n (tuned rank 1, notuning rank 2) - JSON artifact is correct.\n - DuckDB `cohort_metadata` after W2 dedup kept only the tuned row -\n silently discards the notuning variant.\n - `platform_agg` at `pipeline.py:240` overwrites the better rank\n (rank 1) with the worse rank (rank 2) via last-write-wins, so the\n JSON artifact `platforms[]` entry AND the DuckDB `meta_leaderboard`\n row BOTH report `avg_rank = 2.0` when the correct answer is 1.0.\n This is a pre-existing correctness bug in the meta-leaderboard\n that already affects today's Home surface, independent of the PK\n decision.\n\nImplementation (Option B + bug fix):\n\nSchema (`docs/development/browser-duckdb-schema.sql`):\n - `cohort_metadata` PRIMARY KEY changes from\n `(cohort_key, platform_id)` to `(cohort_key, result_id)`.\n - Add columns: `tuning_mode VARCHAR`, `trust_label VARCHAR NOT NULL`,\n `short_id VARCHAR NOT NULL`. These give Home the data it needs to\n label, group, or collapse variants without a secondary join.\n - Keep `idx_cohort_metadata_key (cohort_key)`; add\n `idx_cohort_metadata_platform (cohort_key, platform_id)` so\n platform-grouped queries stay cheap.\n\nPython (`benchbox/core/explorer_pipeline/duckdb_builder.py`):\n - `_create_schema` mirrors the new DDL (PK + 3 columns + index).\n - `_populate_cohort_metadata` deletes the `best` dedup dict and\n `_RANK_IDX` logic; inserts every row from\n `meta[\"cohorts\"][*][\"platforms\"]`. Widens the insert tuple to\n include `tuning_mode`, `trust_label`, `short_id`.\n\nPython (`benchbox/core/explorer_pipeline/pipeline.py`):\n - `_build_meta_leaderboard` emits `tuning_mode`, `trust_label`,\n `short_id` on each entry in `cohort_platforms` so the builder has\n what it needs without re-joining. Thread `full_to_short` into\n the function, or resolve in the builder by looking up\n `short_id_map`.\n - Fix `platform_agg` last-write-wins at `pipeline.py:240`: when the\n same `platform_id` ranks in the same `cohort_key` more than once,\n KEEP THE BEST RANK (lowest integer). Correct for the Home\n meta-leaderboard semantics (one row per platform across cohorts\n - best variant represents the platform). Equivalent minimal\n edit: replace the blind dict assignment with a guarded update,\n or rely on `setdefault` since `sorted_rows` is already best-first.\n\nRegistry (`_project/planning/visible_metrics.yaml`):\n - Update `cohort_metadata` `rank` / `metric_value` / `speedup_vs_best`\n notes to clarify they are PER RESULT_ID, not per platform_id.\n - Add new rows: `cohort_metadata.tuning_mode`,\n `cohort_metadata.trust_label`, `cohort_metadata.short_id` with\n `kind: raw_copy` and appropriate `canonical_ref` pointers.\n\nTests:\n - Extend `test_distinct_tuning_modes_produce_distinct_rows` in\n `tests/unit/core/explorer_pipeline/test_benchmark_summary.py` with\n a DuckDB assertion: after the pipeline run, the cohort contains\n BOTH `result_id`s in `cohort_metadata` with distinct ranks.\n - Add a regression test in\n `tests/unit/core/explorer_pipeline/test_duckdb_browser_contract.py`\n (or a peer file) against the two-bundle same-platform-same-cohort\n fixture: assert `meta_leaderboard.avg_rank` equals the BEST rank\n (1.0, not 2.0), and `n_cohorts` equals the number of distinct\n cohort_keys (not the number of cohort-variants).\n - Add a `visible_metrics` drift test asserting the\n `cohort_metadata` PRIMARY KEY columns declared in the schema\n YAML match the frozen `browser-duckdb-schema.sql`.\n\nExit:\n - `cohort_metadata` PK is `(cohort_key, result_id)` in schema,\n builder, and visible_metrics.\n - Running the pipeline on a two-variant corpus lands BOTH rows in\n `cohort_metadata`; no silent deduplication anywhere in the\n pipeline.\n - `_build_meta_leaderboard` keeps the BEST rank per\n `(platform_id, cohort_key)`; regression test pinning\n `avg_rank == 1.0` on the two-variant reproducer is green.\n - All 217 pre-existing explorer pipeline tests remain green; the\n two new tests are green.\n - W3 does not start until this item is done, because the\n `duckdbQueries.getCohort` helper it builds must match the final\n key shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Preserve all publishable result variants in `cohort_metadata` and fix the `_build_meta_leaderboard` `platform_agg` last-write-wins bug before the frontend migration", "wid": "w8"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "split by cohort.", "dont": "mix cohorts in trend charts", "why": "benchmark/scale geomeans are not one metric"}, {"INSTEAD": "group charts by question.", "dont": "present every chart as an equal chip", "why": "users need analytical hierarchy"}, {"INSTEAD": "keep visible values and accessible names.", "dont": "rely on color alone in heatmaps", "why": "color perception and print/screenshot contexts vary"}, {"INSTEAD": "use log scale or speedup encoding with clear labels.", "dont": "use linear latency bars when one slow outlier collapses the rest of the field", "why": "users cannot compare fast platforms"}], "approach": "Start by making chart intent explicit in registry metadata, then update\nChartPanel navigation. Treat log scale and speedup as distinct encodings:\nlog scale for magnitude across orders of magnitude, speedup when the user is\nasking relative winner/loser questions. Test chart math separately from Preact\nrendering so scale regressions are easy to catch.\n", "blocked_reason": null, "category": "Data Visualization", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": ["explorer-critical-results-ux-defects"], "description": "Fix visualization choices that currently reduce trust or legibility in the\nResults Explorer. Database benchmark latency often spans orders of magnitude,\nso linear bar charts can compress meaningful differences and cause labels to\noverlap. Chart navigation also exposes too many tiny chart chips without an\nanalytical hierarchy. The goal is to make charts answer clear questions:\noverview, per-query behavior, distribution, cost, trend, and rank.", "id": "explorer-chart-correctness-and-navigation", "preserves": ["Benchmark ranking order continues to come from pipeline-emitted primary_metric/primary_order, not frontend guesses.", "Chart registry remains the single source for chart availability and chart metadata.", "Charts remain readable when only one platform/result is present.", "Heatmaps remain keyboard navigable and retain existing role/grid accessibility behavior."], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/index.css"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartMath.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartRegistry.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryLabels.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}], "state": "done", "title": "Explorer: chart correctness, log-scale latency encoding, and chart navigation", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Frontend typecheck and chart tests pass.", "expected": "All chart registry, chart math, heatmap, and ChartPanel tests pass.", "item_id": "explorer-chart-correctness-and-navigation", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run build", "description": "Production build succeeds.", "expected": "Vite build completes without chart-related warnings or errors.", "item_id": "explorer-chart-correctness-and-navigation", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": [], "notes": "In `results-explorer/src/lib/chartRegistry.ts` and ChartPanel, replace the\nflat 10+ chip list with grouped tabs or segmented controls:\n - Overview\n - Per-query\n - Distribution\n - Cost\n - Trend\n - Rank\nEach group should show at most the relevant chart controls for the current\npage/data shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit chart registry and group charts by analytical question", "wid": "w1"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w1"], "notes": "Update PowerBar/ChartPanel latency views so linear bars do not compress\nfaster engines against a slow outlier. Use log scale for latency magnitude\nor speedup-vs-best where comparison is the point. Axis labels must disclose\nlog scale when used.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Use log-scale or ratio-based encoding for latency bars where values span orders of magnitude", "wid": "w2"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w2"], "notes": "The audited SSB Performance Bar showed labels crowding near the origin.\nImplement collision-aware label placement: outside labels when space is\ninsufficient, minimum label gutter, and tooltip fallback for narrow bars.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix label overlap in SSB/star_schema performance bar charts", "wid": "w3"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w1"], "notes": "Carry forward the critical defect fix: trend charts must not mix\nbenchmark/scale/phase cohorts. Render cohort-specific small multiples or\nrequire a selected cohort before showing the time series.\n\nAlready satisfied by `explorer-critical-results-ux-defects` w3/w6 on the\ncurrent base: `buildTrendCohorts` groups by benchmark, scale factor,\nphase, and primary metric, and `PlatformIndex.test.tsx` asserts TPC-H and\nSSB trend points do not cross cohort sections.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Split platform trend charts by cohort or metric", "wid": "w4"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": [], "notes": "Query IDs should sort Q1, Q2, ..., Q10 rather than lexicographic 1, 10,\n11, ..., 2. Labels should preserve benchmark-specific prefixes where\npresent (`Q1`, `q01`, `query_001`) rather than bare ambiguous numerals.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add natural query sorting and explicit query labels in heatmaps and tables", "wid": "w5"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w5"], "notes": "MetaLeaderboard and QueryHeatmap should have text/value labels sufficient\nfor interpretation without relying only on color. Add or preserve\ncolor-blind-safe sequential palettes and reduced-color rendering where\nsupported by CSS/media queries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add high-contrast and reduced-color paths for heatmaps", "wid": "w6"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": "Extend chartMath and component tests for log-scale mapping, label\ncollision behavior, cohort-split trend data, natural query sort, and\nheatmap accessible names/values.\n\nExisting W2-W6 commits cover log-scale chart math, ChartPanel log-scale\nrendering, collision-aware value label placement, cohort-split trend\nrendering, natural query sort, and MetaLeaderboard heatmap values. W7\ncloses the remaining exact accessible-name assertion for QueryHeatmap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add chart math and rendering tests for scale, sorting, and labels", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change computeECDFPoints for w2", "why": "the math is correct; only the SVG path rendering changes"}, {"INSTEAD": "(not specified)", "dont": "gate Histogram/Box/CDF tabs on platform count alone", "why": "gate on actual non-null timing values"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-chart-coverage-review-followups", "id": 314, "reason": "Interactivity is out of scope for this follow-up item", "resolution": "promoted", "resolved_item": "interactive-cdfchart-tooltip-showing-exact-percentile-at-cur", "resolved_reason": null, "summary": "Interactive CDFChart tooltip showing exact percentile at cursor position"}], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-close-chart-type-coverage-gaps. The two Required issues (PowerBar\ncolor index, PercentileLadder caller color index) were fixed immediately\npost-review. This item tracks the remaining four findings.", "id": "explorer-chart-coverage-review-followups", "preserves": ["BenchmarkIndex existing tab gating logic for percentiles/power/phases/cost tabs", "CDFChart parity test (computeECDFPoints) must still pass - only the SVG path rendering changes", "RankTable ordinal display for n=1,2,3 must still return 1st/2nd/3rd"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from chart-coverage code review", "verifications": [{"command": "cd results-explorer && npm test -- RankTable", "description": "ordinal handles 21st/22nd/23rd correctly", "expected": "tests pass including ordinal edge cases", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm test -- NewCharts", "description": "CDFChart unit test still passes after step-render change", "expected": "CDFChart svg test passes", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm test -- BenchmarkIndex", "description": "BenchmarkIndex test passes with updated tab gating", "expected": "all BenchmarkIndex tests pass", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck && npm test", "description": "Full test suite green", "expected": "all green", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "RankTable.tsx:21-25 - ordinal() only special-cases 1/2/3; everything\nelse falls through to `${n}th`. Renders \"21th\", \"22nd\"\u2192\"22th\" for\nn >= 21 that end in 1/2/3 (except 11/12/13 which are correctly \"th\").\nUnlikely with current platform counts but technically wrong.\n\nFix: standard ordinal suffix rule -\n const suffixes = [\"th\",\"st\",\"nd\",\"rd\"];\n const v = n % 100;\n return `${n}${suffixes[(v - 20) % 10] ?? suffixes[v] ?? suffixes[0]}`;\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix ordinal() in RankTable to handle n >= 21 correctly", "wid": "w1"}, {"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "CDFChart.tsx:103-115 uses `L` (lineto) to connect CDF points, producing\nsmooth diagonal lines. A true ECDF is a step function: horizontal\nsegment to the x-value, then vertical jump to the next cumulative\nfraction. The Python reference (textcharts.cdf_chart) renders steps.\nVisual divergence is minor on dense data (22+ queries) but visible on\nsparse distributions.\n\nFix: render H\u2192V steps instead of diagonal L commands. No math change\nneeded - computeECDFPoints output is already correct.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Render CDFChart as ECDF steps, not smooth lines", "wid": "w2"}, {"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "BenchmarkIndex.tsx:383-386 - The Histogram, Box Plot, and CDF tabs are\nunconditionally shown (hardcoded true). When all platform timings are\nnull or empty, clicking these tabs shows a blank section because each\nchart guards internally and returns null. The Percentiles, Power, Phases,\nand Cost tabs are already data-gated.\n\nFix: add a `hasTimings` guard:\n const hasTimings = s.platforms.some(\n (p) => Object.values(p.timings).some((v) => v !== null && v > 0)\n );\nThen use hasTimings for the histogram/distribution/cdf entries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Data-gate Histogram / Box Plot / CDF tabs in BenchmarkIndex", "wid": "w3"}, {"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "chartRegistry.meta.test.ts:19-36 - CANONICAL_IDS is a third hardcoded\ncopy of the chart type list (alongside chart_types.py and chartRegistry.ts).\nThe test detects TS-side drift but NOT Python-side drift: if someone adds\na 17th type to chart_types.py without updating the test's list, the test\npasses.\n\nFix options:\n (a) Parse chart_types.py at test time using Node fs.readFileSync +\n regex to extract chart IDs from _CHART_SPECS.\n (b) Emit a machine-readable chart-ids.json from the Python pipeline\n as a build artifact and compare against that.\n (c) Accept the current triple-write but add a comment warning that\n chart_types.py is the authority and all three must be kept in sync.\n\nOption (c) is the lowest friction; (a) is the most robust. Avoid (b)\nsince it couples the static build to the test suite.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace hardcoded CANONICAL_IDS in meta-test with dynamic chart_types.py parse", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "make this ADR conditional on Phase 8/9 migration work", "why": "the explorer surface should be decided independently of the remaining migration phases (which TODO `migration-phase-api-surface-gate` handles separately)."}, {"INSTEAD": "(not specified)", "dont": "relocate any code in this TODO", "why": "moving code without the locked decision creates churn and re-litigates the choice in the migration PR. Defer all moves to `explorer-cli-surface-migration`."}], "approach": "Read-mostly. Two grep-based inventories (w1, w2) inform an ADR (w3).\nUse `docs/development/adr/TEMPLATE-results-data-extraction.md` as the\nstructural template if a TEMPLATE-adr.md is not present.\n\nDo not pre-commit to one of the four options before the inventories are\ncomplete; the migration-cost numbers are load-bearing.\n", "blocked_reason": null, "category": "Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-adr", "id": 155, "reason": "Owned by TODO `explorer-cli-surface-migration` (depends on this TODO).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Verified: benchbox/cli/commands has no explorer.py, and _project/scripts/explorer_publish.py + explorer_pipeline/ exist -- the ADR-recommended code move already happened.", "summary": "Actually move any code per the ADR's recommendation."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-adr", "id": 156, "reason": "Owned by TODO `migration-phase-api-surface-gate`; this ADR is scoped to the explorer surface only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Owning item migration-phase-api-surface-gate is already DONE; the broader CLI user/maintainer-split revisit is resolved or moot there.", "summary": "Revisit the user-vs-maintainer split for other parts of the `benchbox` CLI."}], "deps": [], "description": "`benchbox explorer build` (and the hidden `benchbox explorer build-contract`)\npublish a static read-model for the `results-explorer` React app. They are\nnot benchmark-runner operations, yet they sit on the user-facing `benchbox`\nCLI surface and ship under `benchbox/cli/commands/explorer.py` alongside a\n7-file core package at `benchbox/core/explorer_pipeline/`.\n\nProvenance audit (this session):\n - Both commands were added in commit `f9d08d38b` (\"Migration: populate\n develop branch from private working tree (Phase 4 w4)\", 2026-04-27).\n That was a bulk migration squash, not a discrete reviewed PR.\n - The breadcrumb at `.github/workflows/docs.yml:105` attributes them to\n PR #46, but #46 is `docs(contributing): name the version JSON key +\n non-uv fallback` \u2014 a contributing-results.md docs PR, unrelated to the\n CLI surface. The in-tree provenance trail lies.\n - No `_project/specs/` document scopes the explorer CLI surface; Phase-4\n migration TODO only lists the file paths as \"to migrate\", not as a\n command surface to design.\n\nThis TODO is the decision step. It does not move any code. It produces an\nADR that locks the destination so downstream migration (TODO\n`explorer-cli-surface-migration`) has a concrete target.\n\nThe four candidate destinations are:\n\n 1. **hide-only**: `hidden=True` on `explorer_group`. Cheapest. Does not\n solve intermingling \u2014 code stays in `benchbox/`, import surface stays\n public, only `--help` discoverability changes. Likely rejected.\n 2. **new sibling package** `benchbox_explorer/` with its own\n `console_script` (e.g. `benchbox-explorer-publish`). Separate pip\n artifact; clean conceptual boundary; external CI/scripts can still\n script against a stable binary.\n 3. **`_project/scripts/explorer/`** invoked via `uv run -- python -m\n _project.scripts.explorer.publish`. In-tree maintainer tool; not\n installed; tightest \"this is not a public surface\" signal.\n 4. **co-locate in `results-explorer/`** as a Python sub-package called\n from `results-explorer/scripts/`. Tightest coupling to the only\n consumer. Mixes Python and Node trees.\n\nThe ADR must also resolve three gating sub-questions:\n\n a. Does `benchbox/core/explorer_pipeline/` ALSO relocate, or only the\n CLI wrapper? Argument for \"yes\": same intermingling concern; the\n package serves only the explorer site. Argument for \"no\": existing\n tests under `tests/unit/core/explorer_pipeline/` and import sites\n like `benchbox/cli/help.py` would need follow-up. Decision here\n shapes the migration cost.\n\n b. Backwards-compat alias for one release: keep\n `benchbox explorer build` as a thin deprecation shim that prints a\n `DeprecationWarning` and forwards to the new entry point, then drop\n it in the next minor release? Or hard-cut and update all known\n callers in the same PR? BenchBox is v0.2.x pre-1.0, so a hard cut\n is defensible; the ADR should still record the call.\n\n c. End-user error path when a stale snapshot is detected: today,\n `results-explorer/src/db.ts:196` tells the user to run\n `benchbox explorer build`. After the move, the message must point\n at the maintainer-only entry. For non-maintainers (e.g., someone\n running `npm run dev` against a downloaded snapshot), the message\n should redirect to \"refresh the snapshot from the published site\"\n or similar, not to a CLI they shouldn't be running.", "id": "explorer-cli-surface-adr", "preserves": ["Existing CI builds, browser-fixture generation, and UAT explorer-smoke must continue to work unchanged; this TODO does not touch them.", "No code is moved or renamed in this TODO. ADR-only.", "`benchbox --help` output is unchanged in this TODO."], "prior_art": [{"concept": "_project/scripts/ \u2014 existing in-tree maintainer scripts directory (validate_todo.py, generate_indexes.py, etc.); candidate destination for option 3, reuse layout if chosen.", "decision": "reuse", "path": "_project/scripts/"}, {"concept": "benchbox/cli/commands/__init__.py:22 \u2014 registration site for `explorer_group`; reuse the registration pattern (or its absence) for whatever replaces it.", "decision": "reuse", "path": "benchbox/cli/commands/__init__.py:22"}, {"concept": "benchbox/cli/commands/explorer.py \u2014 current Click subgroup; supersede or relocate per ADR.", "decision": "supersede", "path": "benchbox/cli/commands/explorer.py"}, {"concept": "benchbox/core/explorer_pipeline/ \u2014 7-file core package (contract.py, duckdb_builder.py, models.py, pipeline.py, ranking.py, transformer.py, compare_math.py); reuse intact, only the housing changes.", "decision": "reuse", "path": "benchbox/core/explorer_pipeline/"}, {"concept": "docs/development/adr/ \u2014 existing ADR location with TEMPLATE-results-data-extraction.md as the structural reference; reuse format.", "decision": "reuse", "path": "docs/development/adr/"}, {"concept": "results-explorer/scripts/explorer-build-contract.mjs \u2014 Node-side consumer that calls `benchbox explorer build-contract` to fetch the output contract; reference for how an external script binds to the entry point.", "decision": "reuse", "path": "results-explorer/scripts/explorer-build-contract.mjs"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "docs/development/adr/adr-explorer-cli-surface.md"}], "state": "done", "title": "ADR: pick a destination for the explorer publish toolchain", "verifications": [{"command": "test -f docs/development/adr/adr-explorer-cli-surface.md && wc -l docs/development/adr/adr-explorer-cli-surface.md", "description": "ADR file exists and validates as markdown.", "expected": "non-zero line count", "item_id": "explorer-cli-surface-adr", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE '^## (Decision|Recommendation|Sub-question)' docs/development/adr/adr-explorer-cli-surface.md", "description": "ADR records a single recommended option and answers all three sub-questions (a, b, c).", "expected": ">= 4", "item_id": "explorer-cli-surface-adr", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -cE 'docs.yml|generate-browser-fixtures|explorer_pipeline' docs/development/adr/adr-explorer-cli-surface.md", "description": "Inventories from w1 and w2 are present in the ADR body.", "expected": ">= 3", "item_id": "explorer-cli-surface-adr", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": [], "notes": "Before writing the ADR proper, re-run the provenance probes that this\nTODO's description cites and commit the resulting facts as a\n\"Provenance Facts\" section in the ADR. This durability-pins the\nevidence so future readers do not have to re-derive it.\n\nCommands:\n - `git show --stat f9d08d38b -- benchbox/cli/commands/explorer.py`\n \u2192 expect ~221-line file addition under that path.\n - `git log -1 --format=%s%n%n%b f9d08d38b`\n \u2192 expect title \"Migration: populate develop branch from private\n working tree (Phase 4 w4)\".\n - `gh pr view 46 --json title,body | head -2`\n \u2192 expect title to be `docs(contributing): ...` (NOT the explorer\n CLI), confirming the breadcrumb at `.github/workflows/docs.yml:105`\n is mis-attributed.\n - `git log --all --diff-filter=A --oneline -- benchbox/cli/commands/explorer.py`\n \u2192 expect only `f9d08d38b` and `d28dbe66f` (the consistency-fix PR).\n - `git blame .github/workflows/docs.yml -L 100,108`\n \u2192 expect `0182c9556c` (v0.2.1 release commit), confirming the\n misattributed Note line.\n\nOutput: a 5-10 line \"Provenance Facts\" block at the top of the ADR\nrecording: introduction commit, PR (none / migration squash),\nbreadcrumb misattribution target, and the command set above so a\nfuture reader can re-run verbatim.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate provenance evidence and pin it in the ADR", "wid": "w0"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w0"], "notes": "Grep targets (run from repo root):\n - `git grep -n \"benchbox explorer build\\|benchbox explorer build-contract\"`\n - `git grep -nE \"explorer_group|from benchbox\\.cli\\.commands\\.explorer\"`\n - `git grep -nE \"from benchbox\\.core\\.explorer_pipeline\"`\n\nKnown seed list to confirm and extend:\n - `.github/workflows/docs.yml:103`\n - `results-explorer/scripts/generate-browser-fixtures.mjs:489`\n - `results-explorer/scripts/explorer-build-contract.mjs:17`\n - `tests/uat/.../explorer_smoke.py` (UAT W7 phase)\n - `Makefile` `uat-explorer-smoke` target (line ~1612)\n - `docs/development/browser-test-architecture.md:35,119,130`\n - `docs/operations/results-phase-2-runbook.md:109`\n - `results-explorer/src/db.ts:196` (frontend error remediation text)\n - audit docs under `_project/audits/results-explorer-*.md`\n - tests under `tests/unit/cli/test_explorer_build_contract.py`\n\nOutput: a single table in the ADR with each call site, its category\n(CI / fixture / UAT / docs / runtime / test), and migration cost\n(one-liner / multi-line / non-trivial).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory every call site of the current CLI surface", "wid": "w1"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w1"], "notes": "The decision on sub-question (a) (does the pipeline also move?) turns\non how leaky the package's public import surface is.\n\nGrep targets:\n - `git grep -nE \"from benchbox\\.core\\.explorer_pipeline|import benchbox\\.core\\.explorer_pipeline\"` excluding the package itself.\n\nOutput: list of every import site outside `benchbox/core/explorer_pipeline/`,\ngrouped by importer (CLI / test / other).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory imports of `benchbox.core.explorer_pipeline.*` from outside the package", "wid": "w2"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w1", "w2"], "notes": "Path: `docs/development/adr/adr-explorer-cli-surface.md`.\n\nStructure:\n - Context (the provenance audit summary, citing this TODO).\n - Forces: user-vs-maintainer surface separation; CI dependency stability;\n external script consumers; migration cost; reversibility; consistency\n with other maintainer tools in `_project/scripts/`.\n - Options 1-4 with explicit pros/cons and migration-cost estimate from\n the w1/w2 inventories.\n - Decision matrix scoring each option on the forces.\n - Recommendation with rationale.\n - Resolutions to sub-questions (a), (b), (c) from the description.\n - Consequences: what TODO `explorer-cli-surface-migration` will do.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Draft the ADR with a 4-option decision matrix", "wid": "w3"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w3"], "notes": "Open a PR with only the ADR (no code moves). On merge, this TODO is\nCompleted and `explorer-cli-surface-migration` is unblocked.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Land the ADR on develop", "wid": "w4"}], "worktree": "explorer-cli-surface-adr"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "change the snapshot's output contract or column schema", "why": "that work is owned by TODO `explorer-read-model-schema-versioning`."}, {"INSTEAD": "(not specified)", "dont": "fix the stale-snapshot dev-DX wall in this PR", "why": "that's TODO `explorer-dev-snapshot-dx`. Mixing them makes review harder and entangles the rename with behavioral change."}, {"INSTEAD": "(not specified)", "dont": "keep `benchbox explorer` as a permanent alias unless the ADR explicitly chose option (b) with a deprecation shim. A bare `pass`-through alias defeats the whole point.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "update historical audit docs that record commands actually run at the time. Only the active runbook and developer docs need updates; history can read 'we ran benchbox explorer build on 2026-05-10'.", "why": "(unstated)"}], "approach": "Strictly a relocation + caller-update PR. Read the ADR's \"Consequences\"\nlist and execute it line-by-line. Do not introduce new behavior, new\nflags, or new pipeline logic. Schema-versioning and dev-DX are explicit\nout-of-scope items handled by sibling TODOs.\n\nSequencing: relocate (w2, w3) before updating callers (w4) so the\ncallers point at the new live location, not a future path. Add the\npinning test (w5) last so it asserts the steady state.\n", "blocked_reason": null, "category": "Refactoring", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-migration", "id": 157, "reason": "Owned by TODO `explorer-dev-snapshot-dx`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Owning TODO explorer-dev-snapshot-dx is completed (_project/DONE/main/active/explorer-dev-snapshot-dx.yaml).", "summary": "Pre-dev rebuild hook or in-tree dev snapshot."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-migration", "id": 158, "reason": "Owned by TODO `explorer-read-model-schema-versioning`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: explorer-read-model-schema-versioning.yaml is already in _project/DONE \u2014 this work is complete.", "summary": "Schema versioning for the read-model contract."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-migration", "id": 159, "reason": "Owned by TODO `migration-phase-api-surface-gate`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Owning TODO migration-phase-api-surface-gate is Completed; the Phase 8/9 API-surface gate already executed.", "summary": "Phase 8/9 migration API-surface gate."}], "deps": ["explorer-cli-surface-adr"], "description": "Execute the decision locked by TODO `explorer-cli-surface-adr`:\n - Remove `explorer` from the user-facing `benchbox` CLI surface.\n - Relocate `benchbox/cli/commands/explorer.py` (and, per ADR sub-question\n a, optionally `benchbox/core/explorer_pipeline/`) to the destination\n the ADR chose.\n - Update every call site: CI workflow, browser-fixture generator, UAT\n explorer-smoke phase, runbook, developer docs, frontend remediation\n string in `results-explorer/src/db.ts:196`, and audit cross-refs.\n - Delete the incorrect provenance breadcrumb at\n `.github/workflows/docs.yml:105` that misattributes the CLI surface\n to PR #46.\n - Add a regression test that pins the frontend remediation text to the\n live invocation of whatever entry point the ADR chose, so renames\n do not silently rot the user-facing error message.\n\nThis TODO does NOT redesign the read-model contract (TODO\n`explorer-read-model-schema-versioning`) or address the stale-snapshot\ndev-DX wall (TODO `explorer-dev-snapshot-dx`). Those are separable.", "id": "explorer-cli-surface-migration", "preserves": ["Browser-fixture generator continues to produce byte-identical fixture sets (verified by `verify-browser-fixtures.mjs`).", "CI explorer-build step in `.github/workflows/docs.yml` continues to produce a valid `results-explorer/public/data/results.duckdb` (only the invocation changes).", "Existing tests under `tests/unit/core/explorer_pipeline/` continue to pass (relocated if needed but not rewritten).", "Output contract (schema of `results.duckdb` and copied bundles) is unchanged by this TODO.", "UAT `make uat-explorer-smoke` continues to pass with the new invocation."], "prior_art": [{"concept": ".github/workflows/docs.yml:103-105 \u2014 CI invocation + the wrong PR #46 attribution comment; update invocation, delete the comment.", "decision": "reuse", "path": ".github/workflows/docs.yml:103-105"}, {"concept": "benchbox/cli/commands/explorer.py \u2014 current location; remove and relocate per ADR.", "decision": "reuse", "path": "benchbox/cli/commands/explorer.py"}, {"concept": "benchbox/core/explorer_pipeline/ \u2014 pipeline package; relocation gated by ADR sub-question (a).", "decision": "reuse", "path": "benchbox/core/explorer_pipeline/"}, {"concept": "docs/development/browser-test-architecture.md:35,119,130 \u2014 developer docs referencing the old command.", "decision": "reuse", "path": "docs/development/browser-test-architecture.md:35,119,130"}, {"concept": "docs/operations/results-phase-2-runbook.md:109 \u2014 runbook references the old command.", "decision": "reuse", "path": "docs/operations/results-phase-2-runbook.md:109"}, {"concept": "results-explorer/scripts/explorer-build-contract.mjs:17 \u2014 EXPECTED_COMMAND constant; update to the new invocation.", "decision": "reuse", "path": "results-explorer/scripts/explorer-build-contract.mjs:17"}, {"concept": "results-explorer/scripts/generate-browser-fixtures.mjs:489 \u2014 Node-side caller; update spawn invocation.", "decision": "reuse", "path": "results-explorer/scripts/generate-browser-fixtures.mjs:489"}, {"concept": "results-explorer/src/db.ts:196 \u2014 frontend remediation string; update to the new invocation and pin to it via test.", "decision": "reuse", "path": "results-explorer/src/db.ts:196"}, {"concept": "tests/uat/.../explorer_smoke.py + Makefile target `uat-explorer-smoke` (line ~1612) \u2014 UAT phase invocation.", "decision": "reuse", "path": "tests/uat/.../explorer_smoke.py"}, {"concept": "tests/unit/cli/test_explorer_build_contract.py \u2014 existing contract tests; if the CLI moves out of benchbox/cli/, this test file moves with it or is replaced.", "decision": "reuse", "path": "tests/unit/cli/test_explorer_build_contract.py"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/docs.yml"}, {"kind": "only_modify", "path_glob": ""}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/explorer.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/**"}, {"kind": "only_modify", "path_glob": "docs/development/browser-test-architecture.md"}, {"kind": "only_modify", "path_glob": "docs/operations/results-phase-2-runbook.md"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/explorer-build-contract.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/generate-browser-fixtures.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/**"}, {"kind": "only_modify", "path_glob": "tests/uat/**"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_explorer_build_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/**"}], "state": "done", "title": "Execute the explorer-CLI-surface ADR and remove explorer from benchbox --help", "verifications": [{"command": "uv run benchbox --help | grep -c '^\\s*explorer'", "description": "`explorer` is no longer in the public `benchbox` CLI surface.", "expected": "0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 1}, {"command": "make test-fast", "description": "Fast tests pass.", "expected": "exit 0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 2}, {"command": "make lint && make typecheck", "description": "Lint + typecheck pass.", "expected": "exit 0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 3}, {"command": " --data-dir results-data --output /tmp/explorer-migration-smoke && test -f /tmp/explorer-migration-smoke/results.duckdb", "description": "Explorer-build pipeline still produces the snapshot via the new invocation.", "expected": "exit 0, file exists", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- --run db-remediation-pin", "description": "Frontend Vitest pin test passes.", "expected": "test exits 0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -c 'added in PR #46' .github/workflows/docs.yml", "description": "The wrong PR-#46 attribution is deleted from docs.yml.", "expected": "0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": [], "notes": "Read `docs/development/adr/adr-explorer-cli-surface.md` end-to-end.\nIf the inventories of call sites and pipeline imports already exist\nthere, reuse them; do not redo. Build the migration checklist from\nthe ADR's \"Consequences\" section.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm the ADR is merged and read its inventories", "wid": "w1"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w1"], "notes": "Move `benchbox/cli/commands/explorer.py` to the ADR's chosen\ndestination. Update:\n - `benchbox/cli/commands/__init__.py:22,62,106` \u2014 remove the\n `from .explorer import explorer_group` import and `explorer_group`\n registration + `__all__` entry.\n - `benchbox/cli/help.py` \u2014 any references to explorer commands in\n the help text.\nRun `uv run benchbox --help` and confirm `explorer` no longer appears.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Relocate the CLI entry point per ADR", "wid": "w2"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w1"], "notes": "Gated by ADR sub-question (a). If \"yes\": move the package + its tests\nunder `tests/unit/core/explorer_pipeline/` to the ADR's destination.\nUpdate imports in the relocated CLI entry from w2.\n\nIf \"no\": leave the package in place. Note this in the migration PR\ndescription so future readers understand the half-move was deliberate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Relocate `benchbox/core/explorer_pipeline/` if the ADR requires it", "wid": "w3"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w2", "w3"], "notes": "Process each of the call sites from the ADR's w1 inventory. Specifically:\n - `.github/workflows/docs.yml:103` \u2014 replace invocation; DELETE the\n `# Note: requires 'benchbox explorer' CLI subcommand (added in PR #46)`\n line at :105 (the attribution is wrong; PR #46 is a docs PR).\n - `results-explorer/scripts/generate-browser-fixtures.mjs:489`\n - `results-explorer/scripts/explorer-build-contract.mjs:17`\n (EXPECTED_COMMAND constant)\n - `tests/uat/.../explorer_smoke.py` and `Makefile:uat-explorer-smoke`\n - `docs/development/browser-test-architecture.md`\n - `docs/operations/results-phase-2-runbook.md:109`\n - `results-explorer/src/db.ts:196` (remediation string)\n - audit docs that cite the old command \u2014 only the active runbook is\n load-bearing; historical audits can keep the old command since they\n record what was run at the time. Decide per-file in the PR review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update every call site identified in the ADR inventory", "wid": "w4"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w4"], "notes": "The current failure mode: `results-explorer/src/db.ts:196` embeds the\nstring `'benchbox explorer build'`. If the CLI is renamed, nothing\nforces this string to update. Two options:\n\n (i) Import the canonical invocation from a shared TS module sourced\n from the same contract file the Node scripts already use\n (`results-explorer/scripts/explorer-build-contract.mjs`), so\n there is one source of truth.\n (ii) Add a test that asserts the current `db.ts` remediation string\n matches the exported `EXPECTED_COMMAND` from the contract\n script.\n\nPrefer (i). Implement in the Vitest suite under\n`results-explorer/src/lib/__tests__/`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test pinning the frontend remediation string", "wid": "w5"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w5"], "notes": "Local verification:\n - `uv run benchbox --help | grep -c explorer` \u2192 0\n - `cd results-explorer && npm run lint && npm test`\n - `make lint && make typecheck`\n - `make test-fast`\n - Manual: run the relocated entry point against `results-data/` and\n confirm `results.duckdb` is generated with the expected schema.\n\nOpen PR via `make pr-open`. Title prefix `refactor(explorer):`. Body\nreferences the merged ADR and lists each call site touched.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the full verification matrix and open the PR", "wid": "w6"}], "worktree": "explorer-cli-surface-migration"} -{"anti_patterns": [{"INSTEAD": "extend the registry first if a new type is genuinely warranted", "dont": "add a new chart in the explorer without a corresponding entry in chart_types.py", "why": "that recreates the divergence we're trying to eliminate"}, {"INSTEAD": "read the CLI's exact percentile method and either match it in TS or pre-compute in Python and ship in artifact", "dont": "compute percentiles in TS using `array.sort()[Math.floor(n*p)]`", "why": "Python's numpy.percentile uses linear interpolation by default and the numbers will differ"}, {"INSTEAD": "hide the section entirely with a small note about cost data availability", "dont": "render cost_scatter when cost_usd is null for all results", "why": "an empty scatter plot looks broken"}, {"INSTEAD": "add rank_table as an alternate view, let users switch", "dont": "replace the existing QueryHeatmap with rank_table on TPC-DS just because the heatmap is wide", "why": "users have built mental models around the heatmap"}, {"INSTEAD": "extend chartTheme.ts with a dedicated TIME_SERIES_PALETTE if the 4-color palette is insufficient", "dont": "reuse PALETTE colors for time_series lines without a contrast check", "why": "4 line colors must be distinguishable when overlapping"}, {"INSTEAD": "every chart gets a fixture", "dont": "skip parity fixtures for 'simple' charts like rank_table", "why": "rank ties are exactly the kind of edge case where Python and JS sort algorithms can differ silently"}], "approach": "Tier-by-tier (w1 \u2192 w4) so user-visible value lands in priority order.\nTier 1 (percentile_ladder, cdf_chart) ships first because that's the\nhighest-impact gap for benchmark credibility.\n\nPer-chart workflow (apply uniformly):\n 1. Read the CLI implementation (find the textcharts call site,\n identify input shape and reduction).\n 2. Decide where the math lives. Default: in\n `transformer.py`, emitted via the appropriate artifact\n (BenchmarkSummary for cross-platform; DetailResult for\n single-result). Avoid TS-side computation.\n 3. Generate parity fixtures.\n 4. Build the SVG component using `chartTheme.ts` constants and\n `useElementSize` for responsiveness.\n 5. Wire into the appropriate page (Compare, ResultDetail,\n BenchmarkIndex, PlatformIndex).\n 6. Vitest unit + parity test.\n 7. Manual smoke at 3 viewport widths.\n\nThis item is sequenced AFTER the foundational items\n(`explorer-establish-cli-explorer-parity-tests`,\n`explorer-emit-comparison-artifact-from-pipeline`,\n`explorer-unify-visualization-foundations`) because each new chart\nneeds the parity fixtures, theme constants, and pipeline-emitted\ndata structures those items deliver.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 160, "reason": "Out of scope for this item. Those should be added to chart_types.py registry first via a separate planning round.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'build entirely new chart types' with no specific types defined; reason says it needs its own planning round first.", "summary": "Build entirely NEW chart types that don't exist in CLI either"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 161, "reason": "Premature optimization - SVG handles realistic data sizes; revisit if profiling shows pain points.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly premature optimization pending profiling evidence of pain points; no such evidence cited or found.", "summary": "WebGL/Canvas rendering for high-density visualizations (e.g. histogram with thousands of points)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 162, "reason": "Product feature, not a coverage gap. Address after the chart catalog is complete.", "resolution": "promoted", "resolved_item": "user-saveable-chart-configurations-dashboards", "resolved_reason": null, "summary": "User-saveable chart configurations / dashboards"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 163, "reason": "Cross-cutting UX work; better tackled as a focused interactivity item once all charts exist.", "resolution": "promoted", "resolved_item": "interactive-drilldowns-click-chart-bar-to-filter-table", "resolved_reason": null, "summary": "Interactive drilldowns (click chart bar to filter table)"}], "deps": ["explorer-emit-comparison-artifact-from-pipeline", "explorer-establish-cli-explorer-parity-tests", "explorer-unify-visualization-foundations"], "description": "The 2026-04-15 review found that the CLI registers **16 canonical\nchart types** in `benchbox/core/visualization/chart_types.py:17-45`,\nbut the explorer implements only **6 visualization components**\n(QueryHeatmap, QueryTimingChart, GroupedQueryChart,\nNormalizedSpeedupChart, DivergingBarChart, MetaLeaderboard) -\nand only 3-4 of those have CLI counterparts that compute the same\nvalues. **10 CLI chart types have no explorer equivalent.**\n\nMissing chart types and their importance for benchmark credibility:\n\n | Chart | Why it matters |\n |----------------------|------------------------------------------------------------------|\n | `percentile_ladder` | Tail-latency P50/P90/P95/P99 - central to fair benchmark claims |\n | `cdf_chart` | Distribution shape - exposes \"fast median, slow tail\" patterns |\n | `rank_table` | Compact per-query ranking - denser than heatmap for many queries |\n | `distribution_box` | Box-plot of per-query latencies - shows variance/outliers |\n | `query_histogram` | Per-query latency distribution |\n | `time_series` | Trend over runs - regression detection |\n | `cost_scatter` | Cost vs performance - central to cloud-warehouse comparisons |\n | `power_bar` | Dedicated TPC Power@Size visual |\n | `sparkline_table` | Compact metrics summary with embedded micro-charts |\n | `stacked_phase` | Phase breakdown (load/warmup/power/throughput) |\n\nThis item ports each missing chart type to the explorer in priority\norder. Every new chart consumes pipeline-emitted data\n(per `explorer-emit-comparison-artifact-from-pipeline` - extending\nthe artifact emission as needed) and ships with a parity fixture\n(per `explorer-establish-cli-explorer-parity-tests`) so the math\nis verified against the canonical Python implementation from day one.\n\nOut of scope: building entirely NEW chart types that don't exist in\nthe CLI either. Those should be evaluated and added to both sides\nthrough the canonical chart_types.py registry first.", "id": "explorer-close-chart-type-coverage-gaps", "preserves": ["BenchmarkSummary and DetailResult artifact backward compatibility - extensions add new fields, do not rename or remove existing ones", "CLI chart_types.py registry as the single canonical source - explorer mirrors it, never the other way around", "Existing 6 explorer chart components must continue to work - new charts add tabs/sections, do not replace existing visualizations", "ResultDetail / BenchmarkIndex / PlatformIndex existing layouts and navigation patterns - additions are introduced as tabs or new sections, not page rewrites"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/**"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/**"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/manifest.ts"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/**"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/**"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartRegistry.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "tests/parity/fixtures/**"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/**"}], "state": "done", "title": "Close chart-type coverage gaps: percentile_ladder, cdf, rank_table, distribution_box, and 6 others", "verifications": [{"command": "ls results-explorer/src/components/{PercentileLadder,CDFChart,RankTable,DistributionBox,QueryHistogram,PowerBar,StackedPhase,TimeSeries,CostScatter,SparklineTable}.tsx", "description": "All 10 missing chart components exist as files", "expected": "all 10 files present", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 1}, {"command": "ls tests/parity/fixtures/{percentile_ladder,cdf,rank_table,distribution_box,query_histogram,power_bar,stacked_phase,time_series,cost_scatter,sparkline_table}.json", "description": "Every new chart has a corresponding parity fixture", "expected": "all 10 fixtures present", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm test -- chartMath.parity", "description": "Parity suite passes for all new charts", "expected": "all 10 chart fixtures pass equivalence assertions", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm test -- chartRegistry", "description": "Explorer chartRegistry mirrors CLI chart_types", "expected": "registry-parity test passes - 16 entries on both sides, no divergence", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- 'PercentileLadder|CDFChart|RankTable|DistributionBox|QueryHistogram|PowerBar|StackedPhase|TimeSeries|CostScatter|SparklineTable'", "description": "Each new chart has Vitest unit coverage", "expected": "tests cover empty data, single result, missing fields, and at least one realistic data shape per chart", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -v", "description": "Pipeline emits the new derived data fields", "expected": "tests cover percentile arrays, distribution stats, etc. as added to BenchmarkSummary / DetailResult", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 6}, {"command": "cd results-explorer && npm run typecheck && npm test && cd .. && uv run -- python -m pytest -m fast -q", "description": "TypeScript compiles, full test suite green", "expected": "all green", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 1 - highest-impact missing chart for tail-latency credibility.\nPlacement: ResultDetail.tsx side panel + Compare.tsx tab alongside\nSpeedup/Diverging/Grouped.\n\nPercentile method MUST match the Python reference exactly\n(numpy.percentile linear interpolation) - pre-compute in the\npipeline and ship in the artifact, do not recompute in TS.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `percentile_ladder`: emit p50/p90/p95/p99 arrays + ladder SVG component", "wid": "w1"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w5", "w6", "w7"], "notes": "Tier 4 - compact alternative to the heatmap for the home page or\nplatform overview. Each row carries a per-query trend mini-chart.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `sparkline_table`: compact metrics table with embedded micro-charts", "wid": "w10"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w10", "w8", "w9"], "notes": "Final integration step:\n - Create `results-explorer/src/lib/chartRegistry.ts` mirroring\n the canonical 16-entry list from\n benchbox/core/visualization/chart_types.py.\n - Each registry entry: `{ id, title, description, requires:\n DataRequirements, component: ComponentRef, cli_equivalent: str }`.\n - Meta-test fails if the explorer registry diverges from the\n CLI registry (new chart type added on one side without the other).\n - Update `docs/visualization-parity.md` (created by\n `explorer-establish-cli-explorer-parity-tests`) with the\n full chart-type matrix and the policy: \"New chart types are\n added to chart_types.py first; explorer follows.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `chartRegistry.ts` mirroring chart_types.py + meta-test enforcing registry parity", "wid": "w11"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 1 - distribution-shape companion to percentile_ladder.\nPlacement: Compare.tsx tab + per-result detail page.\nSame pre-compute discipline: CDF samples emitted by Python.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `cdf_chart`: emit CDF samples + SVG component", "wid": "w2"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 2 - per-query ranks (1..N) across platforms with optional\n\u2191\u2193 rank-change arrows. Useful for TPC-DS (99 queries) where the\nheatmap becomes unwieldy. Add as alternate view alongside the\nheatmap; the heatmap stays the default.\n\nRank ties are a known divergence risk (Python and JS sort\nalgorithms differ silently) - fixture MUST cover ties.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `rank_table`: emit per-query rank matrix + table component", "wid": "w3"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 2 - reveals platforms with high run-to-run variance even\nwhen median looks good. Consumes raw per-sample timings already\nin DetailResult.queries; reduction helper in transformer.py emits\nthe five-number summary.\nPlacement: alternate Compare view + ResultDetail\n\"Per-query distribution\" section.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `distribution_box`: emit min/Q1/median/Q3/max per query + box-plot component", "wid": "w4"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w1", "w2", "w3", "w4"], "notes": "Tier 3 - match the CLI auto-split behavior (when query count > 33\nthe CLI splits into multiple panels). Bin edges decided in Python\nand shipped in the artifact so explorer cannot diverge.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `query_histogram`: emit binned histogram + SVG component (auto-split when query_count > 33)", "wid": "w5"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w1", "w2", "w3", "w4"], "notes": "Tier 3 - currently buried in Compare summary cards as a small dl\nrow. Promote to a first-class chart on BenchmarkIndex when\npower_score is the primary metric for the benchmark family\n(consult lib/ranking.ts from explorer-unify-visualization-foundations).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `power_bar`: dedicated TPC Power@Size visualization (first-class on BenchmarkIndex)", "wid": "w6"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w1", "w2", "w3", "w4"], "notes": "Tier 3 - useful for TPC-H/TPC-DS where load and throughput phases\nmatter for comparison fairness. Phase totals already in\nBenchmarkSummary; new component renders them as stacked bars\nper platform.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `stacked_phase`: phase breakdown (load/warmup/power/throughput) stacked bars", "wid": "w7"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w5", "w6", "w7"], "notes": "Tier 4 - surfaces regressions across releases.\nPlacement: PlatformIndex.tsx (per-platform history) + new\nBenchmarkIndex \"Trends\" tab.\n\n4-color PALETTE may be insufficient for many overlapping lines -\nextend chartTheme.ts with a dedicated TIME_SERIES_PALETTE if a\ncontrast check fails.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `time_series`: trend chart of geomean_ms / power_score over run_date, per platform", "wid": "w8"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w5", "w6", "w7"], "notes": "Tier 4 - central for cloud DW comparisons\n(Snowflake/BigQuery/Athena/Databricks).\nPlacement: BenchmarkIndex when cost_usd is populated for >0\nresults. Hide section entirely when no cost data exists\n(an empty scatter looks broken).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `cost_scatter`: cost_usd vs power_score (or geomean_ms) scatter", "wid": "w9"}], "worktree": "main"} +{"anti_patterns": [{"dont": "default to the slowest result as baseline", "instead": "default to fastest", "why": "speedup is conventionally measured relative to the alternative, not the worst option"}, {"dont": "pull in a charting library (D3, Chart.js, Recharts)", "instead": "inline SVG is sufficient", "why": "bundle size and dependency overhead aren't justified for 2 chart types"}, {"dont": "use linear scale for the speedup chart", "instead": "log scale is required for legibility", "why": "OLAP deltas span orders of magnitude"}], "approach": "Both charts are inline SVG components (consistent with existing\nQueryTimingChart pattern). No external charting library needed for\nthese two chart types - the math is simple and SVG gives full control\nover log scale rendering.\n\nLog scale: manually compute SVG x-positions from log2(speedup).\nRender grid lines at 0.25x, 0.5x, 1x, 2x, 4x. Label them.\n\nDiverging bar: sort queries by abs(delta_pct) descending so the most\nimpactful queries appear at top. Include a summary line: \"N queries\nfaster, M queries slower, median delta: \u00b1X%\".\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-align-duration-metric", "explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found that the explorer compare view\nships only 2 inline SVG components (grouped bar chart + simple ratio\ntable) while the CLI offers 15 chart types. The most strategically\nimportant gap for the \"dynamic comparison tool\" claim is the absence\nof charts that make per-query performance deltas legible at a glance:\n\n 1. Normalized speedup chart (log scale, baseline-relative) - answers\n \"how much faster/slower is platform B vs platform A per query?\"\n The current grouped bar chart doesn't normalize, so a 1ms vs 2ms\n delta looks the same as a 100ms vs 200ms delta.\n\n 2. Diverging bar chart (per-query improvement/regression) - answers\n \"which queries got faster and which got slower?\" A positive/negative\n bar per query makes regressions immediately obvious. The CLI's\n `diverging_bar` chart type is the primary tool for this.\n\nThese two charts close the most visible CLI parity gap and directly\nsupport the compare view's core UX claim.\n\nA phase breakdown (stacked load vs query time) and percentile ladder\nare valuable but deferred - they require phase timing fields that are\nnot yet in the manifest.", "id": "explorer-add-comparison-charts", "preserves": ["Compare view URL format - chart selection is UI state, not URL state (no new query params needed)", "Existing grouped bar chart - it is the default for 3+ result comparisons and remains accessible via the chart selector", "Null-safety: if geomean_ms is missing, disable normalized speedup with a clear message"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (no pipeline changes needed)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/DivergingBarChart.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/NormalizedSpeedupChart.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}], "state": "done", "title": "Add normalized speedup and diverging bar charts to explorer compare view", "verifications": [{"command": "ls results-explorer/src/components/NormalizedSpeedupChart.tsx results-explorer/src/components/DivergingBarChart.tsx", "description": "Both new chart components exist", "expected": "both files present", "item_id": "explorer-add-comparison-charts", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'NormalizedSpeedupChart\\|DivergingBarChart\\|chart.*selector\\|chartType' results-explorer/src/pages/Compare.tsx", "description": "Chart selector renders in Compare", "expected": "both chart components referenced", "item_id": "explorer-add-comparison-charts", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-add-comparison-charts", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": [], "notes": "Log scale is important because OLAP query deltas span orders of\nmagnitude. A linear scale makes small queries illegible when one\nlarge query dominates.\n\nBar color: green (speedup > 1), red (speedup < 1), neutral (= 1).\nLog scale axis labels: 0.1x, 0.25x, 0.5x, 1x, 2x, 4x, 10x.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement NormalizedSpeedupChart: speedup = baseline_ms / this_ms per query, log-scale horizontal bars, green/red for faster/slower. Baseline = first selected result.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": [], "notes": "Convention: red bars = regression (slower than baseline), green bars =\nimprovement (faster). Magnitude-sorted by abs(delta) so the biggest\nchanges are at the top. Match CLI diverging_bar semantics.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement DivergingBarChart: delta_ms = this_ms - baseline_ms per query, horizontal diverging bars centered at 0, % change labels, sorted by abs(delta).", "wid": "w2"}, {"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add chart selector to Compare.tsx: toggle between Grouped Bar, Normalized Speedup, Diverging Bar. Default Normalized Speedup for 2-result; Grouped Bar for 3+.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-add-comparison-charts", "needs": ["w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add baseline selector for normalized speedup chart: when comparing 3+ results, a dropdown lets the user pick which result is the baseline. Default to the fastest (lowest geomean_ms) result.", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add interactive filtering or controls to the disclosure panel", "instead": "action controls belong elsewhere", "why": "this is read-only provenance"}, {"dont": "expand the panel by default", "instead": "disclosure is for the methodologically curious", "why": "most visitors just want the numbers"}, {"dont": "omit the metric definition", "instead": "always include the definition", "why": "'Geomean query time' is not obvious to a first-time visitor"}], "approach": "Static disclosure - no new data fetching required. All the displayed\nfields come from the detail result object already loaded by ResultDetail.\nThe panel is a simple definition list rendered from props.\n\nCollapsed by default with a clear \"How this was measured \u2193\" label.\nThe label should be visible on first load so the panel is discoverable.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found that a visitor to a result detail\npage cannot determine:\n - What the \"Duration\" number means (sum? mean? median?)\n - Whether tuning was applied (currently just a boolean)\n - Whether warmup queries are included in the timing\n - What execution mode was used (SQL vs DataFrame)\n - What hardware the result was produced on beyond OS/arch/CPU count\n\nThis matters because BenchBox's differentiator over ClickBench is\n\"rich per-query detail - execution plans, tuning configurations,\nvalidation status\". If a visitor can't interpret the numbers, that\nclaim is hollow. The absence of methodology disclosure also makes it\nimpossible to assess whether two results are meaningfully comparable.\n\nThis TODO adds a \"How this was measured\" disclosure section to the\nResultDetail page that surfaces the aggregation method, execution mode,\ntuning state, test type, and key environment parameters.", "id": "explorer-add-methodology-disclosure", "preserves": ["Collapsed by default - visitors who don't care are not burdened", "Existing ResultDetail layout above the disclosure panel - disclosure is additive", "Graceful rendering for bundles that lack execution_mode / tuning_mode (null-safe)"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/components/MethodologyDisclosure.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}], "state": "done", "title": "Add methodology disclosure panel to result detail pages", "verifications": [{"command": "grep -n 'MethodologyDisclosure' results-explorer/src/pages/ResultDetail.tsx", "description": "MethodologyDisclosure component renders in ResultDetail", "expected": "imported and used", "item_id": "explorer-add-methodology-disclosure", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-add-methodology-disclosure", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'execution_mode.*null\\|null.*execution_mode\\|??' results-explorer/src/components/MethodologyDisclosure.tsx", "description": "Component renders for null execution_mode (old bundle compatibility)", "expected": "null-safe access present", "item_id": "explorer-add-methodology-disclosure", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": [], "notes": "Metric definition copy: \"Primary metric is geometric mean of per-query\nexecution times (measurement runs only). Wall-clock total is shown\nseparately. Lower geomean = faster.\"\n\nTest type: \"Power test (single-stream sequential)\" or \"Throughput test\n(N concurrent streams)\".\n\nQuery scope: \"Full benchmark (22 queries)\" or \"Query subset: Q1, Q6, Q17\".\n\nEnvironment: already surfaced (OS/arch/CPU/memory/Python). Add\nexecution_mode and tuning_mode here too.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Design the disclosure panel content: list each disclosed item (metric definition, execution mode, tuning mode, test type, query scope, environment summary) and write the copy for each.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement MethodologyDisclosure component in results-explorer/src/components/. Collapsible panel, collapsed by default. Accepts detail result props; renders the disclosed items as a definition list.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate MethodologyDisclosure into ResultDetail.tsx below the query timing table. Ensure it renders for all bundles including those predating the extended manifest (graceful null handling).", "wid": "w3"}, {"evidence": null, "item_id": "explorer-add-methodology-disclosure", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add comparability summary row to Compare.tsx header: execution_mode + tuning_mode + test_type per result. Surfaces differences without navigating to detail pages.", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fetch the .tuning.json sidecar eagerly", "instead": "lazy-fetch on expand only", "why": "most visitors won't expand the panel"}, {"dont": "hard-block cross-tuning comparisons", "instead": "warn visibly but allow", "why": "intentional tuned vs notuning comparisons are valid and valuable"}, {"dont": "show tuning_hash to end users", "instead": "show tuning_mode and tuning summary instead", "why": "it is an internal grouping key"}], "approach": "Tuning mode label is simple: read tuning_mode from manifest, render a\nstyled badge (Tuned=green, No Tuning=gray, Auto=blue). Mirror the\nTrustBadge component pattern.\n\nTuning config panel on detail page: lazy-fetch the .tuning.json sidecar\nURL (already derivable from bundle_path convention). Render as a\ncollapsible key-value table. Only fetch on expand - not on page load.\n\nCross-tuning comparison: the strategy doc says tuning_mode is a\ncomparability-breaking dimension that should warn, not hard-block. The\nwarning must be visually prominent (yellow banner) but must not prevent\nthe comparison from rendering - a user may intentionally compare tuned\nvs notuning to see the delta.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found that the explorer records only\n`has_tuning: boolean` despite tuning config being one of BenchBox's\nprimary differentiators. A visitor comparing DuckDB \"tuned\" vs \"notuning\"\nruns has no way to know they differ on this axis, let alone see what\ntuning was applied.\n\nThis TODO:\n 1. Surfaces tuning_mode (\"tuned\" / \"notuning\" / \"auto\") as a visible\n label on result detail pages and browse tables\n 2. Loads and displays the tuning config summary from the .tuning.json\n sidecar (if present) on the detail page\n 3. Enables the Compare view to show a tuning-mode label per result so\n cross-tuning comparisons are legible (not blocked - just labeled)\n\nPrerequisite: explorer-extend-manifest-and-pipeline must land first\n(tuning_mode and tuning_hash must be in the manifest).", "id": "explorer-add-tuning-config-visibility", "preserves": ["Existing compare view behavior when tuning is the same across both results (no regression)", "Results without a .tuning.json sidecar render cleanly - tuning panel shows 'No tuning config recorded' not an error", "TrustBadge layout unchanged - TuningBadge is a separate element"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (covered by prerequisite TODO)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/TuningBadge.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/"}], "state": "done", "title": "Surface tuning config in explorer detail pages and enable cross-tuning comparison", "verifications": [{"command": "grep -n 'TuningBadge\\|tuning_mode' results-explorer/src/pages/BenchmarkIndex.tsx", "description": "Tuning badge renders in browse tables", "expected": "TuningBadge imported and used", "item_id": "explorer-add-tuning-config-visibility", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'tuning_mode\\|Different tuning' results-explorer/src/pages/Compare.tsx", "description": "Cross-tuning warning renders in Compare", "expected": "warning logic present", "item_id": "explorer-add-tuning-config-visibility", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-add-tuning-config-visibility", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add tuning_mode label to BenchmarkIndex and PlatformIndex browse tables: small badge (styled like trust_label badge) showing 'Tuned' / 'No Tuning' / 'Auto'. Use tuning_mode from manifest.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add tuning section to ResultDetail.tsx: show tuning_mode prominently; collapsible panel lazy-fetches .tuning.json sidecar and renders key-value config.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": ["w1", "w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update Compare.tsx: add tuning_mode label per result in the comparison header. If the two results have different tuning_mode values, show a visible 'Different tuning config' warning badge.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-add-tuning-config-visibility", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add tuning filter to DuckDB-WASM browse queries: allow filtering by tuning_mode in Browse pages (All / Tuned / No Tuning). Uses tuning_mode column added by pipeline extension.", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "recompute geomean in the browser", "instead": "read manifest.geomean_ms directly", "why": "it is precomputed in the manifest"}, {"dont": "remove total_duration_s from the UI", "instead": "demote it, don't delete it", "why": "it is useful context"}, {"dont": "silently show geomean as 'Duration'", "instead": "always label it", "why": "that hides the metric choice from the reader"}], "approach": "The geomean_ms field will exist in the manifest after explorer-extend-manifest-\nand-pipeline lands. This TODO is a UI-layer change: swap the field used for\nsorting, update column labels, and add a metric definition. No new data\nfetching or computation needed.\n\nTooltip copy: \"Geometric mean of per-query execution times (measurement\nruns only). More comparable than wall-clock total when query counts differ.\"\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-extend-manifest-and-pipeline"], "description": "Post-launch audit (2026-04-14) found a metric discrepancy: the CLI uses\ngeometric mean of per-query times as the primary comparison metric; the\nexplorer uses total_duration_s (wall-clock sum). These produce different\nnumbers for the same result, and two results with different query counts\n(e.g., full TPC-H vs a query subset) produce incomparable totals.\n\nThis TODO:\n 1. Makes geomean_ms the primary sort/compare axis in the Compare view,\n Browse tables, and any cohort summary\n 2. Relabels total_duration_s wherever it appears so readers know what\n they are looking at\n 3. Adds a clear metric definition in the UI (tooltip or footnote)\n\nPrerequisite: explorer-extend-manifest-and-pipeline must land first\n(geomean_ms must exist in the manifest before this TODO can consume it).", "id": "explorer-align-duration-metric", "preserves": ["All data loading logic - this is display/labeling changes only", "Existing URL/query-param format for compare view", "Null-safety: if geomean_ms is null (e.g., old bundle), fall back to total_duration_s display with a 'N/A' geomean", "total_duration_s remains visible - it is honest wall-clock data; just secondary"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (pipeline changes are in the prerequisite TODO)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/components/ (no component changes needed)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/"}], "state": "done", "title": "Align explorer duration metric to geomean; retire wall-clock as primary comparison axis", "verifications": [{"command": "grep -n 'geomean_ms' results-explorer/src/pages/Compare.tsx", "description": "Compare view sorts by geomean_ms", "expected": "at least one sort reference to geomean_ms", "item_id": "explorer-align-duration-metric", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'Wall-clock\\|total_duration' results-explorer/src/pages/Compare.tsx", "description": "total_duration_s is still visible but labeled", "expected": "present with descriptive label", "item_id": "explorer-align-duration-metric", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-align-duration-metric", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-align-duration-metric", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update Compare.tsx: use geomean_ms as primary sort/compare axis. Keep total_duration_s as secondary column labeled 'Wall-clock total'.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-align-duration-metric", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update BenchmarkIndex.tsx and PlatformIndex.tsx browse tables: sort default by geomean_ms ascending. Relabel 'Duration' column to 'Geomean (ms)' with a tooltip explaining the metric.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-align-duration-metric", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update ResultDetail.tsx summary section: show geomean_ms as 'Geomean query time' prominently; move total_duration_s to secondary metadata row with label 'Total wall-clock'.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-align-duration-metric", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add metric definition tooltip or footnote: 'Geometric mean of per-query execution times (measurement runs only). Lower is faster. Comparable across runs with different query counts.'", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "drop the secondary metric from the UI", "instead": "show both, primary prominent, secondary muted", "why": "some users will want to cross-reference"}, {"dont": "hard-code the benchmark \u2192 metric mapping in TypeScript", "instead": "serialize the choice into the summary artifact", "why": "it creates two sources of truth"}, {"dont": "recompute power_score in the pipeline", "instead": "read the pre-computed value and null if absent", "why": "the CLI already emits it per-bundle"}], "approach": "The cleanest path is a single registry lookup in the pipeline.\nPipeline reads summary.benchmark, looks up the metric config, attaches\nit to the emitted BenchmarkSummary. The frontend is purely declarative\nfrom that point - no family logic in TypeScript.\n\nRegistry entries are strings keyed on canonical benchmark family names\n(tpch, tpcds, star_schema, clickbench, nyctaxi, tsbs-devops, h2odb,\ndatavault, ...). Unknown families fall back to geomean_ms asc.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-emit-benchmark-summary-artifact"], "description": "The strategy review (2026-04-15) identified that defaulting to geometric\nmean as the single ranking metric across all benchmarks is wrong:\n\n- TPC-H and TPC-DS define **power_score** (= 3600 / geomean \u00d7 scale\n adjustment) as the standard metric. Users arriving from published\n TPC results expect to see that number.\n- ClickBench and SSB have no published aggregate; geomean is a\n reasonable default there.\n- Custom or ad-hoc benchmarks should expose both so users can pick.\n\nThis TODO introduces a per-benchmark-family ranking metric registry and\nthreads it through the pipeline and UI. The matrix leaderboard uses\n`power_score` for TPC-H/TPC-DS and `geomean_ms` for everything else;\nboth are shown in each row, but the primary sort is the canonical one.\n\nScope excludes computing power_score (the CLI already does this) - this\nTODO only ensures the explorer surfaces and ranks by the right metric.", "id": "explorer-align-ranking-metric-with-tpc-standards", "preserves": ["CLI-computed power_score values - do not recompute in the pipeline; read from bundle metadata", "Existing `geomean_ms` field on platform rows - remains present as secondary metric for all benchmarks", "The 'Canonical Duration Metric' decision from 2026-04-14 - refines, does not overturn"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "_project/analysis/benchbox-results-platform-strategy.md"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Select ranking metric per benchmark family (power_score for TPC, geomean elsewhere)", "verifications": [{"command": "grep -n 'RANKING_METRIC_BY_FAMILY' benchbox/core/explorer/models.py", "description": "Registry is defined and exported", "expected": "constant defined", "item_id": "explorer-align-ranking-metric-with-tpc-standards", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import gzip, json; d = json.loads(gzip.open(\"results-explorer/public/data/benchmarks/tpch_sf0.01_power.json.gz\").read()); assert d[\"ranking\"][\"primary_metric\"] == \"power_score\"'", "description": "tpch artifact declares power_score as primary metric", "expected": "no assertion failure", "item_id": "explorer-align-ranking-metric-with-tpc-standards", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c 'import gzip, json; d = json.loads(gzip.open(\"results-explorer/public/data/benchmarks/star_schema_sf0.01_power.json.gz\").read()); assert d[\"ranking\"][\"primary_metric\"] == \"geomean_ms\"'", "description": "star_schema artifact declares geomean_ms as primary metric", "expected": "no assertion failure", "item_id": "explorer-align-ranking-metric-with-tpc-standards", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": [], "notes": "Map benchmark family \u2192 {primary: str, secondary: str, primary_order:\n'asc'|'desc'}. Initial entries:\n - tpch \u2192 power_score (desc)\n - tpcds \u2192 power_score (desc)\n - star_schema \u2192 geomean_ms (asc)\n - clickbench \u2192 geomean_ms (asc)\nFallback for unknown families: geomean_ms (asc).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add RANKING_METRIC_BY_FAMILY registry in benchbox/core/explorer/models.py", "wid": "w1"}, {"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": ["w1"], "notes": "Shape: {primary_metric, secondary_metric, primary_order}. Lives\nalongside platform rows so the frontend does not need to know the\nregistry.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend BenchmarkSummary serialization with a top-level `ranking` field", "wid": "w2"}, {"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": ["w2"], "notes": "Display primary metric in a dedicated column with its unit;\ndisplay secondary metric below it in muted text. Power score has\nno unit suffix (composite); geomean is \"Xms\". Column header is\ndynamic: \"Power Score\" vs \"Geomean\" vs family-specific label.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "In QueryHeatmap.tsx, read summary.ranking and sort/display by primary_metric", "wid": "w3"}, {"evidence": null, "item_id": "explorer-align-ranking-metric-with-tpc-standards", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cross-reference RANKING_METRIC_BY_FAMILY in the strategy doc's 'Canonical Duration Metric' section", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change sort behavior in w1 for any currently-registered benchmark family", "instead": "(not specified)", "why": "only the mechanism changes"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-emit-benchmark-summary-artifact. The two Required issues\n(missing percentile_stats test, missing sort-order test) were fixed\nimmediately post-review. This item tracks the remaining nit/consider findings.", "id": "explorer-benchmark-summary-review-followups", "preserves": ["_natural_sort_key output for all current query IDs - empty-string filter is behavior-equivalent in w2", "_platform_row_sort_key behavior for all current benchmark families - only extensibility changes in w1"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from benchmark-summary artifact code review", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_benchmark_summary.py -v -n 0", "description": "Benchmark summary tests still pass", "expected": "16 passed", "item_id": "explorer-benchmark-summary-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Full fast test suite green", "expected": "all green", "item_id": "explorer-benchmark-summary-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:78-87 - _platform_row_sort_key hardcodes sort direction: negate\npower_score (higher-is-better), use geomean_ms as-is (lower-is-better).\nThis is correct for all current benchmark families but duplicates logic\nalready encoded in RankingConfig.primary_order.\n\nIf a future family added a metric where the direction didn't match these\nassumptions, the sort would silently be wrong.\n\nFix: pass `benchmark` name to _platform_row_sort_key and look up\nget_ranking_config(benchmark).primary_order to derive the negation flag:\n cfg = get_ranking_config(benchmark)\n primary_val = getattr(row, cfg.primary_metric)\n if primary_val is None:\n return (ineligible, float(\"inf\"))\n metric = -primary_val if cfg.primary_order == \"desc\" else primary_val\n return (ineligible, metric)\n\nNote: _build_benchmark_summaries already has the benchmark name in scope\nand already calls get_ranking_config(benchmark) at line 142.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Consult RankingConfig.primary_order in _platform_row_sort_key", "wid": "w1"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:54-56 - re.split(r\"(\\d+)\", s) produces empty strings between\nnumber groups (e.g., \"Q10\" \u2192 [\"\", \"Q\", \"10\", \"\"]). Python's sort handles\nempty strings correctly (they sort first), but the intent is clearer with:\n return [int(c) if c.isdigit() else c.lower() for c in re.split(r\"(\\d+)\", s) if c]\n\nLow-risk: behavior is identical for all current query IDs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Filter empty strings from _natural_sort_key output", "wid": "w2"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "transformer.py:272 - filter `dt.display_ms is not None and dt.display_ms > 0`\nexcludes zero-millisecond measurements. For extremely fast queries (sub-ms\nrounded to 0), this would silently exclude them from percentile stats.\n\nFix: add a comment explaining the reason - presumably 0ms indicates a\ntiming error or warmup artifact, not a genuine instant result. If the\nintent is to exclude non-positive values, document it. If 0ms is valid,\nchange to `>= 0`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document why percentile filter uses display_ms > 0 not >= 0", "wid": "w3"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:381,395 - Path traversal check uses string prefix:\n if not str(detail_path).startswith(str(details_dir.resolve()) + \"/\"):\n\nPython 3.9+ has Path.is_relative_to() which is more readable and avoids\nthe string manipulation:\n if not detail_path.is_relative_to(details_dir.resolve()):\n\nFunctionally identical; Python \u22653.9 is already required by the project.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Use Path.is_relative_to() for path-traversal guard", "wid": "w4"}, {"evidence": null, "item_id": "explorer-benchmark-summary-review-followups", "needs": [], "notes": "pipeline.py:165-256 - _build_meta_leaderboard is ~90 lines with nested\nfunctions and multiple accumulation passes. Consider extracting:\n _rank_platforms_in_cohort(summary) \u2192 list[dict]\n _compute_average_ranks(cohort_rows) \u2192 dict[str, float]\n\nNot urgent; logic is correct and well-commented. Low value until the\nfunction needs modification.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor _build_meta_leaderboard into smaller helpers", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add new reduction code under `benchbox/core/results/`", "instead": "add new reductions under `benchbox/core/explorer_pipeline/` and import existing utilities read-only from `benchbox/core/results/` as needed", "why": "this TODO's scope_limit keeps that tree read-only"}, {"dont": "allow Home, PlatformIndex, BenchmarkIndex, Compare, or ResultDetail to keep metric reads outside DuckDB", "instead": "migrate every metric-bearing surface", "why": "that violates the core architecture goal"}, {"dont": "dedupe `cohort_metadata` rows by `(cohort_key, platform_id)` or hide publishable variants of the same platform in any canonical table", "instead": "key `cohort_metadata` on `(cohort_key, result_id)`, carry `tuning_mode`/`trust_label`/`short_id` alongside, and let the UI decide any display-time collapsing", "why": "users must be able to compare across tuning levels and trust levels from the DuckDB store, and the pipeline is the wrong layer to decide which variant 'represents' a platform"}, {"dont": "delete metric-JSON emitters in W2", "instead": "W2 is additive, W4 deletes each emitter in the same slice as its last consumer, W7 does the final sweep", "why": "Home/PlatformIndex/BenchmarkIndex/Compare/ResultDetail still read from them until W4"}, {"dont": "keep separate canonical logic in JSON loaders and DuckDB helpers", "instead": "move all interactive metric reads behind one DuckDB contract", "why": "that recreates the current drift risk"}, {"dont": "let `public/data/bundles/*.json` drift into a rendered metric path", "instead": "enforce via the G-8 second assertion that no `src/pages/` or `src/components/` file fetches `/results/data/bundles/`", "why": "bundles are committed JSON source inputs and a download-only affordance, not a runtime read source"}, {"dont": "let bootstrap convenience justify metric-bearing `manifest.json` or `results_schema.json` sidecars", "instead": "keep any surviving JSON bootstrap-only and enforce via a key-whitelist contract test", "why": "that silently recreates split-brain storage"}, {"dont": "preserve beta-era JSON metric contracts just because they already exist", "instead": "optimize for the correct canonical model, not compatibility theater", "why": "the explorer has not shipped and the current contract is wrong"}, {"dont": "recompute display reductions, rankings, compliance classes, speedup ratios, or fastest/slowest spreads in TypeScript", "instead": "persist canonical values in DuckDB and read them directly", "why": "the canonical Python reference computation in `benchbox/core/explorer_pipeline/` owns those semantics"}, {"dont": "rely on dict last-write-wins inside `_build_meta_leaderboard` `platform_agg` when the same `platform_id` ranks more than once in the same cohort", "instead": "compare ranks on write and keep the best (or rely on best-first iteration order with `setdefault`)", "why": "the second write clobbers the better rank with a worse one and poisons `avg_rank` for the Home meta-leaderboard"}, {"dont": "require derived user-visible metrics to equal verbatim JSON source fields", "instead": "prove final-value parity against the canonical Python reference computation instead", "why": "many visible values are canonical reductions or rollups"}, {"dont": "start implementation before the research gate closes the schema and cutover details", "instead": "lock the rules first and record them", "why": "this is a contract refactor across pipeline and UI"}, {"dont": "tighten `db.ts` to reject on attach failure until the last manifest.ts consumer migrates", "instead": "W3 keeps the fallback in place; W4 lands the hardening in the same slice that deletes `manifest.ts`", "why": "pages still on the JSON path rely on the soft failure path"}, {"dont": "treat pre-calculated CLI JSON artifacts, current TS reduction libs, or any bridge artifact as the parity oracle", "instead": "define parity as `DuckDB read-path value == canonical Python reference computation in `benchbox/core/explorer_pipeline/` (with read-only imports from `benchbox/core/results/`)`, within the per-metric tolerance declared in `visible_metrics.yaml`", "why": "doing so freezes today's drift into tomorrow's contract"}, {"dont": "widen scope into CLI, hosted API, or submission-path changes", "instead": "leave write-path contracts untouched", "why": "this TODO is about the static explorer read model only"}], "approach": "Follow a research-first but opinionated cutover, not an additive\ncompatibility migration.\n\nFirst, inventory every runtime metric fetch and every pipeline artifact that\ncurrently feeds it. Then define the DuckDB schema and materializations that\nreplace those paths. After the research gate, wire the explorer pipeline\ndirectly from committed JSON objects into DuckDB, move all metric surfaces\nonto typed DuckDB queries, and remove metric-bearing JSON outputs and\nloaders.\n\nTemporary parity checks are allowed as verification scaffolding, but JSON is\nnot a second product contract. If a surface shows metrics to the user, those\nmetrics must exist in DuckDB first.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-18T00:00:00Z", "completed_pr": null, "created_at": "2026-04-18T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-canonical-browser-duckdb-read-model", "id": 403, "reason": "This TODO is static-first only; the read path remains browser-side and read-only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Explicit scope decision: explorer stays static/browser-side; introducing a hosted/authenticated API is a declined direction, not deferred work.", "summary": "Introduce any hosted API, authenticated query service, or server-side execution path"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-canonical-browser-duckdb-read-model", "id": 404, "reason": "The adopted product strategy keeps benchmark landing pages matrix-first and the query workbench secondary.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Adopted product strategy explicitly keeps matrix-first landing pages; making /results/query the default landing page was rejected.", "summary": "Make `/results/query` the default landing page for the explorer"}], "deps": [], "description": "Rework the results explorer so benchmark landing pages stay matrix-first\nwhile DuckDB-WASM becomes the single runtime store for every user-visible\nmetric. Today the implementation is split-brain: `Query.tsx` already runs\nagainst DuckDB, but `Home.tsx`, `PlatformIndex.tsx`, `BenchmarkIndex.tsx`,\n`Compare.tsx`, and `ResultDetail.tsx` still fetch metric-bearing JSON\nthrough `results-explorer/src/manifest.ts`.\n\nThe target contract is stricter than \"prefer DuckDB when convenient\":\ncommitted JSON result objects are immutable source inputs; the explorer\npipeline projects them directly into `results.duckdb`; and every\nuser-visible metric, aggregate, rank, compare cell, detail timing, and\ncohort summary is read from DuckDB. Any JSON that survives after this work\nis bootstrap-only navigation metadata and must not carry user-visible\nmetrics.\n\nThis item still needs a research gate before implementation, but that gate\nis for schema and cutover precision, not to re-open whether JSON remains a\nfirst-class metric surface. The end state is DuckDB-only for explorer\nmetrics.\n\nVerified current-state anchors:\n - `results-explorer/src/db.ts` attaches `/results/data/results.duckdb`\n with `ATTACH ... AS bench (READ_ONLY)`.\n - `results-explorer/src/pages/Query.tsx` already queries `bench.results`\n but still bootstraps schema from `results_schema.json`.\n - `results-explorer/src/manifest.ts` fetches `manifest.json`,\n `benchmarks/*.json.gz`, `details/*.json`, `compare/*.json`,\n `meta_leaderboard.json`, and `short_ids.json`, making JSON the\n dominant runtime read path today.\n - `benchbox/core/explorer_pipeline/duckdb_builder.py` currently emits only\n a thin `results` table plus `results_schema.json`, so DuckDB cannot yet\n serve Home, platform, benchmark, compare, and detail surfaces alone.", "id": "explorer-canonical-browser-duckdb-read-model", "preserves": ["Any temporary bridge artifact used during W2-W4 page-by-page migration is regenerated from the same pipeline pass that writes DuckDB, never hand-patched, and cannot define, override, or outlive user-visible metrics", "Benchmark landing pages remain matrix-first default routes; `/results/query` stays a secondary analysis surface rather than the primary explorer entrypoint", "Committed JSON result objects remain immutable source inputs; the browser does not fetch them directly for metric rendering", "DuckDB is always attached read-only in the browser; no implementation step may weaken `ATTACH ... (READ_ONLY)` or permit persistent mutation from the SQL workbench", "Every result trusted enough to publish appears in the DuckDB published-results dataset with its identity preserved (`result_id`, `tuning_mode`, `trust_label`); the pipeline never silently collapses, dedupes, or hides publishable variants - users must be able to compare across tuning levels AND trust levels from the same store", "Every user-visible metric, rank, aggregate, and detail timing shown in the explorer originates from DuckDB rather than JSON sidecars or TypeScript-only recomputation", "Final-value parity for derived metrics is defined against the canonical Python reference computation in `benchbox/core/explorer_pipeline/` (with read-only imports from `benchbox/core/results/`), NOT against pre-calculated CLI JSON, current TS reduction libs, or bridge artifacts", "Raw JSON bundles under `public/data/bundles/` are a download-only affordance; no `src/pages/` or `src/components/` file fetches from `/results/data/bundles/` at runtime", "Shareable compare flows remain available after cutover, but route convenience is subordinate to the DuckDB-only metric contract", "The explorer remains static-hosted and browser-executed; no server API, auth dependency, or hosted query runtime is introduced by this TODO"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": ".github/workflows/"}, {"kind": "do_not_modify", "path_glob": "benchbox/cli/"}, {"kind": "do_not_modify", "path_glob": "docs/reference/hosted-results-contract.md"}, {"kind": "only_modify", "path_glob": "_project/planning/visible_metrics.yaml"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "docs/development/benchbox-results-platform-strategy.md"}, {"kind": "only_modify", "path_glob": "docs/development/browser-duckdb-schema.sql"}, {"kind": "only_modify", "path_glob": "results-explorer/e2e/"}, {"kind": "only_modify", "path_glob": "results-explorer/playwright.config.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}], "state": "done", "title": "Make DuckDB-WASM the sole browser store for all user-visible explorer metrics", "verifications": [{"command": "rg -n \"DuckDB-only metric contract|direct ingest contract|bootstrap-only|metric-bearing JSON\" _project/TODO/main/planning/explorer-canonical-browser-duckdb-read-model.yaml docs/development || true", "description": "Research gate records the DuckDB-only metric contract before code work begins", "expected": "The contract and cutover rules are recorded before implementation starts", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run --project _project/scripts -- python _project/scripts/validate_todo.py _project/TODO/main/planning/explorer-canonical-browser-duckdb-read-model.yaml && uv run --project _project/scripts -- python _project/scripts/todo_cli.py check-graph", "description": "TODO metadata and graph validate cleanly", "expected": "TODO file validates; dependency graph check passes", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -q", "description": "Explorer pipeline tests remain green after DuckDB schema expansion", "expected": "all tests pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck && npm test && npm run build", "description": "Explorer frontend typecheck, tests, and production build remain green", "expected": "all green", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -c \"import duckdb; con=duckdb.connect('results-explorer/public/data/results.duckdb', read_only=True); names={r[0] for r in con.execute('PRAGMA show_tables').fetchall()}; required={'results','query_display_timings','query_executions','result_detail_metrics','benchmark_matrix_cells','benchmark_rankings','platform_index_rows','cohort_metadata','meta_leaderboard','short_ids'}; missing=required-names; assert not missing, missing\"", "description": "Canonical DuckDB base tables exist for the browser metric contract", "expected": "no assertion failure", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 5}, {"command": "if rg -n \"/results/data/(benchmarks/|details/|compare/|meta_leaderboard\\.json|results_schema\\.json|short_ids\\.json)\" results-explorer/src; then exit 1; else echo CLEAN; fi", "description": "No runtime metric JSON fetches remain in the explorer source tree", "expected": "CLEAN", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 6}, {"command": "if rg -n \"/results/data/bundles/\" results-explorer/src/pages results-explorer/src/components; then exit 1; else echo CLEAN; fi", "description": "No page or component fetches the raw-bundle download path", "expected": "CLEAN", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 7}, {"command": "bash -c 'for f in meta_leaderboard.json short_ids.json results_schema.json; do if [ -e \"results-explorer/public/data/$f\" ]; then echo FAIL_$f; exit 1; fi; done; for d in details compare benchmarks; do if [ -d \"results-explorer/public/data/$d\" ]; then echo FAIL_$d; exit 1; fi; done; echo CLEAN'", "description": "Pipeline output contains no legacy metric-bearing artifacts (bundles/ is allowed as download-only)", "expected": "CLEAN", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 8}, {"command": "bash -c 'f=results-explorer/public/data/manifest.json; if [ ! -e \"$f\" ]; then echo ABSENT; exit 0; fi; uv run -- python -c \"import json,sys; keys=set(json.load(open(\\\"$f\\\")).keys()); allowed={\\\"routes\\\",\\\"navigation\\\",\\\"build_meta\\\",\\\"schema_version\\\"}; extra=keys-allowed; sys.exit(0 if not extra else (print(\\\"EXTRA_KEYS:\\\",sorted(extra)) or 1))\"'", "description": "Surviving manifest.json (if any) carries only whitelisted non-metric keys", "expected": "ABSENT or no extra keys", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 9}, {"command": "git diff --name-only main...HEAD -- benchbox/core/results/ | wc -l | tr -d ' '", "description": "No reduction code was added under `benchbox/core/results/` by this TODO", "expected": "0", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 10}, {"command": "cd results-explorer && npm test -- duckdbMetricParity", "description": "Cross-surface parity holds for canonical display values", "expected": "home, benchmark, compare, detail, and query parity tests pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 11}, {"command": "cd results-explorer && npm test -- Query", "description": "Read-only SQL enforcement still holds", "expected": "DDL/DML attempts fail; read-only SELECT flows still pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 12}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast Python suite stays green after the migration", "expected": "all tests pass", "item_id": "explorer-canonical-browser-duckdb-read-model", "last_result": null, "last_run": null, "seq": 13}], "work": [{"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": [], "notes": "Deliverables:\n 1. A concrete call-site inventory for Home, PlatformIndex,\n BenchmarkIndex, Compare, ResultDetail, and Query showing the\n current data source and the target DuckDB table/view for each\n user-visible metric.\n 2. A schema decision record mapping every user-visible explorer metric\n to DuckDB tables/views, plus the minimal bootstrap JSON that may\n remain and proof that it carries no metrics.\n 3. A direct ingest contract documenting `committed JSON -> explorer\n pipeline -> DuckDB` and explicitly rejecting metric-bearing JSON as\n a steady-state runtime contract.\n 4. A cutover/deletion order showing which JSON readers and JSON\n artifacts are removed or demoted once DuckDB parity is proven.\n 5. A committed `visible_metrics.yaml` fixture enumerating every\n user-visible metric column across the ten canonical tables and\n classifying each as `raw_copy` (source-fidelity contract against\n the committed bundle field) or `derived` (final-value parity\n contract against the canonical Python reference computation).\n This file is the registry G-2 consumes and is required before W2\n starts.\n 6. A definition-of-terms block pinning the meaning of the parity\n vocabulary used across this TODO:\n - \"canonical Python reference computation\" = the pipeline's own\n Python implementation inside\n `benchbox/core/explorer_pipeline/` (plus read-only imports\n from `benchbox/core/results/`). TypeScript is never a\n reference oracle.\n - \"final-value parity\" = the rendered user-visible value\n produced by the DuckDB-backed read path equals the canonical\n Python reference computation for the same inputs, within a\n documented tolerance per metric. Parity is NOT defined\n against whatever the CLI, a pre-calculated JSON artifact, or\n the current TS reduction libraries happen to emit today.\n - \"source-fidelity parity\" = raw fields copied into DuckDB\n equal the corresponding field in the committed JSON bundle\n verbatim.\n 7. A compute-ownership note answering: for every derived metric in\n `visible_metrics.yaml`, does the canonical Python reference\n computation already exist under\n `benchbox/core/explorer_pipeline/` or `benchbox/core/results/`,\n or must a new module be added under\n `benchbox/core/explorer_pipeline/` (the only write-legal home for\n new reduction code under this TODO's scope_limit)? If a derived\n metric is TS-only today (e.g. currently lives in\n `queryReduction.ts`, `chartMath.ts`, `ranking.ts`, or\n `compliance.ts`), record it here and plan to port the logic into\n `benchbox/core/explorer_pipeline/` during W2.\n\nQuantified research gates (each must be green or explicitly risk-accepted\nbefore w2 starts; record findings in the schema decision record above):\n\n - RG-1 Cold-start init budget: measure time from page load to first\n successful SQL result against `results.duckdb` (cold cache, mid-tier\n laptop, Safari/Chrome/Firefox). Pass: p50 <1500ms, p95 <3500ms at\n current corpus; p95 <5000ms at 10x corpus.\n - RG-2 HTTP range-reads: verify `DuckDBDataProtocol.HTTP` with\n READ_ONLY ATTACH issues range requests and that benchbox.dev /\n GH Pages returns `Accept-Ranges: bytes` with 206 responses for a\n single-row `WHERE result_id = ?` query. Pass: the sum of HTTP\n Content-Length bytes transferred for a cold single-row lookup is\n \u226410% of the total `results.duckdb` file size, measured on the\n current corpus.\n - RG-3 DB size budget: build DB at 1x, 5x, 20x, 100x synthetic corpus\n (duplicated bundles with mutated ids). Record the `results.duckdb`\n file size on disk at each scale and publish the 1x baseline in the\n schema decision record. Pass: on-disk `results.duckdb` size grows\n sub-linearly in bundle count AND stays \u226420 MB at 100x. If the\n measured 1x baseline makes 20 MB at 100x structurally impossible\n (e.g. baseline already >1 MB per bundle), record a revised numeric\n ceiling in the decision record and escalate to RG-5 evaluation.\n - RG-4 COEP/COOP hosting: confirm either the `mvp` single-threaded\n DuckDB-WASM bundle meets RG-1 budgets, OR COEP/COOP headers can be\n served by the deploy target. Pass: a chosen WASM bundle works with\n the current hosting configuration.\n - RG-5 Single DB vs partitioned: only if RG-3 fails. Prototype\n multi-DB `ATTACH` from WASM. Default is single DB per the \"one and\n only\" rule; partitioning is a fallback, not a preference.\n - RG-6 View materialization budget: benchmark each canonical surface\n query cold against 100x corpus. Pass: <250ms p95 as a plain VIEW,\n otherwise promote to materialized `CREATE TABLE AS`.\n - RG-7 Attach-failure UX: every page renders a diagnostic ErrorMessage\n when the DB fails to attach (0-byte, 404, CORS). Pass: unit-tested\n per page, no silent JSON fallback.\n - RG-8 toJSON fidelity: round-trip every column type through\n `queryRows` in a Vitest matrix. Pass: no precision loss on DOUBLE /\n BIGINT / DECIMAL; date handling is explicit.\n - RG-9 Read-only fuzz: adversarial SQL (CREATE TABLE, ATTACH, COPY TO,\n INSTALL, LOAD, HTTPFS) against the READ_ONLY attached DB in WASM.\n Pass: all fail harmlessly with no side effects.\n\nResearch gate exit criterion: all RGs green or documented accepted-risk\nnote reviewed with the user, and a frozen `schema.sql` fixture committed\nalongside the decision record.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Research gate: inventory every metric-bearing JSON read, define the DuckDB-only contract, and lock the cutover rules before implementation", "wid": "w1"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w1"], "notes": "Minimum DuckDB-resident contracts (tables or materialized views):\n - `results`\n - `query_display_timings`\n - `query_executions`\n - `result_detail_metrics`\n - `benchmark_matrix_cells`\n - `benchmark_rankings`\n - `platform_index_rows`\n - `cohort_metadata`\n - `meta_leaderboard`\n - `short_ids`\nFinal names can change in w1, but every surface above must have an\nexplicit DuckDB contract before frontend migration begins.\n\nPipeline implementation rules:\n - `duckdb_builder.py` rewrites the 19-column stub (currently lines\n 25-131) into a full-schema builder. Uses `CREATE OR REPLACE` on each\n build; the corpus is the only source of truth, so there are no\n migrations.\n - `pipeline.py` `run()` becomes: enumerate `bundles/*.json`, validate\n via `SchemaV2Validator`, compute derived values in memory (display\n timings, short_id, is_ranking_eligible, compliance_class, matrix\n cells, rankings, meta-leaderboard rows, cohort metadata), bulk\n insert in one transaction, copy raw bundles to\n `public/data/bundles/` for the raw-bundle download affordance ONLY.\n - All derived-value reductions consumed by the pipeline live under\n `benchbox/core/explorer_pipeline/` or are imported read-only from\n `benchbox/core/results/`. This is the canonical Python reference\n computation for G-2 parity. If a reduction is currently TS-only\n (e.g. in `queryReduction.ts`, `chartMath.ts`, `ranking.ts`,\n `compliance.ts`), port the algorithm into\n `benchbox/core/explorer_pipeline/` in this work item; the TS copy\n becomes dead code to be deleted in W4 once no page imports it.\n - W2 is ADDITIVE on the pipeline side. The pipeline starts emitting\n the ten canonical tables and keeps all existing metric-bearing\n JSON emitters running as bridge artifacts until the consuming\n page migrates in W4. No metric-JSON emitter is deleted at W2\n exit. This is the only reading that is consistent with \"do not\n remove the artifact still backing a live page until that page\n migrates\" - W4 removes each emitter in the same slice as its\n last consumer, and W7 does the final sweep.\n - Bridge artifacts are scaffolding, never a second accepted metric\n contract. During W2-W4, if a bridge artifact and DuckDB disagree,\n DuckDB is canonical by rule and the bridge artifact must be\n regenerated from the same pipeline pass, not hand-patched.\n - If `manifest.json` survives past W7, it is strictly non-metric\n route/navigation bootstrap derived from DuckDB-owned identities\n only. A key-whitelist contract test (see G-8) enforces this.\n - No TypeScript-side recomputation of display reductions, geomean,\n rank, compliance class, speedup ratios, or any other user-visible\n metric is permitted downstream. TS is allowed to do presentation\n formatting (locale formatting, unit suffixes, truncation) on\n values already produced by the canonical Python reference\n computation and read from DuckDB.\n\nExit:\n - All ten canonical tables/materializations exist with their W1-\n defined schemas (G-1).\n - Source-fidelity tests pass for every `raw_copy` metric in\n `visible_metrics.yaml` (raw bundle field == DuckDB column, byte/\n value exact).\n - Final-value parity tests pass for every `derived` metric in\n `visible_metrics.yaml`: for the same inputs, the DuckDB-backed\n read path and the canonical Python reference computation produce\n the same user-visible value within the per-metric tolerance\n declared in the fixture. Parity is NOT defined against\n pre-calculated CLI JSON artifacts or current TS reductions.\n - Schema contract test passes; short-id uniqueness test passes;\n no-computed-metric-view contract test passes.\n - All existing metric-bearing JSON emitters still run as bridge\n artifacts; `public/data/` still contains the current shape.\n Removal is deliberately deferred to W4/W7.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor the explorer pipeline so DuckDB is built directly from committed JSON artifacts and materializes every user-visible metric surface", "wid": "w2"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w8"], "notes": "Consolidate TS-side data access around `results-explorer/src/db.ts`\nplus typed query helpers. Retire `results-explorer/src/manifest.ts` as\na metric source. If manifest/bootstrap JSON survives, it is route and\nnavigation metadata only. `Query.tsx` is fully migrated in this work\nitem (it already queries DuckDB - W3 removes its last JSON\ndependency, `results_schema.json`) and is NOT reopened in W4. W4\nhandles Home, PlatformIndex, BenchmarkIndex, ResultDetail, and\nCompare. Query's schema discovery uses DuckDB introspection\n(`duckdb_tables`, `duckdb_columns`, `DESCRIBE`) or a DuckDB-resident\n`schema_metadata` table if introspection lands in the RG-1 hot path.\nThe `db.ts:1-17` header comment (which still calls DuckDB a \"Phase 2\"\nfeature) is rewritten in this work item to match the new contract.\n\nNew modules:\n - `results-explorer/src/lib/duckdbQueries.ts` - typed helpers per\n canonical surface: `getMetaLeaderboard`, `getBenchmarkMatrixCells`,\n `getBenchmarkRanking`, `getPlatformIndexRows`,\n `getResultDetailMetrics`, `getQueryDisplayTimings`, `getCohort`,\n `resolveShortId`, `listResults`.\n - `results-explorer/src/lib/duckdbSchema.ts` - introspection-based\n schema discovery for the Query workbench column picker.\n\n`db.ts` hardening sequencing:\n - In W3 the silent JSON fallback at `db.ts:86-89` STAYS in place.\n Removing it while Home/PlatformIndex/BenchmarkIndex/ResultDetail/\n Compare still import from `manifest.ts` would turn an attach\n failure into a hard error for pages that don't yet use DuckDB.\n - What W3 adds now: a `DbProvider` context exposing `{ db, schema,\n state }` and a `useQuery(sql, params, deps)` hook with\n `{ loading, error, data }` so pages do not each guard on init.\n These are opt-in; manifest-backed pages continue to work.\n - The attach-failure hardening - reject `getDb()` on attach\n failure, delete the silent JSON fallback, and wire per-page\n ErrorMessage UI (RG-7) - moves into W4 and lands in the SAME\n slice as the final manifest.ts consumer's migration. This\n guarantees no mid-migration tab state where some pages still\n depend on a soft attach failure.\n\nExit:\n - Vitest coverage on the data-layer helpers.\n - In-browser cold-start measurement confirms RG-1 budgets.\n - `Query.tsx` no longer depends on `results_schema.json`.\n - `duckdbQueries.ts` + `duckdbSchema.ts` land with typed tests.\n - The shared DuckDB data layer is ready for page-by-page migration\n in W4.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build the shared frontend DuckDB access layer and cut `Query.tsx` off from `results_schema.json`; defer `db.ts` attach-failure hardening until W4", "wid": "w3"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w3"], "notes": "Every user-visible number on these surfaces must originate from DuckDB\nqueries or DuckDB-backed helpers. Benchmark landing pages remain the\ndefault graph surface. `/results/query` remains secondary in product\npositioning, but the same DuckDB contract must back both flows. Query\nwas fully migrated in W3 and is not reopened here.\n\nSuggested migration order (one page per slice, commit per page):\nHome -> PlatformIndex -> BenchmarkIndex -> ResultDetail -> Compare.\n\nPer-slice rule: when a page migrates to DuckDB, the SAME commit\n (a) deletes that page's imports from `manifest.ts`,\n (b) deletes the metric-JSON emitter in the pipeline that backed\n that page (e.g. `meta_leaderboard.json` with Home,\n `benchmarks/*.json.gz` with BenchmarkIndex, `details/*.json`\n with ResultDetail, `compare/*.json` + `short_ids.json` with\n Compare),\n (c) deletes the TS reduction module(s) whose last caller\n disappeared (typically `queryReduction.ts`, `chartMath.ts`,\n `ranking.ts`, `compliance.ts` on various slices), since the\n canonical Python reference computation was ported into\n `benchbox/core/explorer_pipeline/` in W2.\n\nPer-page rule: never recompute a user-visible metric in TypeScript\n(display_ms, geomean_ms, rank, speedup, compliance class, speedup\nratios, fastest/slowest spreads). Read materialized columns from the\ncanonical tables. Presentation formatting (locale, unit suffixes,\ntruncation) remains TS-side.\n\nDelete `results-explorer/src/manifest.ts` in the final slice once the\nlast consumer is gone. In that same final slice, land the `db.ts`\nattach-failure hardening deferred from W3:\n - Reject `getDb()` on attach failure instead of swallowing the\n error.\n - Delete the silent JSON fallback at `db.ts:86-89` and the stale\n `db.ts:1-17` header comment's \"Phase 2 / fallback\" framing.\n - Wire the per-page ErrorMessage UI required by RG-7.\nThis co-scheduling is intentional: the fallback must not outlive the\nlast page that relied on it, and must not be removed before then.\n\nPrune transport-only JSON artifact interfaces from\n`results-explorer/src/types.ts` once their last caller disappears. Do\nnot delete render-layer domain types solely because the transport\nchanged; keep, move, or rename them only when DuckDB row models make\nthe old shape redundant.\n\nExit: chart-parity and accessibility tests green on every migrated\npage; no page imports `manifest.ts`; `db.ts` rejects on attach\nfailure with no JSON fallback; TS reduction libs whose callers are\ngone are deleted (not just dead-coded); any removed TS types are\nconfirmed unused rather than deleted by naming policy alone.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate remaining pages to DuckDB, delete each bridge artifact in the same slice, and land `db.ts` hardening with the last manifest consumer", "wid": "w4"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w4"], "notes": "Starter queries must cover common analyst questions across results,\nper-query timings, cohort comparisons, trust/tuning breakdowns, and\ndetail-level drill-down. Keep the SQL surface read-only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Expand the query workbench into the canonical analyst surface with starter queries, cohort-aware templates, and downloads against the same DuckDB contract", "wid": "w5"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w4", "w5"], "notes": "Required assertions:\n - home stats and leaderboard rows == DuckDB query results\n - benchmark matrix cell == compare cell == detail timing == SQL query\n result where applicable\n - no runtime metric fetches remain for `benchmarks/*.json.gz`,\n `details/*.json`, `compare/*.json`, `meta_leaderboard.json`,\n `short_ids.json`, or `results_schema.json`\n - read-only DuckDB enforcement still blocks DDL/DML from the\n workbench\n\nExit-gate inventory (all runnable locally - CI wiring is out of scope\nper `do_not_modify: [\".github/workflows/\"]` and is filed as a\nfollow-up TODO):\n\n - G-1 Schema contract:\n `tests/unit/core/explorer_pipeline/test_duckdb_browser_contract.py`\n asserts all ten canonical tables exist with expected columns.\n - G-2 Metric parity (driven by `visible_metrics.yaml` from W1):\n for every metric row in the fixture, the test dispatches on the\n `kind` field and compares the DuckDB read-path value against the\n contract for that kind:\n - `raw_copy`: assert `DuckDB column == committed bundle field`\n verbatim for a sampled corpus.\n - `derived`: assert `DuckDB-backed read path value ==\n canonical Python reference computation` for the same\n inputs, within the `tolerance` declared per metric. The\n canonical Python reference computation lives under\n `benchbox/core/explorer_pipeline/` (or is imported read-only\n from `benchbox/core/results/`); it is NOT the CLI's\n pre-calculated JSON, NOT the current TS reduction libraries,\n and NOT any bridge artifact. Final-rendered-value parity is\n the contract.\n - G-3 Cross-surface parity:\n `results-explorer/src/__tests__/duckdbMetricParity.test.ts` asserts\n Home stats == `meta_leaderboard`, benchmark matrix cell ==\n compare cell == detail timing == direct SQL result.\n - G-4 Chart parity (React): extend existing parity harness to feed\n DuckDB-shaped rows through every component; DOM must match the\n ASCII baseline.\n - G-5 Performance budgets (local): Node + `@duckdb/duckdb-wasm`\n harness loads a 100x synthetic fixture; asserts init p95 <3500ms,\n per-query p95 <250ms (or RG-6 budget), DB file size \u2264 RG-3 ceiling.\n - G-6 Accessibility: existing axe-core suite re-run against\n DuckDB-fed pages.\n - G-7 Headless E2E: Playwright (added under this TODO as new\n tooling; listed in `files_affected.added`) loads `/results/`,\n waits for DB ready, navigates, asserts a DuckDB-sourced row\n renders, opens the workbench, runs `SELECT COUNT(*) FROM\n results`, and asserts attach-failure UX by loading a broken\n fixture.\n - G-8 Legacy-artifact guard: post-build assertion fails if any of\n `details/`, `compare/`, `meta_leaderboard.json`, `short_ids.json`,\n `results_schema.json`, or `benchmarks/*.gz` appear under\n `public/data/`. Bundles handling: `public/data/bundles/` MAY\n remain as the raw-bundle download affordance, but a second\n assertion greps the explorer source tree and fails if any\n `.tsx`/`.ts` file under `results-explorer/src/pages/` or\n `results-explorer/src/components/` fetches from\n `/results/data/bundles/` - bundles are download-only and must\n never back a rendered metric. If `manifest.json` survives, a\n dedicated contract test loads it and asserts its top-level keys\n are a subset of an explicit whitelist recorded in the schema\n decision record (W1 deliverable 2), such as `{routes,\n navigation, build_meta}`. Any metric-looking key\n (`benchmark_summary`, `result_count`, `platform_version`,\n `power_score`, `geomean_ms`, etc.) appearing in that file fails\n the gate.\n - G-9 Read-only fuzz: Vitest runs ~50 adversarial SQL strings\n (CREATE TABLE, ATTACH, COPY TO, INSTALL, LOAD, HTTPFS) against the\n READ_ONLY-attached DB; all fail harmlessly. The test is agnostic\n about HOW each fails - whether because the extension is compiled\n out of the chosen DuckDB-WASM bundle or because READ_ONLY attach\n rejects it at runtime. Open question #6 is resolved by whichever\n mechanism satisfies G-9 in the bundle chosen under RG-4.\n - G-10 Short-ID uniqueness: Python test rebuilds DB against a 100x\n corpus and asserts `SELECT short_id, count(*) FROM short_ids\n GROUP BY 1 HAVING count(*) > 1` is empty.\n - G-11 No-computed-metric views: contract test enumerates every\n view in `main` via `information_schema.views` /\n `duckdb_columns`, parses each view's SQL with\n `sqlglot.parse_one(..., read='duckdb')`, and asserts that every\n projected column in the outermost SELECT is either a bare\n `Column` reference, a `Column` aliased via `AS`, or a\n whitelisted passthrough (`CAST`, `COALESCE(col, literal)`).\n Views may NOT introduce arithmetic, aggregation, CASE\n expressions, `IIF`, window functions, or function calls beyond\n the whitelist - any such projection means a user-visible metric\n is being produced by the view rather than persisted in a base\n table and fails the gate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add exit gates proving no metric-bearing JSON runtime reads remain and all explorer surfaces agree on DuckDB values", "wid": "w6"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w6"], "notes": "Remove benchmark/detail/compare/meta-leaderboard JSON outputs once the\nDuckDB-backed contract passes parity tests. If `manifest.json` or any\nroute helper remains, strip it to non-metric navigation or identity\ndata only and document that rule explicitly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Delete metric-bearing JSON artifacts and stale compatibility framing once DuckDB-backed parity is proven", "wid": "w7"}, {"evidence": null, "item_id": "explorer-canonical-browser-duckdb-read-model", "needs": ["w2"], "notes": "Surfaced during W2 contract tests: the same platform can legitimately\nappear in a cohort more than once (different `tuning_mode`, different\n`trust_label`, different run). The originating schema pinned\n`cohort_metadata` PRIMARY KEY to `(cohort_key, platform_id)` and the\nW2 builder silently deduplicated by keeping only the best-ranked row,\ndropping the other publishable variant from the DuckDB store.\n\nThat violates the project principle: ALL results we trust enough to\npublish must be available in the DuckDB published-results dataset so\nusers can compare across tuning levels AND trust levels from the same\nstore. The table is the per-cohort published-result set; the UI\ndecides any display-time grouping.\n\nEmpirical evidence recorded with this TODO (`/tmp/reproduce_cohort_duplicates.py`-style\ntwo-bundle reproducer, same platform `duckdb`, tuning_mode\n`tuned` vs `notuning`, both in `tpch-sf0.1-power`):\n - `meta_leaderboard.json` `cohorts[0].platforms` kept BOTH variants\n (tuned rank 1, notuning rank 2) - JSON artifact is correct.\n - DuckDB `cohort_metadata` after W2 dedup kept only the tuned row -\n silently discards the notuning variant.\n - `platform_agg` at `pipeline.py:240` overwrites the better rank\n (rank 1) with the worse rank (rank 2) via last-write-wins, so the\n JSON artifact `platforms[]` entry AND the DuckDB `meta_leaderboard`\n row BOTH report `avg_rank = 2.0` when the correct answer is 1.0.\n This is a pre-existing correctness bug in the meta-leaderboard\n that already affects today's Home surface, independent of the PK\n decision.\n\nImplementation (Option B + bug fix):\n\nSchema (`docs/development/browser-duckdb-schema.sql`):\n - `cohort_metadata` PRIMARY KEY changes from\n `(cohort_key, platform_id)` to `(cohort_key, result_id)`.\n - Add columns: `tuning_mode VARCHAR`, `trust_label VARCHAR NOT NULL`,\n `short_id VARCHAR NOT NULL`. These give Home the data it needs to\n label, group, or collapse variants without a secondary join.\n - Keep `idx_cohort_metadata_key (cohort_key)`; add\n `idx_cohort_metadata_platform (cohort_key, platform_id)` so\n platform-grouped queries stay cheap.\n\nPython (`benchbox/core/explorer_pipeline/duckdb_builder.py`):\n - `_create_schema` mirrors the new DDL (PK + 3 columns + index).\n - `_populate_cohort_metadata` deletes the `best` dedup dict and\n `_RANK_IDX` logic; inserts every row from\n `meta[\"cohorts\"][*][\"platforms\"]`. Widens the insert tuple to\n include `tuning_mode`, `trust_label`, `short_id`.\n\nPython (`benchbox/core/explorer_pipeline/pipeline.py`):\n - `_build_meta_leaderboard` emits `tuning_mode`, `trust_label`,\n `short_id` on each entry in `cohort_platforms` so the builder has\n what it needs without re-joining. Thread `full_to_short` into\n the function, or resolve in the builder by looking up\n `short_id_map`.\n - Fix `platform_agg` last-write-wins at `pipeline.py:240`: when the\n same `platform_id` ranks in the same `cohort_key` more than once,\n KEEP THE BEST RANK (lowest integer). Correct for the Home\n meta-leaderboard semantics (one row per platform across cohorts\n - best variant represents the platform). Equivalent minimal\n edit: replace the blind dict assignment with a guarded update,\n or rely on `setdefault` since `sorted_rows` is already best-first.\n\nRegistry (`_project/planning/visible_metrics.yaml`):\n - Update `cohort_metadata` `rank` / `metric_value` / `speedup_vs_best`\n notes to clarify they are PER RESULT_ID, not per platform_id.\n - Add new rows: `cohort_metadata.tuning_mode`,\n `cohort_metadata.trust_label`, `cohort_metadata.short_id` with\n `kind: raw_copy` and appropriate `canonical_ref` pointers.\n\nTests:\n - Extend `test_distinct_tuning_modes_produce_distinct_rows` in\n `tests/unit/core/explorer_pipeline/test_benchmark_summary.py` with\n a DuckDB assertion: after the pipeline run, the cohort contains\n BOTH `result_id`s in `cohort_metadata` with distinct ranks.\n - Add a regression test in\n `tests/unit/core/explorer_pipeline/test_duckdb_browser_contract.py`\n (or a peer file) against the two-bundle same-platform-same-cohort\n fixture: assert `meta_leaderboard.avg_rank` equals the BEST rank\n (1.0, not 2.0), and `n_cohorts` equals the number of distinct\n cohort_keys (not the number of cohort-variants).\n - Add a `visible_metrics` drift test asserting the\n `cohort_metadata` PRIMARY KEY columns declared in the schema\n YAML match the frozen `browser-duckdb-schema.sql`.\n\nExit:\n - `cohort_metadata` PK is `(cohort_key, result_id)` in schema,\n builder, and visible_metrics.\n - Running the pipeline on a two-variant corpus lands BOTH rows in\n `cohort_metadata`; no silent deduplication anywhere in the\n pipeline.\n - `_build_meta_leaderboard` keeps the BEST rank per\n `(platform_id, cohort_key)`; regression test pinning\n `avg_rank == 1.0` on the two-variant reproducer is green.\n - All 217 pre-existing explorer pipeline tests remain green; the\n two new tests are green.\n - W3 does not start until this item is done, because the\n `duckdbQueries.getCohort` helper it builds must match the final\n key shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Preserve all publishable result variants in `cohort_metadata` and fix the `_build_meta_leaderboard` `platform_agg` last-write-wins bug before the frontend migration", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "mix cohorts in trend charts", "instead": "split by cohort.", "why": "benchmark/scale geomeans are not one metric"}, {"dont": "present every chart as an equal chip", "instead": "group charts by question.", "why": "users need analytical hierarchy"}, {"dont": "rely on color alone in heatmaps", "instead": "keep visible values and accessible names.", "why": "color perception and print/screenshot contexts vary"}, {"dont": "use linear latency bars when one slow outlier collapses the rest of the field", "instead": "use log scale or speedup encoding with clear labels.", "why": "users cannot compare fast platforms"}], "approach": "Start by making chart intent explicit in registry metadata, then update\nChartPanel navigation. Treat log scale and speedup as distinct encodings:\nlog scale for magnitude across orders of magnitude, speedup when the user is\nasking relative winner/loser questions. Test chart math separately from Preact\nrendering so scale regressions are easy to catch.\n", "blocked_reason": null, "category": "Data Visualization", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": ["explorer-critical-results-ux-defects"], "description": "Fix visualization choices that currently reduce trust or legibility in the\nResults Explorer. Database benchmark latency often spans orders of magnitude,\nso linear bar charts can compress meaningful differences and cause labels to\noverlap. Chart navigation also exposes too many tiny chart chips without an\nanalytical hierarchy. The goal is to make charts answer clear questions:\noverview, per-query behavior, distribution, cost, trend, and rank.", "id": "explorer-chart-correctness-and-navigation", "preserves": ["Benchmark ranking order continues to come from pipeline-emitted primary_metric/primary_order, not frontend guesses.", "Chart registry remains the single source for chart availability and chart metadata.", "Charts remain readable when only one platform/result is present.", "Heatmaps remain keyboard navigable and retain existing role/grid accessibility behavior."], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/index.css"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartMath.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartRegistry.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryLabels.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}], "state": "done", "title": "Explorer: chart correctness, log-scale latency encoding, and chart navigation", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Frontend typecheck and chart tests pass.", "expected": "All chart registry, chart math, heatmap, and ChartPanel tests pass.", "item_id": "explorer-chart-correctness-and-navigation", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run build", "description": "Production build succeeds.", "expected": "Vite build completes without chart-related warnings or errors.", "item_id": "explorer-chart-correctness-and-navigation", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": [], "notes": "In `results-explorer/src/lib/chartRegistry.ts` and ChartPanel, replace the\nflat 10+ chip list with grouped tabs or segmented controls:\n - Overview\n - Per-query\n - Distribution\n - Cost\n - Trend\n - Rank\nEach group should show at most the relevant chart controls for the current\npage/data shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit chart registry and group charts by analytical question", "wid": "w1"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w1"], "notes": "Update PowerBar/ChartPanel latency views so linear bars do not compress\nfaster engines against a slow outlier. Use log scale for latency magnitude\nor speedup-vs-best where comparison is the point. Axis labels must disclose\nlog scale when used.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Use log-scale or ratio-based encoding for latency bars where values span orders of magnitude", "wid": "w2"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w2"], "notes": "The audited SSB Performance Bar showed labels crowding near the origin.\nImplement collision-aware label placement: outside labels when space is\ninsufficient, minimum label gutter, and tooltip fallback for narrow bars.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix label overlap in SSB/star_schema performance bar charts", "wid": "w3"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w1"], "notes": "Carry forward the critical defect fix: trend charts must not mix\nbenchmark/scale/phase cohorts. Render cohort-specific small multiples or\nrequire a selected cohort before showing the time series.\n\nAlready satisfied by `explorer-critical-results-ux-defects` w3/w6 on the\ncurrent base: `buildTrendCohorts` groups by benchmark, scale factor,\nphase, and primary metric, and `PlatformIndex.test.tsx` asserts TPC-H and\nSSB trend points do not cross cohort sections.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Split platform trend charts by cohort or metric", "wid": "w4"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": [], "notes": "Query IDs should sort Q1, Q2, ..., Q10 rather than lexicographic 1, 10,\n11, ..., 2. Labels should preserve benchmark-specific prefixes where\npresent (`Q1`, `q01`, `query_001`) rather than bare ambiguous numerals.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add natural query sorting and explicit query labels in heatmaps and tables", "wid": "w5"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w5"], "notes": "MetaLeaderboard and QueryHeatmap should have text/value labels sufficient\nfor interpretation without relying only on color. Add or preserve\ncolor-blind-safe sequential palettes and reduced-color rendering where\nsupported by CSS/media queries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add high-contrast and reduced-color paths for heatmaps", "wid": "w6"}, {"evidence": null, "item_id": "explorer-chart-correctness-and-navigation", "needs": ["w2", "w3", "w4", "w5", "w6"], "notes": "Extend chartMath and component tests for log-scale mapping, label\ncollision behavior, cohort-split trend data, natural query sort, and\nheatmap accessible names/values.\n\nExisting W2-W6 commits cover log-scale chart math, ChartPanel log-scale\nrendering, collision-aware value label placement, cohort-split trend\nrendering, natural query sort, and MetaLeaderboard heatmap values. W7\ncloses the remaining exact accessible-name assertion for QueryHeatmap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add chart math and rendering tests for scale, sorting, and labels", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change computeECDFPoints for w2", "instead": "(not specified)", "why": "the math is correct; only the SVG path rendering changes"}, {"dont": "gate Histogram/Box/CDF tabs on platform count alone", "instead": "(not specified)", "why": "gate on actual non-null timing values"}], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-chart-coverage-review-followups", "id": 314, "reason": "Interactivity is out of scope for this follow-up item", "resolution": "promoted", "resolved_item": "interactive-cdfchart-tooltip-showing-exact-percentile-at-cur", "resolved_reason": null, "summary": "Interactive CDFChart tooltip showing exact percentile at cursor position"}], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-close-chart-type-coverage-gaps. The two Required issues (PowerBar\ncolor index, PercentileLadder caller color index) were fixed immediately\npost-review. This item tracks the remaining four findings.", "id": "explorer-chart-coverage-review-followups", "preserves": ["BenchmarkIndex existing tab gating logic for percentiles/power/phases/cost tabs", "CDFChart parity test (computeECDFPoints) must still pass - only the SVG path rendering changes", "RankTable ordinal display for n=1,2,3 must still return 1st/2nd/3rd"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from chart-coverage code review", "verifications": [{"command": "cd results-explorer && npm test -- RankTable", "description": "ordinal handles 21st/22nd/23rd correctly", "expected": "tests pass including ordinal edge cases", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm test -- NewCharts", "description": "CDFChart unit test still passes after step-render change", "expected": "CDFChart svg test passes", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm test -- BenchmarkIndex", "description": "BenchmarkIndex test passes with updated tab gating", "expected": "all BenchmarkIndex tests pass", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck && npm test", "description": "Full test suite green", "expected": "all green", "item_id": "explorer-chart-coverage-review-followups", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "RankTable.tsx:21-25 - ordinal() only special-cases 1/2/3; everything\nelse falls through to `${n}th`. Renders \"21th\", \"22nd\"\u2192\"22th\" for\nn >= 21 that end in 1/2/3 (except 11/12/13 which are correctly \"th\").\nUnlikely with current platform counts but technically wrong.\n\nFix: standard ordinal suffix rule -\n const suffixes = [\"th\",\"st\",\"nd\",\"rd\"];\n const v = n % 100;\n return `${n}${suffixes[(v - 20) % 10] ?? suffixes[v] ?? suffixes[0]}`;\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix ordinal() in RankTable to handle n >= 21 correctly", "wid": "w1"}, {"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "CDFChart.tsx:103-115 uses `L` (lineto) to connect CDF points, producing\nsmooth diagonal lines. A true ECDF is a step function: horizontal\nsegment to the x-value, then vertical jump to the next cumulative\nfraction. The Python reference (textcharts.cdf_chart) renders steps.\nVisual divergence is minor on dense data (22+ queries) but visible on\nsparse distributions.\n\nFix: render H\u2192V steps instead of diagonal L commands. No math change\nneeded - computeECDFPoints output is already correct.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Render CDFChart as ECDF steps, not smooth lines", "wid": "w2"}, {"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "BenchmarkIndex.tsx:383-386 - The Histogram, Box Plot, and CDF tabs are\nunconditionally shown (hardcoded true). When all platform timings are\nnull or empty, clicking these tabs shows a blank section because each\nchart guards internally and returns null. The Percentiles, Power, Phases,\nand Cost tabs are already data-gated.\n\nFix: add a `hasTimings` guard:\n const hasTimings = s.platforms.some(\n (p) => Object.values(p.timings).some((v) => v !== null && v > 0)\n );\nThen use hasTimings for the histogram/distribution/cdf entries.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Data-gate Histogram / Box Plot / CDF tabs in BenchmarkIndex", "wid": "w3"}, {"evidence": null, "item_id": "explorer-chart-coverage-review-followups", "needs": [], "notes": "chartRegistry.meta.test.ts:19-36 - CANONICAL_IDS is a third hardcoded\ncopy of the chart type list (alongside chart_types.py and chartRegistry.ts).\nThe test detects TS-side drift but NOT Python-side drift: if someone adds\na 17th type to chart_types.py without updating the test's list, the test\npasses.\n\nFix options:\n (a) Parse chart_types.py at test time using Node fs.readFileSync +\n regex to extract chart IDs from _CHART_SPECS.\n (b) Emit a machine-readable chart-ids.json from the Python pipeline\n as a build artifact and compare against that.\n (c) Accept the current triple-write but add a comment warning that\n chart_types.py is the authority and all three must be kept in sync.\n\nOption (c) is the lowest friction; (a) is the most robust. Avoid (b)\nsince it couples the static build to the test suite.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace hardcoded CANONICAL_IDS in meta-test with dynamic chart_types.py parse", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "make this ADR conditional on Phase 8/9 migration work", "instead": "(not specified)", "why": "the explorer surface should be decided independently of the remaining migration phases (which TODO `migration-phase-api-surface-gate` handles separately)."}, {"dont": "relocate any code in this TODO", "instead": "(not specified)", "why": "moving code without the locked decision creates churn and re-litigates the choice in the migration PR. Defer all moves to `explorer-cli-surface-migration`."}], "approach": "Read-mostly. Two grep-based inventories (w1, w2) inform an ADR (w3).\nUse `docs/development/adr/TEMPLATE-results-data-extraction.md` as the\nstructural template if a TEMPLATE-adr.md is not present.\n\nDo not pre-commit to one of the four options before the inventories are\ncomplete; the migration-cost numbers are load-bearing.\n", "blocked_reason": null, "category": "Architecture", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-adr", "id": 155, "reason": "Owned by TODO `explorer-cli-surface-migration` (depends on this TODO).", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Verified: benchbox/cli/commands has no explorer.py, and _project/scripts/explorer_publish.py + explorer_pipeline/ exist -- the ADR-recommended code move already happened.", "summary": "Actually move any code per the ADR's recommendation."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-adr", "id": 156, "reason": "Owned by TODO `migration-phase-api-surface-gate`; this ADR is scoped to the explorer surface only.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Owning item migration-phase-api-surface-gate is already DONE; the broader CLI user/maintainer-split revisit is resolved or moot there.", "summary": "Revisit the user-vs-maintainer split for other parts of the `benchbox` CLI."}], "deps": [], "description": "`benchbox explorer build` (and the hidden `benchbox explorer build-contract`)\npublish a static read-model for the `results-explorer` React app. They are\nnot benchmark-runner operations, yet they sit on the user-facing `benchbox`\nCLI surface and ship under `benchbox/cli/commands/explorer.py` alongside a\n7-file core package at `benchbox/core/explorer_pipeline/`.\n\nProvenance audit (this session):\n - Both commands were added in commit `f9d08d38b` (\"Migration: populate\n develop branch from private working tree (Phase 4 w4)\", 2026-04-27).\n That was a bulk migration squash, not a discrete reviewed PR.\n - The breadcrumb at `.github/workflows/docs.yml:105` attributes them to\n PR #46, but #46 is `docs(contributing): name the version JSON key +\n non-uv fallback` \u2014 a contributing-results.md docs PR, unrelated to the\n CLI surface. The in-tree provenance trail lies.\n - No `_project/specs/` document scopes the explorer CLI surface; Phase-4\n migration TODO only lists the file paths as \"to migrate\", not as a\n command surface to design.\n\nThis TODO is the decision step. It does not move any code. It produces an\nADR that locks the destination so downstream migration (TODO\n`explorer-cli-surface-migration`) has a concrete target.\n\nThe four candidate destinations are:\n\n 1. **hide-only**: `hidden=True` on `explorer_group`. Cheapest. Does not\n solve intermingling \u2014 code stays in `benchbox/`, import surface stays\n public, only `--help` discoverability changes. Likely rejected.\n 2. **new sibling package** `benchbox_explorer/` with its own\n `console_script` (e.g. `benchbox-explorer-publish`). Separate pip\n artifact; clean conceptual boundary; external CI/scripts can still\n script against a stable binary.\n 3. **`_project/scripts/explorer/`** invoked via `uv run -- python -m\n _project.scripts.explorer.publish`. In-tree maintainer tool; not\n installed; tightest \"this is not a public surface\" signal.\n 4. **co-locate in `results-explorer/`** as a Python sub-package called\n from `results-explorer/scripts/`. Tightest coupling to the only\n consumer. Mixes Python and Node trees.\n\nThe ADR must also resolve three gating sub-questions:\n\n a. Does `benchbox/core/explorer_pipeline/` ALSO relocate, or only the\n CLI wrapper? Argument for \"yes\": same intermingling concern; the\n package serves only the explorer site. Argument for \"no\": existing\n tests under `tests/unit/core/explorer_pipeline/` and import sites\n like `benchbox/cli/help.py` would need follow-up. Decision here\n shapes the migration cost.\n\n b. Backwards-compat alias for one release: keep\n `benchbox explorer build` as a thin deprecation shim that prints a\n `DeprecationWarning` and forwards to the new entry point, then drop\n it in the next minor release? Or hard-cut and update all known\n callers in the same PR? BenchBox is v0.2.x pre-1.0, so a hard cut\n is defensible; the ADR should still record the call.\n\n c. End-user error path when a stale snapshot is detected: today,\n `results-explorer/src/db.ts:196` tells the user to run\n `benchbox explorer build`. After the move, the message must point\n at the maintainer-only entry. For non-maintainers (e.g., someone\n running `npm run dev` against a downloaded snapshot), the message\n should redirect to \"refresh the snapshot from the published site\"\n or similar, not to a CLI they shouldn't be running.", "id": "explorer-cli-surface-adr", "preserves": ["Existing CI builds, browser-fixture generation, and UAT explorer-smoke must continue to work unchanged; this TODO does not touch them.", "No code is moved or renamed in this TODO. ADR-only.", "`benchbox --help` output is unchanged in this TODO."], "prior_art": [{"concept": "_project/scripts/ \u2014 existing in-tree maintainer scripts directory (validate_todo.py, generate_indexes.py, etc.); candidate destination for option 3, reuse layout if chosen.", "decision": "reuse", "path": "_project/scripts/"}, {"concept": "benchbox/cli/commands/__init__.py:22 \u2014 registration site for `explorer_group`; reuse the registration pattern (or its absence) for whatever replaces it.", "decision": "reuse", "path": "benchbox/cli/commands/__init__.py:22"}, {"concept": "benchbox/cli/commands/explorer.py \u2014 current Click subgroup; supersede or relocate per ADR.", "decision": "supersede", "path": "benchbox/cli/commands/explorer.py"}, {"concept": "benchbox/core/explorer_pipeline/ \u2014 7-file core package (contract.py, duckdb_builder.py, models.py, pipeline.py, ranking.py, transformer.py, compare_math.py); reuse intact, only the housing changes.", "decision": "reuse", "path": "benchbox/core/explorer_pipeline/"}, {"concept": "docs/development/adr/ \u2014 existing ADR location with TEMPLATE-results-data-extraction.md as the structural reference; reuse format.", "decision": "reuse", "path": "docs/development/adr/"}, {"concept": "results-explorer/scripts/explorer-build-contract.mjs \u2014 Node-side consumer that calls `benchbox explorer build-contract` to fetch the output contract; reference for how an external script binds to the entry point.", "decision": "reuse", "path": "results-explorer/scripts/explorer-build-contract.mjs"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "docs/development/adr/adr-explorer-cli-surface.md"}], "state": "done", "title": "ADR: pick a destination for the explorer publish toolchain", "verifications": [{"command": "test -f docs/development/adr/adr-explorer-cli-surface.md && wc -l docs/development/adr/adr-explorer-cli-surface.md", "description": "ADR file exists and validates as markdown.", "expected": "non-zero line count", "item_id": "explorer-cli-surface-adr", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -cE '^## (Decision|Recommendation|Sub-question)' docs/development/adr/adr-explorer-cli-surface.md", "description": "ADR records a single recommended option and answers all three sub-questions (a, b, c).", "expected": ">= 4", "item_id": "explorer-cli-surface-adr", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -cE 'docs.yml|generate-browser-fixtures|explorer_pipeline' docs/development/adr/adr-explorer-cli-surface.md", "description": "Inventories from w1 and w2 are present in the ADR body.", "expected": ">= 3", "item_id": "explorer-cli-surface-adr", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": [], "notes": "Before writing the ADR proper, re-run the provenance probes that this\nTODO's description cites and commit the resulting facts as a\n\"Provenance Facts\" section in the ADR. This durability-pins the\nevidence so future readers do not have to re-derive it.\n\nCommands:\n - `git show --stat f9d08d38b -- benchbox/cli/commands/explorer.py`\n \u2192 expect ~221-line file addition under that path.\n - `git log -1 --format=%s%n%n%b f9d08d38b`\n \u2192 expect title \"Migration: populate develop branch from private\n working tree (Phase 4 w4)\".\n - `gh pr view 46 --json title,body | head -2`\n \u2192 expect title to be `docs(contributing): ...` (NOT the explorer\n CLI), confirming the breadcrumb at `.github/workflows/docs.yml:105`\n is mis-attributed.\n - `git log --all --diff-filter=A --oneline -- benchbox/cli/commands/explorer.py`\n \u2192 expect only `f9d08d38b` and `d28dbe66f` (the consistency-fix PR).\n - `git blame .github/workflows/docs.yml -L 100,108`\n \u2192 expect `0182c9556c` (v0.2.1 release commit), confirming the\n misattributed Note line.\n\nOutput: a 5-10 line \"Provenance Facts\" block at the top of the ADR\nrecording: introduction commit, PR (none / migration squash),\nbreadcrumb misattribution target, and the command set above so a\nfuture reader can re-run verbatim.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate provenance evidence and pin it in the ADR", "wid": "w0"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w0"], "notes": "Grep targets (run from repo root):\n - `git grep -n \"benchbox explorer build\\|benchbox explorer build-contract\"`\n - `git grep -nE \"explorer_group|from benchbox\\.cli\\.commands\\.explorer\"`\n - `git grep -nE \"from benchbox\\.core\\.explorer_pipeline\"`\n\nKnown seed list to confirm and extend:\n - `.github/workflows/docs.yml:103`\n - `results-explorer/scripts/generate-browser-fixtures.mjs:489`\n - `results-explorer/scripts/explorer-build-contract.mjs:17`\n - `tests/uat/.../explorer_smoke.py` (UAT W7 phase)\n - `Makefile` `uat-explorer-smoke` target (line ~1612)\n - `docs/development/browser-test-architecture.md:35,119,130`\n - `docs/operations/results-phase-2-runbook.md:109`\n - `results-explorer/src/db.ts:196` (frontend error remediation text)\n - audit docs under `_project/audits/results-explorer-*.md`\n - tests under `tests/unit/cli/test_explorer_build_contract.py`\n\nOutput: a single table in the ADR with each call site, its category\n(CI / fixture / UAT / docs / runtime / test), and migration cost\n(one-liner / multi-line / non-trivial).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory every call site of the current CLI surface", "wid": "w1"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w1"], "notes": "The decision on sub-question (a) (does the pipeline also move?) turns\non how leaky the package's public import surface is.\n\nGrep targets:\n - `git grep -nE \"from benchbox\\.core\\.explorer_pipeline|import benchbox\\.core\\.explorer_pipeline\"` excluding the package itself.\n\nOutput: list of every import site outside `benchbox/core/explorer_pipeline/`,\ngrouped by importer (CLI / test / other).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Inventory imports of `benchbox.core.explorer_pipeline.*` from outside the package", "wid": "w2"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w1", "w2"], "notes": "Path: `docs/development/adr/adr-explorer-cli-surface.md`.\n\nStructure:\n - Context (the provenance audit summary, citing this TODO).\n - Forces: user-vs-maintainer surface separation; CI dependency stability;\n external script consumers; migration cost; reversibility; consistency\n with other maintainer tools in `_project/scripts/`.\n - Options 1-4 with explicit pros/cons and migration-cost estimate from\n the w1/w2 inventories.\n - Decision matrix scoring each option on the forces.\n - Recommendation with rationale.\n - Resolutions to sub-questions (a), (b), (c) from the description.\n - Consequences: what TODO `explorer-cli-surface-migration` will do.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Draft the ADR with a 4-option decision matrix", "wid": "w3"}, {"evidence": null, "item_id": "explorer-cli-surface-adr", "needs": ["w3"], "notes": "Open a PR with only the ADR (no code moves). On merge, this TODO is\nCompleted and `explorer-cli-surface-migration` is unblocked.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Land the ADR on develop", "wid": "w4"}], "worktree": "explorer-cli-surface-adr"} +{"anti_patterns": [{"dont": "change the snapshot's output contract or column schema", "instead": "(not specified)", "why": "that work is owned by TODO `explorer-read-model-schema-versioning`."}, {"dont": "fix the stale-snapshot dev-DX wall in this PR", "instead": "(not specified)", "why": "that's TODO `explorer-dev-snapshot-dx`. Mixing them makes review harder and entangles the rename with behavioral change."}, {"dont": "keep `benchbox explorer` as a permanent alias unless the ADR explicitly chose option (b) with a deprecation shim. A bare `pass`-through alias defeats the whole point.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "update historical audit docs that record commands actually run at the time. Only the active runbook and developer docs need updates; history can read 'we ran benchbox explorer build on 2026-05-10'.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Strictly a relocation + caller-update PR. Read the ADR's \"Consequences\"\nlist and execute it line-by-line. Do not introduce new behavior, new\nflags, or new pipeline logic. Schema-versioning and dev-DX are explicit\nout-of-scope items handled by sibling TODOs.\n\nSequencing: relocate (w2, w3) before updating callers (w4) so the\ncallers point at the new live location, not a future path. Add the\npinning test (w5) last so it asserts the steady state.\n", "blocked_reason": null, "category": "Refactoring", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-migration", "id": 157, "reason": "Owned by TODO `explorer-dev-snapshot-dx`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Owning TODO explorer-dev-snapshot-dx is completed (_project/DONE/main/active/explorer-dev-snapshot-dx.yaml).", "summary": "Pre-dev rebuild hook or in-tree dev snapshot."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-migration", "id": 158, "reason": "Owned by TODO `explorer-read-model-schema-versioning`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: explorer-read-model-schema-versioning.yaml is already in _project/DONE \u2014 this work is complete.", "summary": "Schema versioning for the read-model contract."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-cli-surface-migration", "id": 159, "reason": "Owned by TODO `migration-phase-api-surface-gate`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Owning TODO migration-phase-api-surface-gate is Completed; the Phase 8/9 API-surface gate already executed.", "summary": "Phase 8/9 migration API-surface gate."}], "deps": ["explorer-cli-surface-adr"], "description": "Execute the decision locked by TODO `explorer-cli-surface-adr`:\n - Remove `explorer` from the user-facing `benchbox` CLI surface.\n - Relocate `benchbox/cli/commands/explorer.py` (and, per ADR sub-question\n a, optionally `benchbox/core/explorer_pipeline/`) to the destination\n the ADR chose.\n - Update every call site: CI workflow, browser-fixture generator, UAT\n explorer-smoke phase, runbook, developer docs, frontend remediation\n string in `results-explorer/src/db.ts:196`, and audit cross-refs.\n - Delete the incorrect provenance breadcrumb at\n `.github/workflows/docs.yml:105` that misattributes the CLI surface\n to PR #46.\n - Add a regression test that pins the frontend remediation text to the\n live invocation of whatever entry point the ADR chose, so renames\n do not silently rot the user-facing error message.\n\nThis TODO does NOT redesign the read-model contract (TODO\n`explorer-read-model-schema-versioning`) or address the stale-snapshot\ndev-DX wall (TODO `explorer-dev-snapshot-dx`). Those are separable.", "id": "explorer-cli-surface-migration", "preserves": ["Browser-fixture generator continues to produce byte-identical fixture sets (verified by `verify-browser-fixtures.mjs`).", "CI explorer-build step in `.github/workflows/docs.yml` continues to produce a valid `results-explorer/public/data/results.duckdb` (only the invocation changes).", "Existing tests under `tests/unit/core/explorer_pipeline/` continue to pass (relocated if needed but not rewritten).", "Output contract (schema of `results.duckdb` and copied bundles) is unchanged by this TODO.", "UAT `make uat-explorer-smoke` continues to pass with the new invocation."], "prior_art": [{"concept": ".github/workflows/docs.yml:103-105 \u2014 CI invocation + the wrong PR #46 attribution comment; update invocation, delete the comment.", "decision": "reuse", "path": ".github/workflows/docs.yml:103-105"}, {"concept": "benchbox/cli/commands/explorer.py \u2014 current location; remove and relocate per ADR.", "decision": "reuse", "path": "benchbox/cli/commands/explorer.py"}, {"concept": "benchbox/core/explorer_pipeline/ \u2014 pipeline package; relocation gated by ADR sub-question (a).", "decision": "reuse", "path": "benchbox/core/explorer_pipeline/"}, {"concept": "docs/development/browser-test-architecture.md:35,119,130 \u2014 developer docs referencing the old command.", "decision": "reuse", "path": "docs/development/browser-test-architecture.md:35,119,130"}, {"concept": "docs/operations/results-phase-2-runbook.md:109 \u2014 runbook references the old command.", "decision": "reuse", "path": "docs/operations/results-phase-2-runbook.md:109"}, {"concept": "results-explorer/scripts/explorer-build-contract.mjs:17 \u2014 EXPECTED_COMMAND constant; update to the new invocation.", "decision": "reuse", "path": "results-explorer/scripts/explorer-build-contract.mjs:17"}, {"concept": "results-explorer/scripts/generate-browser-fixtures.mjs:489 \u2014 Node-side caller; update spawn invocation.", "decision": "reuse", "path": "results-explorer/scripts/generate-browser-fixtures.mjs:489"}, {"concept": "results-explorer/src/db.ts:196 \u2014 frontend remediation string; update to the new invocation and pin to it via test.", "decision": "reuse", "path": "results-explorer/src/db.ts:196"}, {"concept": "tests/uat/.../explorer_smoke.py + Makefile target `uat-explorer-smoke` (line ~1612) \u2014 UAT phase invocation.", "decision": "reuse", "path": "tests/uat/.../explorer_smoke.py"}, {"concept": "tests/unit/cli/test_explorer_build_contract.py \u2014 existing contract tests; if the CLI moves out of benchbox/cli/, this test file moves with it or is replaced.", "decision": "reuse", "path": "tests/unit/cli/test_explorer_build_contract.py"}], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": ".github/workflows/docs.yml"}, {"kind": "only_modify", "path_glob": ""}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/__init__.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/explorer.py"}, {"kind": "only_modify", "path_glob": "benchbox/cli/help.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/**"}, {"kind": "only_modify", "path_glob": "docs/development/browser-test-architecture.md"}, {"kind": "only_modify", "path_glob": "docs/operations/results-phase-2-runbook.md"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/explorer-build-contract.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/generate-browser-fixtures.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/**"}, {"kind": "only_modify", "path_glob": "tests/uat/**"}, {"kind": "only_modify", "path_glob": "tests/unit/cli/test_explorer_build_contract.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/**"}], "state": "done", "title": "Execute the explorer-CLI-surface ADR and remove explorer from benchbox --help", "verifications": [{"command": "uv run benchbox --help | grep -c '^\\s*explorer'", "description": "`explorer` is no longer in the public `benchbox` CLI surface.", "expected": "0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 1}, {"command": "make test-fast", "description": "Fast tests pass.", "expected": "exit 0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 2}, {"command": "make lint && make typecheck", "description": "Lint + typecheck pass.", "expected": "exit 0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 3}, {"command": " --data-dir results-data --output /tmp/explorer-migration-smoke && test -f /tmp/explorer-migration-smoke/results.duckdb", "description": "Explorer-build pipeline still produces the snapshot via the new invocation.", "expected": "exit 0, file exists", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- --run db-remediation-pin", "description": "Frontend Vitest pin test passes.", "expected": "test exits 0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -c 'added in PR #46' .github/workflows/docs.yml", "description": "The wrong PR-#46 attribution is deleted from docs.yml.", "expected": "0", "item_id": "explorer-cli-surface-migration", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": [], "notes": "Read `docs/development/adr/adr-explorer-cli-surface.md` end-to-end.\nIf the inventories of call sites and pipeline imports already exist\nthere, reuse them; do not redo. Build the migration checklist from\nthe ADR's \"Consequences\" section.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-confirm the ADR is merged and read its inventories", "wid": "w1"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w1"], "notes": "Move `benchbox/cli/commands/explorer.py` to the ADR's chosen\ndestination. Update:\n - `benchbox/cli/commands/__init__.py:22,62,106` \u2014 remove the\n `from .explorer import explorer_group` import and `explorer_group`\n registration + `__all__` entry.\n - `benchbox/cli/help.py` \u2014 any references to explorer commands in\n the help text.\nRun `uv run benchbox --help` and confirm `explorer` no longer appears.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Relocate the CLI entry point per ADR", "wid": "w2"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w1"], "notes": "Gated by ADR sub-question (a). If \"yes\": move the package + its tests\nunder `tests/unit/core/explorer_pipeline/` to the ADR's destination.\nUpdate imports in the relocated CLI entry from w2.\n\nIf \"no\": leave the package in place. Note this in the migration PR\ndescription so future readers understand the half-move was deliberate.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Relocate `benchbox/core/explorer_pipeline/` if the ADR requires it", "wid": "w3"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w2", "w3"], "notes": "Process each of the call sites from the ADR's w1 inventory. Specifically:\n - `.github/workflows/docs.yml:103` \u2014 replace invocation; DELETE the\n `# Note: requires 'benchbox explorer' CLI subcommand (added in PR #46)`\n line at :105 (the attribution is wrong; PR #46 is a docs PR).\n - `results-explorer/scripts/generate-browser-fixtures.mjs:489`\n - `results-explorer/scripts/explorer-build-contract.mjs:17`\n (EXPECTED_COMMAND constant)\n - `tests/uat/.../explorer_smoke.py` and `Makefile:uat-explorer-smoke`\n - `docs/development/browser-test-architecture.md`\n - `docs/operations/results-phase-2-runbook.md:109`\n - `results-explorer/src/db.ts:196` (remediation string)\n - audit docs that cite the old command \u2014 only the active runbook is\n load-bearing; historical audits can keep the old command since they\n record what was run at the time. Decide per-file in the PR review.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update every call site identified in the ADR inventory", "wid": "w4"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w4"], "notes": "The current failure mode: `results-explorer/src/db.ts:196` embeds the\nstring `'benchbox explorer build'`. If the CLI is renamed, nothing\nforces this string to update. Two options:\n\n (i) Import the canonical invocation from a shared TS module sourced\n from the same contract file the Node scripts already use\n (`results-explorer/scripts/explorer-build-contract.mjs`), so\n there is one source of truth.\n (ii) Add a test that asserts the current `db.ts` remediation string\n matches the exported `EXPECTED_COMMAND` from the contract\n script.\n\nPrefer (i). Implement in the Vitest suite under\n`results-explorer/src/lib/__tests__/`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a regression test pinning the frontend remediation string", "wid": "w5"}, {"evidence": null, "item_id": "explorer-cli-surface-migration", "needs": ["w5"], "notes": "Local verification:\n - `uv run benchbox --help | grep -c explorer` \u2192 0\n - `cd results-explorer && npm run lint && npm test`\n - `make lint && make typecheck`\n - `make test-fast`\n - Manual: run the relocated entry point against `results-data/` and\n confirm `results.duckdb` is generated with the expected schema.\n\nOpen PR via `make pr-open`. Title prefix `refactor(explorer):`. Body\nreferences the merged ADR and lists each call site touched.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Run the full verification matrix and open the PR", "wid": "w6"}], "worktree": "explorer-cli-surface-migration"} +{"anti_patterns": [{"dont": "add a new chart in the explorer without a corresponding entry in chart_types.py", "instead": "extend the registry first if a new type is genuinely warranted", "why": "that recreates the divergence we're trying to eliminate"}, {"dont": "compute percentiles in TS using `array.sort()[Math.floor(n*p)]`", "instead": "read the CLI's exact percentile method and either match it in TS or pre-compute in Python and ship in artifact", "why": "Python's numpy.percentile uses linear interpolation by default and the numbers will differ"}, {"dont": "render cost_scatter when cost_usd is null for all results", "instead": "hide the section entirely with a small note about cost data availability", "why": "an empty scatter plot looks broken"}, {"dont": "replace the existing QueryHeatmap with rank_table on TPC-DS just because the heatmap is wide", "instead": "add rank_table as an alternate view, let users switch", "why": "users have built mental models around the heatmap"}, {"dont": "reuse PALETTE colors for time_series lines without a contrast check", "instead": "extend chartTheme.ts with a dedicated TIME_SERIES_PALETTE if the 4-color palette is insufficient", "why": "4 line colors must be distinguishable when overlapping"}, {"dont": "skip parity fixtures for 'simple' charts like rank_table", "instead": "every chart gets a fixture", "why": "rank ties are exactly the kind of edge case where Python and JS sort algorithms can differ silently"}], "approach": "Tier-by-tier (w1 \u2192 w4) so user-visible value lands in priority order.\nTier 1 (percentile_ladder, cdf_chart) ships first because that's the\nhighest-impact gap for benchmark credibility.\n\nPer-chart workflow (apply uniformly):\n 1. Read the CLI implementation (find the textcharts call site,\n identify input shape and reduction).\n 2. Decide where the math lives. Default: in\n `transformer.py`, emitted via the appropriate artifact\n (BenchmarkSummary for cross-platform; DetailResult for\n single-result). Avoid TS-side computation.\n 3. Generate parity fixtures.\n 4. Build the SVG component using `chartTheme.ts` constants and\n `useElementSize` for responsiveness.\n 5. Wire into the appropriate page (Compare, ResultDetail,\n BenchmarkIndex, PlatformIndex).\n 6. Vitest unit + parity test.\n 7. Manual smoke at 3 viewport widths.\n\nThis item is sequenced AFTER the foundational items\n(`explorer-establish-cli-explorer-parity-tests`,\n`explorer-emit-comparison-artifact-from-pipeline`,\n`explorer-unify-visualization-foundations`) because each new chart\nneeds the parity fixtures, theme constants, and pipeline-emitted\ndata structures those items deliver.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 160, "reason": "Out of scope for this item. Those should be added to chart_types.py registry first via a separate planning round.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Vague 'build entirely new chart types' with no specific types defined; reason says it needs its own planning round first.", "summary": "Build entirely NEW chart types that don't exist in CLI either"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 161, "reason": "Premature optimization - SVG handles realistic data sizes; revisit if profiling shows pain points.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Explicitly premature optimization pending profiling evidence of pain points; no such evidence cited or found.", "summary": "WebGL/Canvas rendering for high-density visualizations (e.g. histogram with thousands of points)"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 162, "reason": "Product feature, not a coverage gap. Address after the chart catalog is complete.", "resolution": "promoted", "resolved_item": "user-saveable-chart-configurations-dashboards", "resolved_reason": null, "summary": "User-saveable chart configurations / dashboards"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-close-chart-type-coverage-gaps", "id": 163, "reason": "Cross-cutting UX work; better tackled as a focused interactivity item once all charts exist.", "resolution": "promoted", "resolved_item": "interactive-drilldowns-click-chart-bar-to-filter-table", "resolved_reason": null, "summary": "Interactive drilldowns (click chart bar to filter table)"}], "deps": ["explorer-emit-comparison-artifact-from-pipeline", "explorer-establish-cli-explorer-parity-tests", "explorer-unify-visualization-foundations"], "description": "The 2026-04-15 review found that the CLI registers **16 canonical\nchart types** in `benchbox/core/visualization/chart_types.py:17-45`,\nbut the explorer implements only **6 visualization components**\n(QueryHeatmap, QueryTimingChart, GroupedQueryChart,\nNormalizedSpeedupChart, DivergingBarChart, MetaLeaderboard) -\nand only 3-4 of those have CLI counterparts that compute the same\nvalues. **10 CLI chart types have no explorer equivalent.**\n\nMissing chart types and their importance for benchmark credibility:\n\n | Chart | Why it matters |\n |----------------------|------------------------------------------------------------------|\n | `percentile_ladder` | Tail-latency P50/P90/P95/P99 - central to fair benchmark claims |\n | `cdf_chart` | Distribution shape - exposes \"fast median, slow tail\" patterns |\n | `rank_table` | Compact per-query ranking - denser than heatmap for many queries |\n | `distribution_box` | Box-plot of per-query latencies - shows variance/outliers |\n | `query_histogram` | Per-query latency distribution |\n | `time_series` | Trend over runs - regression detection |\n | `cost_scatter` | Cost vs performance - central to cloud-warehouse comparisons |\n | `power_bar` | Dedicated TPC Power@Size visual |\n | `sparkline_table` | Compact metrics summary with embedded micro-charts |\n | `stacked_phase` | Phase breakdown (load/warmup/power/throughput) |\n\nThis item ports each missing chart type to the explorer in priority\norder. Every new chart consumes pipeline-emitted data\n(per `explorer-emit-comparison-artifact-from-pipeline` - extending\nthe artifact emission as needed) and ships with a parity fixture\n(per `explorer-establish-cli-explorer-parity-tests`) so the math\nis verified against the canonical Python implementation from day one.\n\nOut of scope: building entirely NEW chart types that don't exist in\nthe CLI either. Those should be evaluated and added to both sides\nthrough the canonical chart_types.py registry first.", "id": "explorer-close-chart-type-coverage-gaps", "preserves": ["BenchmarkSummary and DetailResult artifact backward compatibility - extensions add new fields, do not rename or remove existing ones", "CLI chart_types.py registry as the single canonical source - explorer mirrors it, never the other way around", "Existing 6 explorer chart components must continue to work - new charts add tabs/sections, do not replace existing visualizations", "ResultDetail / BenchmarkIndex / PlatformIndex existing layouts and navigation patterns - additions are introduced as tabs or new sections, not page rewrites"], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/**"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/**"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/manifest.ts"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/**"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/**"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartRegistry.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "tests/parity/fixtures/**"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/**"}], "state": "done", "title": "Close chart-type coverage gaps: percentile_ladder, cdf, rank_table, distribution_box, and 6 others", "verifications": [{"command": "ls results-explorer/src/components/{PercentileLadder,CDFChart,RankTable,DistributionBox,QueryHistogram,PowerBar,StackedPhase,TimeSeries,CostScatter,SparklineTable}.tsx", "description": "All 10 missing chart components exist as files", "expected": "all 10 files present", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 1}, {"command": "ls tests/parity/fixtures/{percentile_ladder,cdf,rank_table,distribution_box,query_histogram,power_bar,stacked_phase,time_series,cost_scatter,sparkline_table}.json", "description": "Every new chart has a corresponding parity fixture", "expected": "all 10 fixtures present", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm test -- chartMath.parity", "description": "Parity suite passes for all new charts", "expected": "all 10 chart fixtures pass equivalence assertions", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm test -- chartRegistry", "description": "Explorer chartRegistry mirrors CLI chart_types", "expected": "registry-parity test passes - 16 entries on both sides, no divergence", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- 'PercentileLadder|CDFChart|RankTable|DistributionBox|QueryHistogram|PowerBar|StackedPhase|TimeSeries|CostScatter|SparklineTable'", "description": "Each new chart has Vitest unit coverage", "expected": "tests cover empty data, single result, missing fields, and at least one realistic data shape per chart", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -v", "description": "Pipeline emits the new derived data fields", "expected": "tests cover percentile arrays, distribution stats, etc. as added to BenchmarkSummary / DetailResult", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 6}, {"command": "cd results-explorer && npm run typecheck && npm test && cd .. && uv run -- python -m pytest -m fast -q", "description": "TypeScript compiles, full test suite green", "expected": "all green", "item_id": "explorer-close-chart-type-coverage-gaps", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 1 - highest-impact missing chart for tail-latency credibility.\nPlacement: ResultDetail.tsx side panel + Compare.tsx tab alongside\nSpeedup/Diverging/Grouped.\n\nPercentile method MUST match the Python reference exactly\n(numpy.percentile linear interpolation) - pre-compute in the\npipeline and ship in the artifact, do not recompute in TS.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `percentile_ladder`: emit p50/p90/p95/p99 arrays + ladder SVG component", "wid": "w1"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w5", "w6", "w7"], "notes": "Tier 4 - compact alternative to the heatmap for the home page or\nplatform overview. Each row carries a per-query trend mini-chart.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `sparkline_table`: compact metrics table with embedded micro-charts", "wid": "w10"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w10", "w8", "w9"], "notes": "Final integration step:\n - Create `results-explorer/src/lib/chartRegistry.ts` mirroring\n the canonical 16-entry list from\n benchbox/core/visualization/chart_types.py.\n - Each registry entry: `{ id, title, description, requires:\n DataRequirements, component: ComponentRef, cli_equivalent: str }`.\n - Meta-test fails if the explorer registry diverges from the\n CLI registry (new chart type added on one side without the other).\n - Update `docs/visualization-parity.md` (created by\n `explorer-establish-cli-explorer-parity-tests`) with the\n full chart-type matrix and the policy: \"New chart types are\n added to chart_types.py first; explorer follows.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `chartRegistry.ts` mirroring chart_types.py + meta-test enforcing registry parity", "wid": "w11"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 1 - distribution-shape companion to percentile_ladder.\nPlacement: Compare.tsx tab + per-result detail page.\nSame pre-compute discipline: CDF samples emitted by Python.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `cdf_chart`: emit CDF samples + SVG component", "wid": "w2"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 2 - per-query ranks (1..N) across platforms with optional\n\u2191\u2193 rank-change arrows. Useful for TPC-DS (99 queries) where the\nheatmap becomes unwieldy. Add as alternate view alongside the\nheatmap; the heatmap stays the default.\n\nRank ties are a known divergence risk (Python and JS sort\nalgorithms differ silently) - fixture MUST cover ties.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `rank_table`: emit per-query rank matrix + table component", "wid": "w3"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": [], "notes": "Tier 2 - reveals platforms with high run-to-run variance even\nwhen median looks good. Consumes raw per-sample timings already\nin DetailResult.queries; reduction helper in transformer.py emits\nthe five-number summary.\nPlacement: alternate Compare view + ResultDetail\n\"Per-query distribution\" section.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `distribution_box`: emit min/Q1/median/Q3/max per query + box-plot component", "wid": "w4"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w1", "w2", "w3", "w4"], "notes": "Tier 3 - match the CLI auto-split behavior (when query count > 33\nthe CLI splits into multiple panels). Bin edges decided in Python\nand shipped in the artifact so explorer cannot diverge.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `query_histogram`: emit binned histogram + SVG component (auto-split when query_count > 33)", "wid": "w5"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w1", "w2", "w3", "w4"], "notes": "Tier 3 - currently buried in Compare summary cards as a small dl\nrow. Promote to a first-class chart on BenchmarkIndex when\npower_score is the primary metric for the benchmark family\n(consult lib/ranking.ts from explorer-unify-visualization-foundations).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `power_bar`: dedicated TPC Power@Size visualization (first-class on BenchmarkIndex)", "wid": "w6"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w1", "w2", "w3", "w4"], "notes": "Tier 3 - useful for TPC-H/TPC-DS where load and throughput phases\nmatter for comparison fairness. Phase totals already in\nBenchmarkSummary; new component renders them as stacked bars\nper platform.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `stacked_phase`: phase breakdown (load/warmup/power/throughput) stacked bars", "wid": "w7"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w5", "w6", "w7"], "notes": "Tier 4 - surfaces regressions across releases.\nPlacement: PlatformIndex.tsx (per-platform history) + new\nBenchmarkIndex \"Trends\" tab.\n\n4-color PALETTE may be insufficient for many overlapping lines -\nextend chartTheme.ts with a dedicated TIME_SERIES_PALETTE if a\ncontrast check fails.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `time_series`: trend chart of geomean_ms / power_score over run_date, per platform", "wid": "w8"}, {"evidence": null, "item_id": "explorer-close-chart-type-coverage-gaps", "needs": ["w5", "w6", "w7"], "notes": "Tier 4 - central for cloud DW comparisons\n(Snowflake/BigQuery/Athena/Databricks).\nPlacement: BenchmarkIndex when cost_usd is populated for >0\nresults. Hide section entirely when no cost data exists\n(an empty scatter looks broken).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Port `cost_scatter`: cost_usd vs power_score (or geomean_ms) scatter", "wid": "w9"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [], "deps": [], "description": "Surfaced during the funding-chip work (PRs #1091, #1105). Results-explorer\ne2e tests intermittently fail with `No result found for \"\"` (or a\nheading like `TPC-H Results` / `TPC-H - DuckDB` never becoming visible) on a\nresult id that genuinely exists in the corpus.\n\nRoot cause (verified, NOT an id-churn / fixture bug):\n - The id is present. Regenerating the corpus with\n `npm run test:e2e:fixtures` and querying the snapshot shows the row in\n both `bench.results` and `bench.result_detail_metrics`.\n - `queryRows` (results-explorer/src/db.ts) does NOT swallow errors, so this\n is a genuine ZERO-ROW read, not a masked exception, and the snapshot HTTP\n fetch is 200 OK (confirmed from the CI Playwright trace's network log).\n - `getResultDetailMetrics` / the index queries run `WHERE result_id = ?`\n against a cold, partially-fetched HTTP-backed `results.duckdb`. The\n readiness probe `waitForSnapshotRows` / `validateAttachedSnapshot`\n (src/db.ts) only does `SELECT FROM LIMIT 1` per required\n view, which a cold snapshot can satisfy while a keyed lookup on the same\n view still returns nothing.\n - A zero-row detail result is then rendered as a TERMINAL \"No result found\"\n with no retry (ResultDetail.tsx: `if (data === null) setError(...)`).\n\nWhere it bites: the FIRST page load after a fresh build (local ~40s cold vs\n~9s warm; warm re-runs 3/3 clean). CI is always cold, so CI is where it shows\n\u2014 especially the WebKit `@smoke` lane (slowest engine, smoke-only, so its\nfirst test always eats the cold start) and the first snapshot-hitting spec in\nthe Chromium full suite. Re-running the identical commit passes. A real user\ndeep-linking a detail URL against a cold CDN cache can hit the same thing.\n\nKey files:\n - results-explorer/src/db.ts \u2014 waitForSnapshotRows / SNAPSHOT_READY_SCANS\n (probe is existence-only, not keyed); queryRows (retry wrapper, only\n retries isTransientDuckDbSnapshotError, not a cold zero-row read).\n - results-explorer/src/lib/duckdbQueries.ts \u2014 getResultDetailMetrics,\n getPlatformIndexRows (the keyed lookups that come back empty).\n - results-explorer/src/pages/ResultDetail.tsx \u2014 renders null as a terminal\n \"No result found\" with no retry.\n\nRelated project memory: project_explorer_cold_snapshot_race. Same\ndevelop-red-from-flakes churn family as the auto-revert misattribution note.\n", "id": "explorer-cold-snapshot-zero-row-race", "preserves": [], "prior_art": [], "priority": "medium", "scope": [], "state": "planning", "title": "Fix results-explorer cold-snapshot zero-row race (keyed lookups return empty on a cold HTTP-backed DuckDB snapshot)", "verifications": [], "work": [{"evidence": null, "item_id": "explorer-cold-snapshot-zero-row-race", "needs": [], "notes": "Add a focused test (or a throttled-fetch harness) that attaches a cold\nHTTP-backed snapshot and shows `SELECT ... LIMIT 1` on a required view\nsucceeding while `WHERE result_id = ?` on the same view returns 0 rows in\nthe same window. This pins the defect independently of wall-clock timing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Reproduce deterministically: force a cold snapshot and assert the keyed-lookup zero-row window", "wid": "w1"}, {"evidence": null, "item_id": "explorer-cold-snapshot-zero-row-race", "needs": ["w1"], "notes": "Two candidate fixes (pick one, document why in the PR):\n(a) Strengthen waitForSnapshotRows to do a keyed probe (or a COUNT that\n cannot be satisfied from metadata) so the DB instance is not exposed\n until keyed lookups resolve; or\n(b) Treat a cold zero-row keyed read as transient in getDetailResult /\n getResultDetailMetrics and retry a bounded number of times before\n concluding null, so a real id is never rendered as \"No result found\"\n during the cold window.\nDo NOT weaken the genuine not-found path \u2014 an id that truly is absent must\nstill surface \"No result found\" promptly.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Make the readiness gate keyed, or make a cold zero-row read retryable", "wid": "w2"}, {"evidence": null, "item_id": "explorer-cold-snapshot-zero-row-race", "needs": ["w2"], "notes": "Add the w1 repro as a standing regression test. Once green, this unblocks\n(or de-risks) graduate-browser-smoke-to-blocking-gates, whose 10-green\ncriterion this flake was defeating on the WebKit lane.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Regression coverage + reassess graduating WebKit/Firefox smoke to blocking", "wid": "w3"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "separate concerns, separate UI elements", "dont": "conflate warnings with trust labels", "why": "comparability is about methodology, trust is about provenance"}, {"INSTEAD": "warn and allow", "dont": "hard-block on execution_mode or tuning_mode mismatch", "why": "intentional cross-mode comparisons are valid analytical use cases"}, {"INSTEAD": "only warn when the dimension actually differs", "dont": "show warnings for equal values", "why": "that would be noise"}], "approach": "Comparability check runs client-side after both DetailResult objects are\nloaded. Returns a list of warnings; each warning renders as a dismissible\nbanner above the chart area.\n\nWarning text should explain the implication, not just state the fact.\n\"Execution mode differs: sql vs dataframe\" is insufficient - add\n\"results may not reflect platform SQL performance differences\".\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-add-tuning-config-visibility", "explorer-extend-manifest-and-pipeline"], "description": "The strategy doc (\u00a7 Trust, Visibility, and Ranking) lists the dimensions\non which two results are comparability-breaking. Phase 1 enforces\nhard-blocks on benchmark and scale factor mismatch. Execution mode and\ntuning mode are also comparability-breaking - a SQL vs DataFrame\ncomparison or a tuned vs notuning comparison can look like a platform\nperformance difference when it is actually a configuration difference.\n\nPhase 1 couldn't enforce these because execution_mode and tuning_mode\nweren't in the manifest. Once explorer-extend-manifest-and-pipeline\nlands, the Compare view can inspect them and warn.\n\nThis TODO adds visible warnings (not hard-blocks) when:\n - The two results differ on execution_mode (SQL vs DataFrame)\n - The two results differ on tuning_mode (tuned vs notuning)\n - The two results use different query subsets (full vs partial)\n\nWarnings are prominent (banner at top of compare view) but not blocking -\nintentional cross-tuning comparisons are valuable.", "id": "explorer-comparability-warnings", "preserves": ["Compare view still renders when warnings are present", "Cross-tuning and cross-mode comparisons are allowed (not blocked) - warning is advisory, not preventive", "Existing hard-block on benchmark mismatch and scale-factor mismatch - these remain blocking"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (no pipeline changes)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ComparabilityBanner.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}], "state": "done", "title": "Add comparability warnings to Compare view for cross-mode and cross-tuning comparisons", "verifications": [{"command": "grep -n 'comparability\\|ComparabilityWarning\\|execution_mode.*warn\\|tuning_mode.*warn' results-explorer/src/pages/Compare.tsx", "description": "Comparability check function present", "expected": "check logic present", "item_id": "explorer-comparability-warnings", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'benchmark.*mismatch\\|scale_factor.*mismatch\\|hard.block\\|Cannot compare' results-explorer/src/pages/Compare.tsx", "description": "Hard-blocks still enforced", "expected": "hard-block logic still present", "item_id": "explorer-comparability-warnings", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-comparability-warnings", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-comparability-warnings", "needs": [], "notes": "Dimensions to check:\n - execution_mode: warn if \"sql\" vs \"dataframe\"\n - tuning_mode: warn if \"tuned\" vs \"notuning\"\n - query subset: warn if query_count differs and both are paths", "why": "that breaks stable detail-page URLs"}, {"INSTEAD": "unknown IDs fall through to normal 404 handling", "dont": "reject unknown IDs", "why": "a stale bookmark should still try to resolve gracefully"}, {"INSTEAD": "use a deterministic hash", "dont": "use a random or timestamp-based short ID", "why": "it would change on each pipeline run and break shared links"}], "approach": "Hashing is cheap and deterministic; use sha256 so short IDs are\nstable across pipeline runs. The collision-extension rule is the\nonly nuance - implement it as a second pass over the first-pass\ndict: sort entries, find collisions, extend both sides by 2 chars\nuntil unique, repeat.\n\nThe frontend load pattern: fetch short_ids.json on first navigation\nto /results/compare, cache in module scope. For the\nMatrix \u2192 Compare flow, the short_id is already on each PlatformRow\nso no extra fetch is needed.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-matrix-leaderboard-view"], "description": "The current Compare URL format is\n`/results/compare?ids=,,...`.\n\nFull result IDs average ~40 characters (e.g.,\n`tpch-duckdb-sf0.01-20260403-7fe93365`). With 4+ selections plus the\nbase URL, encoded commas, and any future additional query params, the\nURL approaches 2KB. Chrome handles this, but proxies, older browsers,\nand CDN caches have been observed to truncate or reject URLs above\n1024 bytes. Once the matrix leaderboard ships and users can\none-click-compare 6-8 platforms, this becomes a real failure mode.\n\nThis TODO introduces a short-ID scheme: a stable 8-character prefix\nof a deterministic hash of the full result_id. The pipeline emits a\nlookup table `short_ids.json` mapping short \u2192 full; the frontend\nresolves both directions.\n\nCollision handling: if two short IDs collide in the active corpus,\nthe pipeline extends the colliding pair to 10 chars (and so on). This\nis logged at build time. At current corpus scale (12 bundles) and\nforeseeable scale (hundreds), 8 chars is effectively collision-free,\nbut the extend-on-collision rule is required for robustness.", "id": "explorer-compare-url-short-ids", "preserves": ["Direct links to /results/r/ remain unchanged - this TODO only affects Compare query params", "Existing long-form Compare URLs continue to work - backward compatible", "Pipeline remains a single pass - short-ID generation piggybacks on the existing bundle-id enumeration"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/explorer/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/short_ids.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Use short hash IDs in Compare URLs to avoid URL-length limits", "verifications": [{"command": "test -f results-explorer/public/data/short_ids.json && echo exists", "description": "short_ids.json is emitted by the pipeline", "expected": "exists", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import json; d = json.load(open(\"results-explorer/public/data/short_ids.json\")); assert all(len(k) >= 8 for k in d); assert len(set(d.keys())) == len(d)'", "description": "Short IDs are 8 chars by default with no collisions at current corpus size", "expected": "no assertion failure", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'resolveShortId' results-explorer/src/pages/Compare.tsx", "description": "Compare page accepts short IDs", "expected": "function is called", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -n 'short_id' results-explorer/src/components/QueryHeatmap.tsx", "description": "Matrix \u2192 Compare URLs emit short IDs", "expected": "short_id referenced in URL construction", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": [], "notes": "For each result_id, compute sha256(result_id)[:8]. Build a dict\n{short: full}; on collision, extend both entries to 10 chars,\nthen 12, up to 16. Emit to\nresults-explorer/public/data/short_ids.json.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add short_id generation to benchbox/core/explorer/pipeline.py with collision extension", "wid": "w1"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w1"], "notes": "Lets the frontend read short_ids from the matrix artifact directly,\nno second lookup needed for the common case.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `short_id` field to each PlatformRow in BenchmarkSummary", "wid": "w2"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w1"], "notes": "Load short_ids.json on first call, cache in module scope. Unknown\nIDs return input unchanged so stale bookmarks fall through to\nnormal 404 handling.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add resolveShortId/resolveFullId helpers in src/lib/manifest.ts", "wid": "w3"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w2", "w3"], "notes": "Long-form URLs remain valid - this is additive.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit short IDs in Compare URLs from QueryHeatmap; accept either form in Compare.tsx", "wid": "w4"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w4"], "notes": "When Compare.tsx receives any full IDs, replace the URL with the\ncanonical short-ID form so shared links propagate compact form.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add URL canonicalization in Compare.tsx via history.replaceState", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "carry row identity separately from displayed columns.", "dont": "hide the Query Workbench View link when result_id is not visible", "why": "navigation is a core row action"}, {"INSTEAD": "show logical queries and samples as separate counts.", "dont": "label measurement samples as queries", "why": "that overstates benchmark breadth"}, {"INSTEAD": "split by cohort or force a cohort selector.", "dont": "normalize mixed benchmark/scale rows into a single trend line", "why": "geomean values are cohort-relative"}], "approach": "Fix each defect in the smallest owning component first, then add tests around\nthe observable behavior. For Query Workbench, avoid making `result_id` visible\nby default just to fix navigation; separate row identity from displayed\ncolumns. For Result Detail, audit all methodology labels that mention query\nscope so the copy cannot reuse sample-count terminology. For Platform trend,\nuse the existing ranking/cohort fields already emitted by\n`bench.benchmark_rankings` rather than inventing a frontend-only cohort key.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Fix the correctness defects found while auditing the BenchBox Results Explorer\nbefore layering on larger visual and IA changes. These issues directly break\nnavigation or misrepresent benchmark data:\n\n- Query Workbench default `View ->` links can navigate to `/results/r/undefined`\n because the default visible column set omits `result_id` while the row action\n reads `row.result_id`.\n- Result detail methodology can label sample count as query scope, e.g. showing\n \"Query scope: 88 queries\" on a page whose timing section has 22 logical\n queries and 88 individual samples.\n- Platform page \"Performance Trend\" can mix unrelated benchmark/scale cohorts\n such as SSB SF0.01, SSB SF0.1, TPC-H SF0.01, and TPC-H SF0.1 into one\n geomean trend line, which implies a single longitudinal metric where none\n exists.", "id": "explorer-critical-results-ux-defects", "preserves": ["Existing Result Detail route `/results/r/{result_id}` remains unchanged.", "Platform trend charts remain available, but only for comparable cohorts.", "Query Workbench CSV/JSON export continues to emit exactly the columns the user has selected (no extra hidden `result_id`); the projection-only fix keeps `result_id` available to row actions without leaking into export output.", "Query Workbench column picker continues to control visible table columns."], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/components/MethodologyDisclosure.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/TimeSeries.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MethodologyDisclosure.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryFilters.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Query.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/PlatformIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Query.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/ResultDetail.test.tsx"}], "state": "done", "title": "Explorer: fix critical result-browser defects found in design audit", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Frontend typecheck and unit tests pass.", "expected": "TypeScript and Vitest pass, including new regressions for workbench links, methodology counts, and platform trend grouping.", "item_id": "explorer-critical-results-ux-defects", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run build", "description": "Production explorer bundle still builds.", "expected": "Vite production build completes without errors.", "item_id": "explorer-critical-results-ux-defects", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": [], "notes": "In results-explorer/src/pages/Query.tsx, ensure `result_id` is always\navailable for row actions even when not visible. Preferred fix: keep\n`result_id` in the SELECT projection as a hidden key and render only the\nuser-selected columns. Add a regression test where DEFAULT_COLUMNS omits\n`result_id` but the \"View\" link still points to `/results/r/{id}`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Query Workbench row actions independent of visible result_id column", "wid": "w1"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": [], "notes": "In results-explorer/src/pages/ResultDetail.tsx and/or\nMethodologyDisclosure, source logical query scope from `detail.query_count`\nor `detail.display_timings.length`, and sample count from\n`detail.queries.length`. The UI should say, for example, \"22 queries,\n88 measurement samples\" rather than \"88 queries\" when samples are the\nlarger number.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct Result Detail methodology copy to distinguish logical queries from samples", "wid": "w2"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": [], "notes": "In results-explorer/src/pages/PlatformIndex.tsx and\nresults-explorer/src/components/TimeSeries.tsx call sites, group trend\ndata by `(benchmark, scale_factor, phase, primary_metric)` before drawing.\nEither render cohort-specific small multiples or require an explicit cohort\nselector. Do not plot mixed cohorts on one y-axis.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop Platform page trend chart from combining different cohorts into one time series", "wid": "w3"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": ["w1"], "notes": "Extend Query.test.tsx with a case where DEFAULT_COLUMNS omits `result_id`\nbut the rendered \"View\" link still resolves to `/results/r/{id}`. Commit\nwith the w1 fix so navigation is locked in by the same change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Query Workbench regression test for default row link without visible result_id", "wid": "w4"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": ["w2"], "notes": "Extend ResultDetail.test.tsx or MethodologyDisclosure.test.tsx to assert\n\"queries\" and \"measurement samples\" render as distinct counts when sample\ncount exceeds logical query count. Commit with the w2 fix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Result Detail regression test for separate query and sample counts", "wid": "w5"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": ["w3"], "notes": "Extend PlatformIndex.test.tsx with rows from at least two distinct\n`(benchmark, scale_factor)` cohorts and assert the trend renderer either\nsplits cohorts into small multiples or refuses to render a single merged\nline. Commit with the w3 fix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Platform trend regression test against mixed cohorts", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "show it inline below the table", "dont": "hide the methodology panel behind a 'why' link", "why": "the whole point is transparency"}, {"INSTEAD": "filter them out at pipeline time", "dont": "include cohorts with only 1 platform", "why": "rank is meaningless there"}, {"INSTEAD": "use simple rank aggregation with all inputs visible", "dont": "invent a weighted composite score", "why": "any weighting is indefensible and masks the inputs"}, {"INSTEAD": "show N/A and exclude from that platform's average", "dont": "penalize platforms that didn't run a cohort", "why": "missing data is not a loss"}], "approach": "The pipeline already iterates through every bundle once. After the\nper-benchmark artifacts are built, iterate the in-memory accumulators\nagain to compute ranks: for each cohort, sort by the cohort's primary\nmetric and assign 1-based ranks. Aggregate across cohorts per engine.\n\nFrontend is a pure table render. No interactivity in the first cut -\nclicking a cohort header navigates to the matrix leaderboard for that\ncohort.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-emit-benchmark-summary-artifact", "explorer-matrix-leaderboard-view"], "description": "The 2026-04-15 strategy review observed that BenchBox's multi-benchmark\nstory is a real differentiator vs ClickBench but the current Home page\ndoes not lean into it. A cross-benchmark meta-leaderboard answers\n\"which engines consistently do well across benchmarks?\" - a question\nno single-benchmark site can answer.\n\nApproach: compute per-engine performance *rank* within each\n(benchmark, scale, phase) cohort, then surface aggregate rank across\ncohorts. Ranks normalize away the incompatible units (power_score vs\ngeomean_ms) that make direct cross-benchmark comparison impossible.\n\nVisualization: a compact table on the home page with columns per\nbenchmark cohort and a cell showing the engine's rank (e.g., \"1/4\")\nplus a summary \"Average rank: 1.5\" column on the right.\n\nNot included: any claim that the meta-leaderboard is an official or\nweighted score. It is a transparent rank-aggregation across whatever\ncohorts the corpus contains, with all inputs visible and linkable.", "id": "explorer-cross-benchmark-meta-leaderboard", "preserves": ["Home page remains navigable - the meta-leaderboard is additive, not replacing global nav", "No invented composite scores - only transparent rank aggregation", "Recent Results section remains on the home page in a secondary position"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/meta_leaderboard.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}], "state": "done", "title": "Add cross-benchmark meta-leaderboard to the home page", "verifications": [{"command": "test -f results-explorer/public/data/meta_leaderboard.json && echo exists", "description": "meta_leaderboard.json is emitted", "expected": "exists", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import json; d = json.load(open(\"results-explorer/public/data/meta_leaderboard.json\")); assert all(len(c[\"platforms\"]) >= 2 for c in d[\"cohorts\"])'", "description": "Meta-leaderboard only includes cohorts with \u22652 platforms", "expected": "no assertion failure", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'MetaLeaderboard' results-explorer/src/pages/Home.tsx", "description": "Home page renders MetaLeaderboard", "expected": "imported and rendered", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles", "expected": "no errors", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": [], "notes": "Scan every BenchmarkSummary and emit\nresults-explorer/public/data/meta_leaderboard.json with:\n {cohorts: [{benchmark, scale, phase,\n platforms: [{platform, rank, total}]}],\n summary: [{platform, avg_rank, appearances}]}\nExclude cohorts with only 1 platform.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add MetaLeaderboard computation to pipeline.py; emit meta_leaderboard.json", "wid": "w1"}, {"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": ["w1"], "notes": "Table: one column per cohort (abbreviated header 'TPC-H SF0.01\npwr'), cells show rank fraction. Rightmost column 'Avg Rank';\nrows sorted by avg_rank ascending.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build MetaLeaderboard component at src/components/MetaLeaderboard.tsx", "wid": "w2"}, {"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate MetaLeaderboard into Home.tsx as hero panel; demote Recent Results", "wid": "w3"}, {"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": ["w2"], "notes": "Explain: (a) rank is within-cohort; (b) only cohorts with \u22652\nplatforms count; (c) platforms missing from a cohort are N/A, not\npenalized; (d) avg_rank is a simple mean across appearances;\n(e) link to the full methodology discussion.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add transparency disclosure panel below the meta-leaderboard", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "try to make frame-ancestors work by keeping it in meta", "why": "the CSP spec ignores it there; the fix is a header at the host layer, out of this file's scope."}], "approach": null, "blocked_reason": null, "category": "Security", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-06T00:00:00Z", "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": [], "description": "Source: results-explorer/publication remediation re-review (2026-07-06),\nDefect #5 (de-escalated to cosmetic).\n\nresults-explorer/index.html:15 sets the CSP via ``, and its content includes `frame-ancestors 'none'`.\nPer the CSP spec, `frame-ancestors` is IGNORED when the policy is delivered in\na element \u2014 it is only honoured as an HTTP response header. So that\ndirective provides no clickjacking protection today; it reads as a control\nthat does not exist.\n\nThis is cosmetic, NOT a security hole: the load-bearing directive\n`connect-src 'self'` (which confines all network egress and closes the\nduckdb-wasm remote-fetch/exfil channel) DOES work in a meta CSP, and it is\npresent. The only issue is the misleading inert `frame-ancestors` line.\n\nFix: drop the inert directive and replace it with a comment stating that\nframing/clickjacking protection must be applied as an HTTP response header\n(`X-Frame-Options: DENY` or a header-delivered `frame-ancestors`) at the\nhosting/CDN layer (GitHub Pages), because meta CSP cannot express it.", "id": "explorer-csp-frame-ancestors-meta-cleanup", "preserves": ["The duckdb-wasm-compatible directives (script-src 'wasm-unsafe-eval', worker-src blob:) \u2014 removing these would break the explorer.", "connect-src 'self' \u2014 the load-bearing egress confinement; must not change."], "prior_art": [{"concept": "_project/DONE/main/planning/remediate-explorer-csp-hardening.yaml \u2014 the item that added the CSP; SUPERSEDE its noted 'frame-ancestors inert in meta' caveat by actually removing the directive.", "decision": "supersede", "path": "_project/DONE/main/planning/remediate-explorer-csp-hardening.yaml"}, {"concept": "results-explorer/index.html:6-15 \u2014 the existing CSP meta + head comment; EDIT in place, preserve all non-frame-ancestors directives.", "decision": "reuse", "path": "results-explorer/index.html:6-15"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/"}, {"kind": "only_modify", "path_glob": "results-explorer/index.html"}], "state": "done", "title": "Remove the inert frame-ancestors directive from the explorer CSP meta tag", "verifications": [{"command": "grep -oE 'content=\"default-src[^\"]*\"' results-explorer/index.html | grep -c 'frame-ancestors'", "description": "frame-ancestors is gone from the CSP content= directive string (a comment mentioning it is fine)", "expected": "0 (the directive is removed from the meta content string)", "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -oE 'content=\"default-src[^\"]*\"' results-explorer/index.html | grep -c \"connect-src 'self'\"", "description": "the load-bearing connect-src 'self' directive is preserved", "expected": "1 (egress confinement unchanged)", "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run test:e2e:chromium 2>&1 | tail -5", "description": "the explorer still builds and the browser e2e suite passes (no CSP regression; connect-src still enforced)", "expected": "all specs pass \u2014 removing an inert directive is behaviorally a no-op, so this confirms nothing else broke", "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "needs": [], "notes": "In results-explorer/index.html: remove `frame-ancestors 'none'` from the\n`` content string, leaving every\nother directive byte-identical (default-src, script-src incl.\n'wasm-unsafe-eval', worker-src blob:, connect-src 'self', style-src,\nimg-src, font-src, object-src 'none', base-uri 'self'). Add an HTML comment\nnear the tag noting that clickjacking protection needs an HTTP response\nheader at the host/CDN because meta CSP ignores frame-ancestors. If the head\ncomment (index.html:6-7) implies framing is covered, correct it too.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Drop the inert frame-ancestors from the meta CSP; document the header-layer requirement", "wid": "w1"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "couple this work to the schema-versioning refactor in TODO `explorer-read-model-schema-versioning`. That work changes the error shape; this TODO changes when the error fires. Both can land in either order.", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "remove the runtime guard in `db.ts` that detects schema mismatch", "why": "the schema mismatch is real and should still produce an actionable error, just not as the *default* dev experience."}, {"INSTEAD": "(not specified)", "dont": "track the full-corpus `results-explorer/public/data/results.duckdb` in git. The whole point of (ii) is a tiny dev-only snapshot, not the production artifact.", "why": "(unstated)"}], "approach": "Pick one solution shape, implement it minimally, verify with a destructive\nsmoke (rm + npm run dev). Avoid adding configurability that this TODO\ncannot motivate \u2014 if (ii) is chosen, do NOT also implement (i) \"just in\ncase\".\n", "blocked_reason": null, "category": "Developer Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-dev-snapshot-dx", "id": 164, "reason": "Owned by TODO `explorer-read-model-schema-versioning`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Owning TODO explorer-read-model-schema-versioning is now in _project/DONE; this note's fate was resolved there, not left open.", "summary": "Make the error message itself humane (version-diff vs column-diff)."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-dev-snapshot-dx", "id": 165, "reason": "Owned by TODO `explorer-cli-surface-migration`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Owning TODO explorer-cli-surface-migration is DONE and its w2 'Relocate the CLI entry point per ADR' work unit is status:done; the rename already happened.", "summary": "Rename the CLI entry that the docs reference."}], "deps": [], "description": "A fresh-clone or returning developer runs `cd results-explorer && npm run\ndev` and is met with a full-screen error wall:\n\n DuckDB snapshot 'bench.results' is missing required columns: ...\n Regenerate the snapshot via 'benchbox explorer build' or check the\n deployed results.duckdb file.\n\nRoot cause is not a code bug \u2014 it is an architectural papercut:\n - `*.duckdb` is gitignored, so the snapshot is never tracked.\n - CI rebuilds it on every docs deploy\n (`.github/workflows/docs.yml:103`); local devs never do unless they\n know to.\n - `npm run dev` has no predev step that detects staleness.\n - The runtime guard in `results-explorer/src/db.ts:184-199` correctly\n refuses to render against a snapshot whose schema doesn't match the\n frontend's required-column set \u2014 so the user sees a 9-column-diff\n wall the first time the read-model contract bumps.\n\nBecause the read-model contract is changing actively (PRs #344, #354,\n#380 added eligibility columns in the last 30 days), this wall recurs\nevery few weeks for every developer.\n\nThis TODO closes the recurring DX wall. It is separable from the CLI\nsurface relocation (TODO `explorer-cli-surface-migration`) and the\nschema versioning work (TODO `explorer-read-model-schema-versioning`).\n\nThree solution shapes to evaluate:\n\n (i) **`predev` npm script** in `results-explorer/package.json` that\n checks whether `public/data/results.duckdb` is older than the\n most-recent file under `results-data/bundles/`, and rebuilds via\n the canonical entry point if so. Pros: zero new files; works with\n the existing build step. Cons: every `npm run dev` runs an mtime\n check; spawns the Python entry point when rebuilding (3-5s on\n the 528-bundle corpus).\n\n (ii) **Tiny in-tree dev snapshot** under\n `results-explorer/dev-data/results.duckdb` (5-10 cohorts,\n deterministic), NOT gitignored, that the explorer falls back to\n when `public/data/results.duckdb` is missing. Pros: `git clone\n && npm run dev` works instantly with no Python. Cons: adds a\n tracked binary (~500KB-2MB depending on corpus) that needs\n periodic refresh \u2014 but only when the schema bumps, not the data.\n\n (iii) **`make explorer-dev` wrapper target** that runs rebuild then\n vite. Pros: explicit, documented in Makefile. Cons: requires\n developer to know about and use the make target; doesn't help\n someone who runs `npm run dev` from inside `results-explorer/`.\n\nThe right choice depends on how often the schema bumps vs. how often\nthe data bumps; (ii) optimizes for \"schema-stable, data-shifting\" and\n(i) optimizes for \"schema-shifting, data-stable\". (iii) is the most\nconservative and explicit but doesn't fix the default `npm run dev`\npath.", "id": "explorer-dev-snapshot-dx", "preserves": ["Browser-fixture generation (`generate-browser-fixtures.mjs`) is unaffected \u2014 it has its own snapshot regen.", "Production builds (CI docs deploy) continue to use the full corpus, not the dev snapshot or fallback.", "`results-explorer/public/data/results.duckdb` semantics are unchanged when present."], "prior_art": [{"concept": "Makefile \u2014 existing target conventions like `uat-explorer-smoke`; pattern for a possible `explorer-dev` target.", "decision": "reuse", "path": "Makefile"}, {"concept": "results-data/bundles/ \u2014 source-of-truth bundle dir whose mtime can drive a staleness check.", "decision": "reuse", "path": "results-data/bundles/"}, {"concept": "results-explorer/package.json \u2014 npm scripts site; reuse existing `dev` / `build` patterns for any new predev hook.", "decision": "reuse", "path": "results-explorer/package.json"}, {"concept": "results-explorer/public/data/results.duckdb \u2014 current deployed snapshot (gitignored); a dev-data fallback would mirror this layout.", "decision": "reuse", "path": "results-explorer/public/data/results.duckdb"}, {"concept": "results-explorer/scripts/generate-browser-fixtures.mjs \u2014 existing pattern for spawning the Python publish toolchain from Node.", "decision": "reuse", "path": "results-explorer/scripts/generate-browser-fixtures.mjs"}, {"concept": "results-explorer/scripts/serve-browser-tests.mjs \u2014 existing Node-side wrapper around the browser fixture flow; pattern for a Node helper that orchestrates Python + Vite.", "decision": "reuse", "path": "results-explorer/scripts/serve-browser-tests.mjs"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".gitignore"}, {"kind": "only_modify", "path_glob": "CONTRIBUTING.md"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docs/development/browser-test-architecture.md"}, {"kind": "only_modify", "path_glob": "results-data/dev-corpus/**"}, {"kind": "only_modify", "path_glob": "results-explorer/README.md"}, {"kind": "only_modify", "path_glob": "results-explorer/dev-data/**"}, {"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}], "state": "done", "title": "Eliminate the stale-snapshot wall blocking explorer `npm run dev`", "verifications": [{"command": "rm -f results-explorer/public/data/results.duckdb && cd results-explorer && (timeout 30 npm run dev >/tmp/explorer-dev-smoke.log 2>&1 &) && sleep 8 && curl -fsS -o /dev/null -w '%{http_code}\\n' http://localhost:5173/results/", "description": "Fresh-clone simulation: dev server starts when the public snapshot is missing.", "expected": "200", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 1}, {"command": "curl -fsS -o /dev/null -w '%{http_code}\\n' http://localhost:5173/results/data/results.duckdb", "description": "The snapshot URL the app fetches returns 200 after the predev/fallback resolves.", "expected": "200", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 2}, {"command": null, "description": "Manual smoke (recorded in PR body): the rendered page shows leaderboard rows, not the column-diff Error wall.", "expected": "PR description records the screenshot or the absence of the error string in the HTML body.", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm test", "description": "Frontend tests still pass.", "expected": "exit 0", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm run build", "description": "Production build is unaffected.", "expected": "exit 0, dist/ produced", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": [], "notes": "The description cites PRs #344 / #354 / #380 as recent schema bumps.\nBefore implementation, capture a compact pinned snapshot so future\nreaders can re-run the check:\n\n - `gh pr view 344 --json title,mergedAt`\n - `gh pr view 354 --json title,mergedAt`\n - `gh pr view 380 --json title,mergedAt`\n - `git log --oneline --since=60.days -- results-explorer/src/db.ts benchbox/core/explorer_pipeline/duckdb_builder.py`\n\nRecord one line per PR (title + merge date) and the schema-bump count\nover the last 60 days in the PR description that lands this TODO's\nwork. Do NOT commit raw stdout to `_project/verification-logs/`;\npinned summary is enough.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pin the schema-bump cadence evidence cited in the description", "wid": "w0"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w0"], "notes": "Capture decision and rationale in the PR description. No separate ADR\nneeded \u2014 this is a DX choice, not an architectural one. Use the\ncadence numbers from w0. Touch on:\n - Schema-change cadence (from w0).\n - Data-change cadence: `git log --oneline --since=60.days -- results-data/bundles/ | wc -l`.\n - Latency budget for `npm run dev` start time.\n - Onboarding signal: do we want git clone -> npm run dev -> working\n UI as a baseline?\n\nThe recommendation absent overriding signal is (ii) \u2014 a tiny tracked\ndev snapshot \u2014 because:\n - It makes git clone -> npm run dev work with no Python toolchain.\n - It survives schema bumps because the developer rebuilding the\n snapshot is part of the same PR that bumps the schema.\n - The binary is small and changes rarely (only when schema bumps).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pick one of (i), (ii), (iii) with a brief decision note", "wid": "w1"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w1"], "notes": "For (ii) specifically:\n - Define a \"dev corpus\" subset under `results-data/dev-corpus/`\n (e.g., 1-2 cohorts of bundles) that produces a small snapshot.\n - Add a `make explorer-dev-snapshot` target that rebuilds it.\n - Track `results-explorer/dev-data/results.duckdb` (with a README\n explaining when to regen).\n - Modify `results-explorer/src/db.ts` snapshot URL resolution to\n try `data/results.duckdb` first, then fall back to\n `dev-data/results.duckdb` (or a build-time constant chosen by\n Vite mode).\n\nFor (i):\n - Add `predev` and `prebuild` npm scripts that check mtime and\n invoke the publish entry point if needed. Make the check\n skippable via `EXPLORER_SKIP_PREDEV=1` env var for CI / offline.\n\nFor (iii):\n - Add the make target and document it in\n `docs/development/browser-test-architecture.md` and\n `results-explorer/README.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the chosen approach", "wid": "w2"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w2"], "notes": "`results-explorer/src/db.ts:196` currently advises running\n`benchbox explorer build`. After this TODO, the recommended remediation\nfor a local dev should be the new workflow (predev / make target /\nfallback note). Update the string accordingly.\n\nNote: this message will also be touched by TODO\n`explorer-cli-surface-migration` (to point at the relocated entry).\nCoordinate via the merge order; whichever lands second resolves the\nfinal text. If `explorer-cli-surface-migration` lands first, this\nTODO updates the dev-workflow phrasing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update the error-wall remediation message", "wid": "w3"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w3"], "notes": "Smoke commands:\n - `rm -rf results-explorer/public/data && cd results-explorer && npm run dev`\n \u2192 UI loads on first attempt, no error wall.\n - For (i): touch results-data/bundles/.json and re-run\n `npm run dev` \u2192 rebuild fires.\n - For (ii): no rebuild needed; fallback snapshot is served.\n\nDocs:\n - Update `results-explorer/README.md` (if present) with the new\n dev workflow.\n - Add a one-liner to `CONTRIBUTING.md` or\n `docs/development/browser-test-architecture.md` explaining the\n snapshot lifecycle.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Smoke verification + docs update", "wid": "w4"}], "worktree": "explorer-dev-snapshot-dx"} -{"anti_patterns": [{"INSTEAD": "each (platform, tuning_mode) gets its own row", "dont": "collapse tuning modes into a single row per platform", "why": "tuned vs notuning is a comparability-breaking dimension"}, {"INSTEAD": "use null and document the fallback (display_geomean_ms)", "dont": "default missing power_score to 0", "why": "that would sort broken results to the top"}, {"INSTEAD": "use in-memory accumulation only", "dont": "fan out into additional bundle loads", "why": "the pipeline is already O(N) bundles"}, {"INSTEAD": "include phase in the key", "dont": "key the artifact on (benchmark, scale) alone", "why": "it silently mixes power and throughput phases"}, {"INSTEAD": "use display_ms from QueryDisplayTiming (median of passing runs)", "dont": "populate timings cells from the first raw QueryTiming row", "why": "the first row is the cold-cache outlier"}, {"INSTEAD": "use display_geomean_ms from the read-model contract reduction", "dont": "use geomean_ms (raw all-sample) for display ranking", "why": "it averages cold-cache and throughput-stream rows"}, {"INSTEAD": "use platform_id from the read-model contract", "dont": "use platform string directly as the matrix row key", "why": "it may include trust suffixes producing spurious duplicate rows"}], "approach": "The pipeline currently loads each bundle exactly once into memory to\nbuild manifest + details. Extend that same pass: maintain a dict\nkeyed by (benchmark, scale_factor, phase) that accumulates platform\nrows as each bundle is processed. At the end of the pass, emit one\nBenchmarkSummary per key.\n\nDo not refactor the pipeline architecture for this change. Add the\naccumulator and the emission step. Nothing else.\n\nFor phase handling: schema-v2 bundles already carry a `test_type`\n(\"power\" or \"throughput\"). Use that field to route each platform row\ninto the right accumulator bucket. Bundles without an explicit\ntest_type default to \"power\" (matches current CLI behavior).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-fix-read-model-contract"], "description": "The strategy review (2026-04-15) identified that the explorer's current\ndata model - manifest.json (aggregate only) + details/{id}.json (one file\nper result) - cannot support a ClickBench-style Platform \u00d7 Query matrix\nleaderboard without N parallel detail fetches assembled client-side. That\napproach is slow, fragile, and silently breaks when platforms run\nheterogeneous query subsets.\n\nThis TODO introduces a third derived artifact shape emitted by the\nexplorer pipeline:\n\n results-explorer/public/data/benchmarks/{benchmark}_sf{sf}_{phase}.json\n\nOne file per (benchmark, scale_factor, phase) tuple, containing the full\ncross-platform query matrix pre-computed from already-loaded bundles (no\nextra I/O - the pipeline already has everything in memory during its\nsingle pass).\n\nKey schema decisions driven by the adversarial review:\n\n1. Key includes **phase** (not just benchmark/scale). The proposal that\n keyed on (benchmark, scale) alone silently collapses power vs\n throughput vs maintenance phases. Making phase part of the filename\n makes the scope of each matrix explicit and URL-addressable.\n\n2. Tuning_mode is a per-platform-row field, not an artifact-level filter.\n Two DuckDB rows with different tuning modes occupy different matrix\n rows; the UI labels them as such.\n\n3. Each row carries `power_score` (for TPC-H/TPC-DS), `geomean_ms`, and\n `cost_usd` so the UI can choose ranking metric per benchmark family\n without a second fetch.\n\n4. Sparse matrices: platforms that ran a subset of queries produce\n `null` in the missing cells, never crash the build.\n\n5. Artifacts are gzipped on write (`*.json.gz`). TPC-DS SF100 with 30\n platforms \u00d7 99 queries \u00d7 throughput streams can reach 200-500KB raw;\n gzip cuts that ~5\u00d7. Loader handles both for local dev convenience.", "id": "explorer-emit-benchmark-summary-artifact", "preserves": ["Existing details/{id}.json per-result pages - detail page consumers must continue to work", "Existing manifest.json schema - new artifact is additive; do not remove or reshape manifest fields", "ExplorerPipeline.run() remains a single pass over bundles - no extra bundle loads introduced"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/explorer.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/explorer/exporter.py"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/**"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/benchmarks/**"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer/test_benchmark_summary.py"}], "state": "done", "title": "Emit per-benchmark summary artifact for matrix leaderboard view", "verifications": [{"command": "uv run -- python -m benchbox.cli explorer build --dry-run 2>&1 | grep -c 'benchmarks/.*_sf.*_.*\\.json'", "description": "Pipeline emits a summary artifact per (benchmark, scale, phase) tuple", "expected": "non-zero integer matching unique (benchmark, scale, phase) count", "item_id": "explorer-emit-benchmark-summary-artifact", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer/test_benchmark_summary.py -v", "description": "Unit tests pass", "expected": "all tests pass", "item_id": "explorer-emit-benchmark-summary-artifact", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c 'import gzip, json; d = json.loads(gzip.open(\"results-explorer/public/data/benchmarks/tpch_sf0.01_power.json.gz\").read()); assert \"platforms\" in d and \"query_ids\" in d'", "description": "Generated artifact round-trips through gzip + JSON", "expected": "no assertion failure", "item_id": "explorer-emit-benchmark-summary-artifact", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": [], "notes": "BenchmarkSummary fields: benchmark, scale_factor, phase, query_ids\n(ordered), platforms (list of PlatformRow), cell_reduction (string\ndocumenting the reduction policy, e.g.\n\"median_successful_measurement_ms\"), ranking (from\nRANKING_METRIC_BY_FAMILY, set by explorer-align-ranking-metric-with-tpc-standards).\n\nPlatformRow fields:\n result_id, platform_id (canonical), platform (raw display name),\n platform_version, tuning_mode, tuning_hash, execution_mode,\n trust_label, run_date, is_ranking_eligible (bool),\n power_score (nullable), display_geomean_ms (from read-model contract\n display reduction - NOT raw geomean_ms), sample_geomean_ms (raw,\n for audit), cost_usd (nullable),\n timings (dict query_id \u2192 display_ms from QueryDisplayTiming, null\n per missing cell - NOT the first raw duration_ms row).\n\nThe distinction between display_geomean_ms and sample_geomean_ms is\ncritical: the matrix cells MUST use display_ms values (median of\npassing runs per logical query), not raw per-sample rows. See\nexplorer-fix-read-model-contract for the reduction definition.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define BenchmarkSummary + PlatformRow Pydantic models in benchbox/core/explorer/models.py", "wid": "w1"}, {"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": ["w1"], "notes": "Zero additional disk I/O - accumulate from detail results already\nloaded in the single-pass scan. Phase values: \"power\" and\n\"throughput\" in Phase 1. Use the same sf_str formatting as existing\nmanifest paths.\n\nCanonical-row selection: for each (platform_id, benchmark, scale,\nphase) group, call `select_canonical_row()` from the read-model\ncontract to pick the default ranked row. Non-canonical variants\n(different tuning modes, trust tiers) are included as additional\nPlatformRows marked `is_ranking_eligible: false`. This lets the\nmatrix default view show one row per platform while the \"All\nvariants\" filter reveals the full set.\n\nSort canonical rows first by `is_ranking_eligible` descending, then\nby primary ranking metric (power_score or display_geomean_ms).\nNon-eligible rows follow in the same metric order.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend ExplorerPipeline.run() to emit benchmarks/{b}_sf{sf}_{phase}.json.gz per (benchmark, scale, phase) tuple", "wid": "w2"}, {"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": ["w2"], "notes": "Cases: (a) normal multi-platform; (b) sparse query subset\n(platform missing Q17 \u2192 null cell); (c) mixed tuning modes produce\ndistinct rows; (d) phase separation (power and throughput produce\nseparate files); (e) gzip output is valid and round-trips.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write unit tests in tests/unit/core/explorer/test_benchmark_summary.py", "wid": "w3"}, {"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate artifacts (uv run benchbox explorer build) and commit refreshed data/ tree", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "`display_geomean_ms` is the canonical comparison value, `geomean_ms` is preserved for audit", "dont": "change the `geomean_ms` semantics", "why": "legacy consumers depend on the raw-sample geomean"}, {"INSTEAD": "disambiguate once in the pipeline and ship the result in the artifact", "dont": "compute platform display labels in two places", "why": "that recreates the parity bug we just fixed"}, {"INSTEAD": "use a structured `{kind, severity, message}` shape if extending, but for v1 mirror the existing string output of buildComparabilityWarnings exactly", "dont": "emit comparability_warnings as free-form strings", "why": "the explorer needs to apply formatting/severity"}, {"INSTEAD": "emit on-demand via CLI subcommand, pre-emit only popular combinations", "dont": "emit comparison artifacts for every N-way combination", "why": "the combinatorics explode (100 results = millions)"}, {"INSTEAD": "keep the fallback and log when it fires", "dont": "remove the per-detail fallback in Compare.tsx during rollout", "why": "pre-emitted artifacts will not exist for every combination users click"}, {"INSTEAD": "version field is mandatory and explorer hard-fails on mismatch", "dont": "skip the `schema_version` field on ComparisonArtifact", "why": "explorer must reject artifacts written by an incompatible older pipeline"}], "approach": "Schema-first: define the Pydantic model (w1) before any logic, so the\nexplorer team can stub the consumer in parallel. Build the disambiguator\n(w2) before the artifact builder (w3) because the builder calls it.\nWire the CLI subcommand (w4) before refactoring Compare (w5) so the\nartifact actually exists to fetch. Pipeline pre-emission for popular\ncombinations (w8) layers on top of the subcommand and is independent\nof the explorer refactor.\n\nSpeedup definition: use `slowest / this` per query (matches the current\nCompare.tsx breakdown table at line 392) for the `speedup_vs_slowest_per_row`\nfield. For the per-baseline view (NormalizedSpeedupChart), the artifact\nprovides the raw `display_ms_per_row` and the chart picks the baseline\nindex - this is the one piece of computation that cannot be pre-emitted\nbecause the user picks the baseline interactively.\n\nHOWEVER: the speedup math `baseline_ms / this_ms` becomes a 3-line\nreduce in the chart, calling the parity-tested `speedupRatio` helper.\nDefinition is unified, drift impossible.\n\nThe `ComparisonArtifact` schema field names mirror the explorer types\nexactly to avoid translation layer bugs (Compare.tsx already uses\n`display_timings`, `display_geomean_ms` from DetailResult - the\nartifact uses the same names).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-emit-comparison-artifact-from-pipeline", "id": 166, "reason": "Combinatorial - 100 results \u00d7 C(100,4) = 3.9M comparisons is not\nreasonable to pre-compute. Address with a smart popularity heuristic\n(top-N most-clicked) once telemetry exists.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Pre-computing all N-way comparisons is explicitly combinatorial-infeasible; deferred pending telemetry to drive a popularity heuristic that doesn't exist yet.", "summary": "Pre-emit comparison artifacts for arbitrary N-way combinations at pipeline time"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-emit-comparison-artifact-from-pipeline", "id": 167, "reason": "Requires turning the static site into an app with a backend.\nOut of scope for the static-hosted Phase 2 explorer. v1 ships\npre-emitted artifacts for popular combinations + browser fallback.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Requires turning the static-hosted explorer into a backend app; explicit architecture decision chose pre-emitted artifacts instead.", "summary": "Server-side computation of comparison artifacts at request time"}], "deps": ["explorer-establish-cli-explorer-parity-tests", "explorer-unify-visualization-foundations"], "description": "The 2026-04-15 review identified that `Compare.tsx` fetches N detail\nJSON files and recomputes everything per render: per-query speedups\n(line 388-393), diverging deltas (DivergingBarChart.tsx:49), normalized\nspeedups (NormalizedSpeedupChart.tsx:57), summary-card metrics\n(Compare.tsx:219-290), and the breakdown table (Compare.tsx:364-429).\n\nThis is the architectural root cause of the parity problem: when the\nCLI and explorer compute independently, drift is inevitable. The\nparity test infrastructure (`explorer-establish-cli-explorer-parity-tests`)\nproves equivalence today but does not eliminate the duplication.\n\nThe fix is to push computation into the pipeline. The pipeline already\nemits per-`(benchmark, sf, phase)` `BenchmarkSummary` artifacts via\n`transformer.py` \u2192 `pipeline.py`. Add a parallel `ComparisonArtifact`\nemitted per ordered set of result_ids: it carries display_timings,\nper-query speedup matrix (one row per (queryId, baselineIdx)),\ndiverging delta matrix, and pre-disambiguated platform display labels.\n\nCompare.tsx becomes a pure renderer that fetches one artifact and\niterates. The chart components stop computing - they receive\npre-computed numeric arrays.\n\nThis work also ports the platform-label disambiguation from\n`result_plotter.py:68-149` (driver version, execution mode, scale\nfactor suffixes) into the artifact, so the explorer no longer renders\nambiguous \"DuckDB / DuckDB\" rows when two results share a platform name.", "id": "explorer-emit-comparison-artifact-from-pipeline", "preserves": ["Comparison URL format `?ids=id1,id2` - short-id resolution behavior unchanged", "Existing `BenchmarkSummary` artifact format and emission - this work adds a parallel artifact, does not modify the summary", "Existing `ComparabilityBanner` UI - only the warning generation moves server-side; the component contract is unchanged", "Existing `DetailResult` model and per-detail JSON file emission - Compare must still work via the fallback path during rollout", "RANKING_METRIC_BY_FAMILY registry - the artifact reads from it but does not modify"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/**"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/explorer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/pipeline.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/DivergingBarChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/NormalizedSpeedupChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryTimingChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "tests/parity/fixtures/comparison_artifact_*.json"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/**"}], "state": "done", "title": "Emit pre-computed comparison artifact from pipeline so Compare page becomes a renderer", "verifications": [{"command": "grep -n 'class ComparisonArtifact' benchbox/core/explorer_pipeline/models.py", "description": "ComparisonArtifact Pydantic model exists with the documented fields", "expected": "class defined with schema_version, rows, query_cells, comparability_warnings, ranking_primary_metric", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_transformer.py::test_disambiguate_platform_labels -v", "description": "Disambiguator produces unique labels", "expected": "passes for single, duplicate, and triple-collision cases", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_comparison_artifact.py -v", "description": "Comparison artifact builder unit tests pass", "expected": "all cases pass", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 3}, {"command": "benchbox explorer build-comparison --ids , --out /tmp/cmp.json && uv run -- python -c 'import json; from benchbox.core.explorer_pipeline.models import ComparisonArtifact; ComparisonArtifact.model_validate(json.load(open(\"/tmp/cmp.json\")))'", "description": "CLI subcommand emits a valid artifact", "expected": "no exception - artifact validates against schema", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- Compare", "description": "Compare.tsx prefers artifact when available", "expected": "tests cover both artifact-present and fallback paths", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -n 'slowestPrimary / primary\\|slowest / fastest' results-explorer/src/pages/Compare.tsx", "description": "No two definitions of speedup remain in Compare.tsx", "expected": "zero matches - both replaced with `row.speedup_vs_slowest_per_row` from the artifact", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest -m fast -q && cd results-explorer && npm test", "description": "Full Python and Vitest suites pass", "expected": "all tests pass", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": [], "notes": "Schema (must match what Compare.tsx will consume):\n\n class ComparisonRow(BaseModel):\n result_id: str\n short_id: str\n platform: str # raw platform name\n display_label: str # disambiguated (e.g. \"DuckDB v1.2.0\")\n platform_id: str\n run_date: str\n trust_label: str\n tuning_mode: str | None\n execution_mode: str | None\n test_type: str | None\n geomean_ms: float | None\n display_geomean_ms: float | None\n power_score: float | None\n total_duration_s: float\n driver_version: str | None\n display_timings: list[QueryDisplayTiming]\n\n class ComparisonQueryCell(BaseModel):\n query_id: str\n display_ms_per_row: list[float | None] # one per ComparisonRow\n fastest_ms: float | None\n slowest_ms: float | None\n speedup_vs_slowest_per_row: list[float | None] # slowest/this\n\n class ComparisonArtifact(BaseModel):\n schema_version: int = 1\n benchmark: str\n scale_factor: float\n generated_at: str\n rows: list[ComparisonRow]\n query_cells: list[ComparisonQueryCell]\n comparability_warnings: list[str]\n ranking_primary_metric: str # \"geomean_ms\" or \"power_score\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define `ComparisonArtifact` Pydantic model in `benchbox/core/explorer_pipeline/models.py`", "wid": "w1"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w1"], "notes": "Algorithm (lifted from CLI, adapt for the artifact rows):\n - Group rows by `platform_id`.\n - If a group has size 1: `display_label = platform`.\n - Else: progressively append disambiguators in priority order\n (driver_version, execution_mode, scale_factor, run_date) until\n all labels in the group are unique.\n - Resulting label format: \"DuckDB v1.2.0\" or\n \"DuckDB v1.2.0 (sql)\" or \"DuckDB v1.2.0 SF1.0\".\n\nAdd unit test in `tests/unit/core/explorer_pipeline/test_transformer.py`\nthat mirrors any tests for the CLI version - and add a parity\nfixture (per `explorer-establish-cli-explorer-parity-tests`) so\nboth sides assert the same labels.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement `_disambiguate_platform_labels(rows)` in transformer.py mirroring `result_plotter.py:68-149` logic", "wid": "w2"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w2"], "notes": "Inputs: list of DetailResult (already fetched by caller),\n list of corresponding ManifestEntry (for trust/short_id).\n\nAlgorithm:\n 1. Validate same benchmark + same scale_factor (raise on mismatch).\n 2. Build ComparisonRow per detail using disambiguated labels.\n 3. Compute union of all query_ids (sorted natural).\n 4. For each query_id:\n - display_ms_per_row from row.display_timings\n - fastest_ms = min of non-null, slowest_ms = max\n - speedup_vs_slowest_per_row[i] = slowest_ms / display_ms[i]\n if both present and >0 else None\n 5. Compute comparability_warnings (mirror\n `buildComparabilityWarnings` in ComparabilityBanner.tsx so\n both sides agree).\n 6. Determine ranking_primary_metric from\n `RANKING_METRIC_BY_FAMILY[benchmark]` (already exists).\n\nPure function - no I/O. Caller writes the JSON.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `build_comparison_artifact(detail_results, manifest_entries)` in transformer.py", "wid": "w3"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w3"], "notes": "Strategy decision: v1 emits on-demand. The alternative (pre-emit\nevery N-way combination) is documented in `deferred` as\ncombinatorially infeasible.\n\nSubcommand contract:\n benchbox explorer build-comparison \\\n --ids id1,id2,id3 \\\n --out data/compare/.json\n\nOutput path: `data/compare/.json`.\nHash MUST be deterministic on sorted input ids (the explorer\ncomputes the same hash to fetch). Schema validates against\n`ComparisonArtifact` Pydantic model on emit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `benchbox explorer build-comparison` CLI subcommand emitting one artifact per --ids set", "wid": "w4"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w4"], "notes": "New explorer flow:\n 1. Resolve short IDs to full IDs (existing).\n 2. Compute deterministic comparison-artifact path:\n `/results/data/compare/${sha256(sortedIds.join(',')).slice(0,16)}.json`.\n 3. Try to fetch it. If 200, render directly from artifact fields.\n 4. If 404, fall back to the current per-detail flow (back-compat\n during rollout) - log a warning so we know when fallback fires.\n\nCharts no longer compute speedups/deltas locally - they receive\npre-computed numeric arrays from the artifact. Per the parity\ntest infrastructure, helper functions in `chartMath.ts` are still\nused for the *renderer* math (axis stops, bar widths) but never\nfor the values.\n\nRemove the dual `slowest/fastest` speedup definition from the\nsummary cards (Compare.tsx:225) and the breakdown table\n(Compare.tsx:392) - both now read `speedup_vs_slowest_per_row`\nfrom the artifact, which is the single source.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor `Compare.tsx` to fetch `ComparisonArtifact` instead of N `DetailResult` files", "wid": "w5"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w3"], "notes": "Cases:\n - Same benchmark/SF passes; different benchmark raises\n - Different SF raises\n - Single result returns artifact with empty speedup_vs_slowest_per_row entries (None)\n - All-failed query produces None entries, not zeros\n - Disambiguation produces unique labels for 2 same-platform-different-driver results\n - speedup_vs_slowest matches `slowest/this` definition exactly\n - comparability_warnings list mirrors what ComparabilityBanner expects (cross-test_type, cross-tuning, query-count mismatch)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `tests/unit/core/explorer_pipeline/test_comparison_artifact.py` covering edge cases", "wid": "w6"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w5", "w6"], "notes": "Requires the parity infrastructure to exist - declared as a hard\ncross-item dep on `explorer-establish-cli-explorer-parity-tests`\nat the item level.\n\nFixture flow:\n - Python emits `tests/parity/fixtures/comparison_artifact_*.json`\n covering at least: 2-result baseline, 3-result baseline,\n missing-query, all-failed-query, same-platform-different-version\n - Vitest reads each fixture and asserts that mounting Compare\n with the artifact produces the documented row counts, fastest\n highlight positions, and speedup numbers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a parity fixture pair so the explorer's renderer assertions cover the artifact contract", "wid": "w7"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w4"], "notes": "Pipeline-level pre-emission for the static site:\n - Config flag in `pipeline.py` (default 0 = off) for the\n number of pre-emitted comparisons per benchmark\n - \"Popular\" defined for v1 as: the N most-recent result\n combinations per benchmark from the manifest. Telemetry-driven\n popularity is deferred.\n - Wired so a single pipeline run produces the artifacts under\n `data/compare/` alongside the manifest.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire pipeline to optionally pre-emit comparison artifacts for top-N popular combinations per benchmark", "wid": "w8"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "`make parity-check` must be run separately to detect diffs", "dont": "commit auto-regenerated fixtures without inspecting the diff", "why": "the fixture IS the contract and silent regeneration hides drift"}, {"INSTEAD": "Python (the canonical compute layer) emits, TS verifies", "dont": "generate fixtures from the TS side", "why": "that makes Python the follower, not the source of truth"}, {"INSTEAD": "components import helpers, no exceptions", "dont": "inline the math back into components after extraction", "why": "that defeats the test contract"}, {"INSTEAD": "every helper must have at least one null-input case", "dont": "skip cases for missing-data inputs (null/None)", "why": "that is exactly where Python and JS often disagree (NaN vs null)"}, {"INSTEAD": "use `Math.abs(a - b) < 1e-9` with explicit tolerance", "dont": "use `toEqual` for floats in the parity suite", "why": "IEEE-754 noise differs between Python and JS"}], "approach": "Build foundation-first: extract pure helpers (w1) before generating\nfixtures (w2), because the fixture generator must mirror the exact\nsignatures the helpers expose.\n\nReuse the project's existing Python visualization functions where they\nexist as pure helpers - `core/visualization/ascii/heatmap.py` and\nsiblings delegate to `textcharts`, but the pre-textcharts color/sort\nmath may live in `result_plotter.py`. If a Python equivalent does not\nexist, ADD it (in Python) rather than picking a TS-only number, so\nparity is bidirectional.\n\nFixture format mirrors the textcharts test pattern (if any) so future\ncontributors recognize the convention. Keep fixtures small and readable\n- explicit `id` on each case is mandatory so failures are debuggable.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-establish-cli-explorer-parity-tests", "id": 168, "reason": "Overkill for trivial math (log10, ratios, medians). Cost in build\ncomplexity and runtime size is not justified when fixture-pinned\ncontracts achieve the same correctness guarantee.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Rejected in its own reason as overkill for trivial math; fixture-pinned contracts already provide the correctness guarantee.", "summary": "WASM-compile the Python compute layer and call it from the browser"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-establish-cli-explorer-parity-tests", "id": 169, "reason": "Tracked separately in `add-hypothesis-property-tests.yaml`. Should\nbuild on this fixture infrastructure once it is established.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Explicitly tracked separately in add-hypothesis-property-tests.yaml, which is open (planning).", "summary": "Property-based parity tests via Hypothesis + fast-check"}], "deps": [], "description": "The 2026-04-15 visualization review identified that the Python CLI\n(`benchbox/core/visualization/`) and the Preact explorer\n(`results-explorer/src/components/`) reimplement the same\nvisualization math in two languages with **zero shared computation\nand zero contract tests**.\n\nConcrete drift surfaces today:\n - Heatmap color: `colorForCell` (QueryHeatmap.tsx:39-45) duplicates\n `textcharts.heatmap` log10-ratio\u2192hue logic.\n - Per-baseline speedup: `NormalizedSpeedupChart.tsx:57` duplicates\n `textcharts.normalized_speedup` (`baseline / this`).\n - Diverging delta: `DivergingBarChart.tsx:49` duplicates the\n `textcharts.diverging_bar` `(this - baseline) / baseline * 100`\n formula, plus an independent magnitude sort at line 63-67.\n - Per-query \"fastest\" highlight: Compare.tsx:388-411 inlines its own\n ranking that has no CLI counterpart and disagrees with Compare's\n summary-card definition (see explorer-unify-visualization-foundations\n for the dual-definition bug).\n\nThe user requirement is explicit: **\"there should be only ONE place\nwhere visualization values are composed, when that is not possible\nwe must have test infrastructure that ENSURES the parallel\nimplementations match.\"** This item builds that test infrastructure.\nCentralizing computation server-side is tracked separately\n(explorer-emit-comparison-artifact-from-pipeline).\n\nStrategy: Python emits canonical fixtures (input data + expected\noutput values) for every chart-type computation. Vitest loads the\nfixtures and asserts that the explorer's pure helper functions produce\nbyte-identical numbers (within float tolerance). Any future drift\nfails CI, regardless of which side changes.", "id": "explorer-establish-cli-explorer-parity-tests", "preserves": ["Existing 36-test Vitest suite continues to pass alongside the new parity suite", "Existing `colorForCell` / `lightnessForCell` exported signatures from QueryHeatmap.tsx - these are imported elsewhere and tested", "Existing chart visual output - extracting helpers must not change pixel-level rendering for the same input data", "Vitest setup at `results-explorer/src/test/setup.ts` - append, do not replace"], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/**"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/ascii/**"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/result_plotter.py"}, {"kind": "only_modify", "path_glob": ".github/workflows/ci.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docs/visualization-parity.md"}, {"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/parity/**"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/DivergingBarChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/NormalizedSpeedupChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/QueryHeatmap.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartMath.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "tests/parity/**"}], "state": "done", "title": "Establish CLI\u2194explorer visualization parity test infrastructure (fixture-driven)", "verifications": [{"command": "grep -E 'export (function|const) (colorForCell|lightnessForCell|speedupRatio|deltaPct|sortByMagnitudeDesc|perQuerySpeedup|geomeanMs)' results-explorer/src/lib/chartMath.ts", "description": "Pure helper module exists and re-exports the documented functions", "expected": "all 7 named exports present", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python tests/parity/generate_visualization_fixtures.py --out tests/parity/fixtures && ls tests/parity/fixtures/*.json", "description": "Fixture generator runs and produces one file per chart computation", "expected": "at least 7 .json files matching the 7 helpers in w1", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm test -- chartMath.parity", "description": "Vitest parity suite runs all fixture cases and passes", "expected": "parity suite present, all cases pass", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 3}, {"command": "make parity-check", "description": "Make target detects fixture drift", "expected": "exit 0 when fixtures match, exit non-zero with diff when they don't", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- QueryHeatmap", "description": "Existing QueryHeatmap unit tests still pass (refactored to use fixtures)", "expected": "all QueryHeatmap tests pass; no hardcoded color numbers remain in the test file", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 5}, {"command": "cd results-explorer && npm test", "description": "Full Vitest suite still passes (36 + new parity tests)", "expected": "all tests pass", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": [], "notes": "Move the math out of the SVG components into pure functions:\n - `colorForCell`, `lightnessForCell` (already pure, re-export from QueryHeatmap)\n - `speedupRatio(baselineMs, thisMs)` from NormalizedSpeedupChart.tsx:57\n - `deltaPct(thisMs, baselineMs)` from DivergingBarChart.tsx:49\n - `sortByMagnitudeDesc(entries)` from DivergingBarChart.tsx:63-67\n - `perQuerySpeedup(timings)` (slowest/fastest) from Compare.tsx:388-393\n - `geomeanMs(samples)` (median-based reduction matching transformer)\nComponents import these helpers; no behavior change. Each helper\nreturns plain numbers - no SVG, no DOM - so Vitest can call them\ndirectly from a fixture loop.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit and extract pure computation functions out of every explorer chart component into testable helpers in `results-explorer/src/lib/chartMath.ts`", "wid": "w1"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w1"], "notes": "Pytest-runnable script (or a regular pytest test using\n`--generate-fixtures` flag) that produces files at\n`tests/parity/fixtures/{chart_name}.json` with shape:\n\n {\n \"chart\": \"heatmap_color\",\n \"version\": 1,\n \"cases\": [\n {\"id\": \"fastest_in_col\", \"input\": {\"ms\": 100, \"minInCol\": 100},\n \"expected\": {\"hue\": 120}},\n {\"id\": \"ten_x_slower\", \"input\": {\"ms\": 1000, \"minInCol\": 100},\n \"expected\": {\"hue\": 0}},\n ...\n ]\n }\n\nOne fixture file per chart computation listed in w1. Cases must\ninclude: identity (ratio=1), boundary (ratio=10 clamp), missing\ndata (None/null), and at least one realistic mid-range value.\n\nFixtures are CHECKED INTO GIT - they are the contract. Regenerating\nthem must be a deliberate act (changes the contract) and require\nthe Vitest suite to accept the new numbers via re-run.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build Python fixture generator at `tests/parity/generate_visualization_fixtures.py` that emits JSON fixtures per chart computation", "wid": "w2"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w2"], "notes": "Pattern: read each `tests/parity/fixtures/*.json` (relative path\ncomputed in vitest config), iterate `cases`, call the corresponding\nTS helper from `chartMath.ts`, assert\n`Math.abs(result - expected) < 1e-9` for floats and strict equality\nfor ints/strings.\n\nUse `describe.each` or generator pattern so each case appears as\nits own test name (debuggable on failure). On a fixture mismatch,\nthe failure message must show: chart name, case id, input, expected\nvs actual, and the suspected source of drift (Python file vs TS file).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Vitest parity suite at `results-explorer/src/__tests__/parity/chartMath.parity.test.ts` that loads each fixture and asserts equivalence", "wid": "w3"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w3"], "notes": "- `make parity-fixtures` regenerates fixtures via the Python script\n- `make parity-check` runs Python generation into a tmpdir, diffs\n against the checked-in fixtures, fails if they differ\n- GitHub Actions: run `make parity-check` AND `cd results-explorer\n && npm test` on every PR\n- Document the workflow in `docs/contributing-to-explorer.md`\n (or new `docs/visualization-parity.md`): \"If you change a chart\n computation, regenerate fixtures with `make parity-fixtures` -\n this changes the contract; reviewers must approve the diff.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire fixture generation into a Make target / CI gate so divergence is detected automatically", "wid": "w4"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w3"], "notes": "The current `QueryHeatmap.test.tsx` tests `colorForCell` against\nhardcoded TS expectations only. Replace those assertions with the\nfixture-driven path so QueryHeatmap is the proof-of-concept for\nthe new infrastructure. Delete the duplicated hardcoded numbers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Backfill parity for the existing `colorForCell` and `lightnessForCell` tests so they share the fixture suite", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "UI consumption comes in follow-up TODOs", "dont": "add UI changes in this TODO", "why": "this is a pipeline/data-model change only"}, {"INSTEAD": "stability matters more than hash quality", "dont": "change the tuning_hash algorithm without updating all callers", "why": "it is used for grouping across results"}, {"INSTEAD": "null-safe extraction throughout", "dont": "error on missing fields", "why": "older bundles predate many fields"}, {"INSTEAD": "add it to the manifest at build time", "dont": "recompute geomean_ms in every browser query", "why": "it should be precomputed"}], "approach": "Source-map first (w1): walk the bundle schema and write down exactly which\nbundle key maps to which manifest field. Avoids guessing during implementation.\n\nCompute geomean_ms at transform time, not in the browser. The browser DuckDB\ncan do it in a query, but pre-computing it means it's available in the JSON\nmanifest and in sort/filter without DuckDB being loaded.\n\ntuning_hash: short SHA256 prefix of the serialized tuning config dict (or\ntuning mode string if no detail is available). Purpose is grouping, not\ncryptographic identity - 8 hex chars is enough.\n\ntest_type: infer from which phases are present in the bundle. If\nphases.power_test is populated \u2192 \"power\"; if phases.throughput_test \u2192 \"throughput\";\nif both \u2192 \"power\" (primary). If neither, leave null.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Post-launch audit (2026-04-14) found that the Phase 1 pipeline transformer\nemits only 11 manifest fields from a bundle that contains 13 blocks and\n50+ fields. This silently drops execution_mode, tuning_mode, platform_version,\ntest_type, validation_status, cost_usd, and the per-query run_type/iter/stream\nfields needed for warmup vs measurement distinction.\n\nThe DuckDB `results` table has 11 columns; all other fidelity TODOs\n(methodology disclosure, tuning visibility, comparability warnings, charts)\ndepend on these fields reaching the manifest and DuckDB schema first.\n\nTarget manifest additions (all sourced from existing bundle fields):\n - geomean_ms - exp(mean(ln(query_ms))) across measurement queries\n - execution_mode - \"sql\" | \"dataframe\" from config.mode\n - tuning_mode - \"tuned\" | \"notuning\" | \"auto\" from config.tuning_mode\n - tuning_hash - stable short hash of tuning config for grouping\n - platform_version - platform.version (distinct from driver_version)\n - test_type - \"power\" | \"throughput\" from phases\n - validation_status - summary.validation.status\n - cost_usd - cost.total_usd (null if absent)\n\nTarget query record additions:\n - run_type - \"warmup\" | \"measurement\" per query timing entry\n - iter, stream - iteration and stream IDs", "id": "explorer-extend-manifest-and-pipeline", "preserves": ["Existing DuckDB column names - extend schema with new columns only", "Existing manifest field names and types - add fields, never rename or remove", "Explorer pages render correctly for bundles that have no tuning config, no cost data, or no run_type on queries", "Null-safety for older bundles that predate any new field - missing \u2192 null, not error"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py (bundle schema is read-only here)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/components/ (no UI changes in this TODO)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ (no UI changes in this TODO)"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}], "state": "done", "title": "Extend explorer pipeline manifest and DuckDB schema with missing bundle fields", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -q", "description": "All pipeline unit tests pass", "expected": "all tests pass", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run python -c \"import json; m=json.load(open('results-explorer/public/data/manifest.json')); r=m['results'][0]; assert 'geomean_ms' in r and 'execution_mode' in r and 'tuning_mode' in r, r.keys()\"", "description": "Regenerated manifest contains new fields for existing bundles", "expected": "(no assertion error)", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"import duckdb; con=duckdb.connect('results-explorer/public/data/results.duckdb'); print([c[0] for c in con.execute('PRAGMA table_info(results)').fetchall()])\"", "description": "DuckDB results table has new columns", "expected": "list includes geomean_ms, execution_mode, tuning_mode, validation_status", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite unaffected", "expected": "all tests pass", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Audit transformer.py and models.py: map each target field to its bundle source path; document any fields that require computation (geomean_ms, tuning_hash) vs direct extraction.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add geomean_ms computation to transformer.py: exp(mean(ln(ms))) over measurement-only queries (run_type=measurement or all if run_type absent). Add to ManifestResult and DetailResult models.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add execution_mode, tuning_mode, tuning_hash, platform_version, test_type, validation_status, cost_usd to ManifestResult and DetailResult. Null-safe for older bundles.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add run_type, iter, stream to QueryTiming in detail JSON. Update transformer.py query extraction loop.", "wid": "w4"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extend DuckDB schema in duckdb_builder.py: add all new manifest columns to the results table; add run_type, iter, stream columns to queries table. Regenerate for existing corpus and verify.", "wid": "w5"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update TypeScript types.ts in results-explorer to match extended ManifestResult and DetailResult shapes. No UI changes - just type alignment.", "wid": "w6"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-run full pipeline over existing corpus; confirm no regressions in existing explorer pages; confirm new fields appear in manifest and DuckDB.", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "file follow-up items instead.", "dont": "block this item on cosmetic facet improvements", "why": "scope creep is how refactors stall"}, {"INSTEAD": "keep product surface identical.", "dont": "change visible facet behavior or labels", "why": "this is a refactor, not a redesign"}, {"INSTEAD": "delete dead branches.", "dont": "keep adapter-name-conditional fallbacks 'just in case'", "why": "the whole point of the normalized contract is to remove them"}, {"INSTEAD": "raise issues against that item if a field is missing or wrong.", "dont": "reshape the normalized contract from the explorer side", "why": "the contract is owned by results-complete-execution-environment-capture"}], "approach": "Sequence as: audit \u2192 model migration \u2192 query migration \u2192 label reconciliation\n\u2192 coverage re-validation \u2192 test update \u2192 cross-corpus check. Treat the\nmigration table from w1 as the contract for w2-w4; do not let label\ndecisions leak into the model layer. Use the normalized field names from\n`results-complete-execution-environment-capture` w1 as the authoritative\nvocabulary; the explorer is a consumer, not a co-designer of the contract.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-02T00:00:00Z", "deferrals": [], "deps": ["explorer-selection-facets-coverage-at-scale", "results-complete-execution-environment-capture"], "description": "`explorer-selection-facets-coverage-at-scale` ships the explorer facet model\n(`results-explorer/src/lib/facetModel.ts`) against current adapter fields\nbecause the normalized execution-environment contract from\n`results-complete-execution-environment-capture` is not yet available. That\nis the right sequencing call for shipping speed, but it leaves behind a\nknown refactor: facet keys like `deployment_class`, `cloud_provider`,\n`cloud_region`, `instance_or_warehouse`, `storage_format`, and `cost_status`\nmust currently be sourced from adapter-specific raw config fields with\nper-adapter fallbacks. Once `results-complete-execution-environment-capture`\nlands, those fields will be emitted as a stable normalized contract on\nevery result artifact.\n\nThis item is the loop-back: migrate the facet model and its consumers off\nthe per-adapter fallbacks and onto the normalized execution-environment\ncontract. The product surface should not change \u2014 same facet rail, same\nURL contract, same coverage signals \u2014 but the data path becomes uniform\nacross adapters and the fallback scraping code goes away.\n\nScope is intentionally narrow: facet model + DuckDB queries + immediate\nconsumers (FacetRail labels, CoverageBadge grouping). Anything outside\nthe scope_limit is a separate item.", "id": "explorer-facet-model-environment-contract-refactor", "preserves": ["Coverage badges and missing-cell rendering continue to match the pre-refactor explorer for identical inputs.", "Existing explorer URLs (shipped pre-refactor) round-trip identically through useUrlState.", "FacetRail chip labels stay user-stable (no surprise renames mid-session).", "Missing coverage continues to not penalize a platform's average rank."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/docs/facet-environment-migration.md"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/CoverageBadge.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/FacetRail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/facetModel.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/queryFilters.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/facetModel.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryFilters.ts"}], "state": "done", "title": "Explorer: refactor facet model onto normalized execution-environment contract", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Frontend typecheck and tests pass.", "expected": "All facet/coverage tests pass; pre-refactor URL round-trip regression test passes.", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:fixtures && npm run build", "description": "Browser build succeeds against the post-environment-capture corpus fixtures.", "expected": "Fixture generation, fixture verification, and Vite build complete without errors.", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 2}, {"command": "rg -n 'COALESCE|adapter ===|platform ===' results-explorer/src/lib/facetModel.ts results-explorer/src/lib/queryFilters.ts results-explorer/src/lib/duckdbQueries.ts || true", "description": "No fallback-scraping code remains in the facet path.", "expected": "No matches in facet-path files (matches in unrelated files are acceptable).", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_visible_metrics_registry.py -q", "description": "Visible metrics parity remains canonical with Python.", "expected": "Registry parity test passes.", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": [], "notes": "Walk every `FacetKey` in `results-explorer/src/lib/facetModel.ts` and\nits serializer/parser pairs. For each key sourced from adapter-specific\nraw config (deployment_class, cloud_provider, cloud_region,\ninstance_or_warehouse, storage_format, cost_status), record:\n - current source field(s) and per-adapter quirks;\n - target field on the normalized execution-environment contract\n (cross-reference `results-complete-execution-environment-capture`\n w1 schema);\n - any value-domain shifts (e.g. cloud_provider casing, region\n aliases).\nOutput: a short migration table committed to\n`results-explorer/docs/facet-environment-migration.md`. No code edits\nin this unit; downstream units consume the table.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit fallback paths in facetModel.ts and document field-by-field migration", "wid": "w1"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w1"], "notes": "Rewrite serializer/parser pairs and the `facetsToWhereClause` helper\nto read from normalized contract columns. Delete per-adapter fallback\nbranches. Preserve URL round-trip for every existing facet key \u2014 URLs\nshipped against the pre-refactor model must still parse.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate facetModel.ts onto normalized contract fields", "wid": "w2"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w2"], "notes": "Update `results-explorer/src/lib/queryFilters.ts` and\n`results-explorer/src/lib/duckdbQueries.ts` so leaderboard, benchmark,\nand platform queries reference normalized columns. Remove `COALESCE`\nand adapter-name-conditional expressions added as fallbacks during\nthe original facet rollout.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DuckDB queries to read normalized contract columns", "wid": "w3"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w2"], "notes": "Normalized contract may use canonical labels (e.g. AWS/GCP/Azure for\ncloud_provider) where the adapter-specific path used adapter-native\nstrings. Update FacetRail chip rendering and CoverageBadge grouping\nso labels remain stable for users (no chip text churn) while consuming\nthe canonical values. Add a small label-mapping module if needed; do\nnot embed mappings inline in components.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile FacetRail and CoverageBadge labels with normalized vocabulary", "wid": "w4"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w3", "w4"], "notes": "Coverage badges, missing-cell rendering, and \"average rank over\ncovered cohorts\" must continue to behave correctly when cohort\nmembership is computed from normalized fields. Confirm: missing\ncoverage still does not penalize average rank; coverage badges\nstill render accurately for the public corpus fixtures.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate coverage signals against the normalized contract", "wid": "w5"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w2", "w3"], "notes": "Update `lib/__tests__/facetModel.test.ts` and\n`lib/__tests__/queryFilters.test.ts` to assert canonical field names\nand value domains. Add a regression test that pre-refactor URLs still\nround-trip cleanly. Remove tests that exercised adapter-fallback\nbranches.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests to assert against normalized contract", "wid": "w6"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w5", "w6"], "notes": "Run the explorer fixtures end-to-end against the public-corpus shape\nthat `results-complete-execution-environment-capture` w14 establishes.\nConfirm: no facet key falls through to a fallback path; no console\nwarnings about missing normalized fields; coverage and rank surfaces\nmatch the pre-refactor explorer for identical inputs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cross-corpus verification: every result bundle facets through the normalized path", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "selection happens in the pipeline after all entries are loaded", "dont": "apply canonical-row selection inside to_manifest_entry", "why": "the transformer processes one bundle at a time"}, {"INSTEAD": "use is_ranking_eligible()", "dont": "hard-code 'maintainer-run' in query logic", "why": "the eligibility rules must be the single source of truth"}, {"INSTEAD": "add display_geomean_ms alongside it", "dont": "rename geomean_ms to sample_geomean_ms", "why": "it breaks any existing consumers reading that field"}, {"INSTEAD": "add platform_id as a new field", "dont": "strip the raw platform string from ManifestEntry", "why": "it is needed for display and debugging"}, {"INSTEAD": "use median for robustness", "dont": "use mean instead of median for display_ms reduction", "why": "a single cold-cache outlier inflates the mean"}], "approach": "All changes are additive to the read model - no existing fields are\nremoved or renamed. Add the new functions and fields, populate them\nin the existing `to_manifest_entry` and `to_detail_result` methods.\n\nThe canonical order:\n 1. Define models + constants (w3, w4) - no behavior, just schema\n 2. Define pure functions (_platform_id, _query_display_ms, select_canonical_row)\n 3. Wire into the transformer methods\n 4. Test with fixtures that exercise the edge cases\n\nDo not change how existing fields are computed. The new fields are\nderived from the same bundle data, just with a different reduction.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": [], "description": "The adversarial review (2026-04-15) identified two critical invariants\nthat must hold before any matrix leaderboard can be meaningful:\n\n**Invariant 1 - Platform identity is separate from provenance/visibility.**\nThe current pipeline writes whatever string appears in the bundle's\nplatform.name field directly to manifest.json and details/. In practice\nthis produces distinct rows for \"DuckDB-trust-ci\", \"DuckDB-trust-community\",\n\"DuckDB-trust-local\", etc., even though they are the same engine variant and\nthe trust information is already captured in the separate trust_label field.\nAny cross-platform matrix built on this read model is misleading because the\nplatform axis is polluted with provenance variants.\n\nFix: extract a canonical `platform_id` (e.g., \"duckdb\") alongside the raw\nplatform name, and define explicit ranking-eligibility rules that gate which\nrows may appear in ranked tables.\n\n**Invariant 2 - One canonical display value per logical query.**\nThe transformer preserves every measurement row (iter, stream) for provenance.\n`_geomean_ms()` therefore averages *all* execution samples, including\nthroughput streams. Meanwhile, Compare.tsx calls `find()` on the queries\narray and takes the first row - currently returning 16ms for a query whose\nsamples are [16, 11, 11, 11] (cold-cache outlier, not representative).\n\nNeither approach is wrong for its stated purpose, but neither is appropriate\nas the display value in a comparison matrix. The fix is a canonical reduction:\nmedian of successful measurement runs for each distinct logical query_id,\nexposed as `display_ms` per query and `display_geomean_ms` at the result\nlevel. The raw `geomean_ms` is preserved as an audit/detail field.\n\nThis TODO does **not** implement the matrix UI or the cohort artifact - it\nonly fixes the data contracts that every downstream feature depends on.\nIt replaces the dangling `explorer-extend-manifest-and-pipeline` dependency\nthat previously blocked `explorer-emit-benchmark-summary-artifact`.", "id": "explorer-fix-read-model-contract", "preserves": ["Existing `geomean_ms` field - raw-sample geomean preserved for backward compat; do not rename or remove", "Existing `platform` field on ManifestEntry and DetailResult - raw bundle string, unchanged for display/debugging", "Existing `queries: list[QueryTiming]` on DetailResult - raw per-sample timings preserved for audit; display_timings is additive", "Existing manifest.json schema shape - this adds fields, does not remove or rename any existing keys"], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/duckdb_builder.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/pipeline.py"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/**"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer/test_read_model_contract.py"}], "state": "done", "title": "Fix explorer read-model contract: platform/provenance split, canonical query reduction, ranking eligibility", "verifications": [{"command": "grep -n 'platform_id' benchbox/core/explorer_pipeline/models.py", "description": "platform_id field present in ManifestEntry", "expected": "field defined in ManifestEntry and DetailResult", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'display_' benchbox/core/explorer_pipeline/models.py", "description": "display_timings and display_geomean_ms present in DetailResult", "expected": "display_timings, display_geomean_ms, QueryDisplayTiming all present", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'RANKING_ELIGIBLE' benchbox/core/explorer_pipeline/models.py", "description": "RANKING_ELIGIBLE constants exported", "expected": "constants and is_ranking_eligible defined", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -n 'select_canonical_row' benchbox/core/explorer_pipeline/models.py", "description": "select_canonical_row exported", "expected": "function defined", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/explorer/test_read_model_contract.py -v", "description": "Unit tests pass", "expected": "all tests pass", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/explorer/ -v", "description": "Existing manifest tests still pass", "expected": "all tests pass", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": [], "notes": "`_platform_id(raw: str) -> str` strips known provenance suffixes\nfrom the bundle platform.name before slugifying:\n - Patterns to strip: `-trust-{ci,community,local,unknown}`,\n `-{ci,community,local}` when adjacent to a version segment.\n - After stripping, lowercase and replace spaces with dashes.\nExamples:\n \"DuckDB-trust-ci\" \u2192 \"duckdb\"\n \"DuckDB\" \u2192 \"duckdb\"\n \"polars-df\" \u2192 \"polars-df\"\n \"ClickHouse Cloud\" \u2192 \"clickhouse-cloud\"\n\nAdd `platform_id: str` to both ManifestEntry and DetailResult.\nPopulate from `_platform_id(platform)` in `to_manifest_entry` and\n`to_detail_result`. The raw `platform` field is unchanged - it\nremains the exact string from the bundle for display and debugging.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `_platform_id()` normalizer and `platform_id` field to ManifestEntry and DetailResult", "wid": "w1"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": [], "notes": "Signature:\n `_query_display_ms(timings: list[QueryTiming], query_id: str) -> float | None`\n\nAlgorithm:\n 1. Filter to timings where `timing.query_id == query_id`\n AND `timing.status == \"pass\"`.\n 2. From passing timings, select `run_type == \"measurement\"` rows;\n if none have explicit run_type, accept all.\n 3. Compute median of `duration_ms` across the filtered set.\n 4. Return None if the filtered set is empty (all failed or query\n not present).\n\nMedian is chosen over mean because a single cold-cache outlier\n(common in power-test runs that include one warmup iteration before\nmeasurement) should not dominate the display value.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define `_query_display_ms()` canonical reduction function in transformer.py", "wid": "w2"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": ["w2"], "notes": "New model in models.py:\n class QueryDisplayTiming(BaseModel):\n query_id: str\n display_ms: float | None # None = all runs failed for this query\n sample_count: int # number of passing runs that contributed\n\nAdd to DetailResult:\n display_timings: list[QueryDisplayTiming] = []\n display_geomean_ms: float | None = None # geomean of display_ms values\n\nThe existing `geomean_ms` field is NOT renamed or removed - it\nremains the raw-sample geomean for backward compatibility and audit.\nManifestEntry also gets `display_geomean_ms: float | None = None`\n(used by the matrix for ranking when power_score is absent).\n\nPopulate in `to_detail_result` and `to_manifest_entry` by calling\n`_query_display_ms()` for each distinct query_id, then computing\ngeomean of the non-None results.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add QueryDisplayTiming model and display_timings + display_geomean_ms fields to DetailResult", "wid": "w3"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": [], "notes": "Per hosted-results-contract.md: mixed self-reported rows are not\neligible for ranked tables. Codify as frozensets:\n\n RANKING_ELIGIBLE_VISIBILITIES: frozenset[str] = frozenset({\n \"public-curated\",\n \"public-verified\",\n })\n\n RANKING_ELIGIBLE_TRUST_LABELS: frozenset[str] = frozenset({\n \"maintainer-run\",\n \"ci-verified\",\n })\n\n def is_ranking_eligible(entry: ManifestEntry) -> bool:\n return (\n entry.visibility in RANKING_ELIGIBLE_VISIBILITIES\n and entry.trust_label in RANKING_ELIGIBLE_TRUST_LABELS\n )\n\nExport all three from models.py `__all__`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define RANKING_ELIGIBLE_VISIBILITIES and RANKING_ELIGIBLE_TRUST_LABELS constants in models.py", "wid": "w4"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": ["w4"], "notes": "Signature:\n `select_canonical_row(entries: list[ManifestEntry]) -> ManifestEntry | None`\n\nGiven multiple ManifestEntries for the same (platform_id, benchmark,\nscale_factor, phase), selects the single canonical row for a ranked\nmatrix display. Returns None if the list is empty.\n\nSelection priority (descending):\n 1. `is_ranking_eligible(entry)` - ranking-eligible rows win\n 2. `run_date` descending - newest run wins within the same tier\n\nThe non-canonical rows are not discarded - they remain in the\nmanifest and detail pages. Only the matrix artifact applies this\nselection when building the default ranked view.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add select_canonical_row() in models.py for matrix default ranking", "wid": "w5"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Cases:\n (a) _platform_id strips known trust suffixes correctly\n (b) _platform_id is idempotent on clean names\n (c) _query_display_ms returns median of passing runs, not first\n (d) _query_display_ms returns None when all runs failed\n (e) _query_display_ms uses measurement-only rows when run_type present\n (f) display_geomean_ms differs from geomean_ms when samples vary\n (g) is_ranking_eligible returns False for community-submission trust\n (h) select_canonical_row picks ranking-eligible over non-eligible\n (i) select_canonical_row picks newest when both are ranking-eligible\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write unit tests in tests/unit/core/explorer/test_read_model_contract.py", "wid": "w6"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use CSS custom properties that switch on a class", "dont": "duplicate the component for high-contrast mode", "why": "two components drift out of sync"}, {"INSTEAD": "fail only on serious/critical", "dont": "fail the build on minor axe violations", "why": "minor issues include color-contrast edge cases we may accept"}, {"INSTEAD": "include text and aria-label on every data-bearing element", "dont": "rely on color alone", "why": "4% of users have color vision deficiencies and screen readers ignore color entirely"}, {"INSTEAD": "use roving tabindex so tab exits the grid quickly", "dont": "use tabindex=0 on every cell", "why": "that makes keyboard nav painful (tab through 22\u00d78 cells)"}], "approach": "Accessibility work is most reliably driven by the tests themselves.\nAdd axe-core first, watch the existing tests fail, then fix\nviolations in priority order (keyboard \u2192 aria-label \u2192 contrast \u2192\nfocus visibility).\n\nFor the reduced-color mode: do not build a separate component tree.\nFlip a CSS class on the grid root; cell colors read from CSS custom\nproperties that switch value based on the class. This keeps the\nrender path single and the switch cost zero.\n\nKeyboard nav is the trickiest part. Use a roving tabindex pattern\n(only one cell has tabindex=0 at a time; arrow keys move it).\nReference pattern: WAI-ARIA Authoring Practices grid pattern.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-matrix-leaderboard-view"], "description": "The 2026-04-15 review flagged that the matrix leaderboard relies heavily\non color to convey information - a known accessibility hazard. Color is\ninvisible to screen readers and insufficient for users with color vision\ndeficiencies. The heatmap must remain readable without color.\n\nThis TODO:\n\n1. Ensures each heat cell carries a descriptive `aria-label` with the\n numeric timing and the relative ratio (\"18ms, 1.6\u00d7 fastest\").\n\n2. Adds a \"reduced color\" visual mode that falls back to monochrome\n lightness steps, controlled by the user's\n `prefers-contrast: more` media query and an explicit toggle.\n\n3. Adds keyboard navigation: tab reaches platform checkboxes,\n arrow keys navigate cells with an `aria-live` announcement.\n\n4. Backfills snapshot tests for the filter bar, empty states, and the\n integration between BenchmarkIndex and QueryHeatmap - the prior TODO\n only specified QueryHeatmap itself.\n\n5. Runs axe-core against the rendered matrix in the test suite and\n fails the build on any serious or critical violations.", "id": "explorer-heatmap-accessibility-and-tests", "preserves": ["Current default color mode - high-contrast is an opt-in, not a default swap", "Existing behavior of the matrix - accessibility is additive layering, not a rewrite", "Snapshot tests from explorer-matrix-leaderboard-view - this TODO extends, does not replace them"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/BenchmarkIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/testing/axe-helper.ts"}], "state": "done", "title": "Accessibility hardening and snapshot test coverage for matrix leaderboard", "verifications": [{"command": "cd results-explorer && npm test -- --testPathPattern='(QueryHeatmap|BenchmarkIndex|Compare|Home)'", "description": "axe-core runs clean on the matrix page (no serious/critical violations)", "expected": "all tests pass", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'aria-label' results-explorer/src/components/QueryHeatmap.tsx", "description": "Heat cells have aria-label with numeric content", "expected": "aria-label present on cell", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -nE 'high-contrast|reduced-color' results-explorer/src/components/QueryHeatmap.tsx", "description": "Reduced-color mode is CSS-class driven, not a separate component", "expected": "mode is a class modifier", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles", "expected": "no errors", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": [], "notes": "Cell: aria-label='${ms}ms, ${ratio}\u00d7 fastest in column' (or\n'no data' for null cells). Column headers: aria-sort='none'.\nRow checkboxes: aria-label='Select ${platform} for comparison'.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enhance QueryHeatmap cells, headers, and checkboxes with descriptive aria labels", "wid": "w1"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1"], "notes": "Activates when prefers-contrast: more matches OR the user toggles\n'High contrast' in the filter bar. Cells use grayscale lightness\n(0-100%) instead of hsl hue. Numeric text remains primary signal.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement reduced-color mode via CSS class + custom properties", "wid": "w2"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1"], "notes": "Arrow keys within the matrix move focus between cells; focused\ncell announces itself via aria-live. Tab exits the grid to the\nnext filter control. Focus outline is visible (not just color).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add keyboard navigation (roving tabindex) and visible focus outlines", "wid": "w3"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1"], "notes": "Cases: (a) filter changes re-fetch correct artifact URL;\n(b) empty state renders when artifact has no platforms;\n(c) list-view toggle switches rendered component;\n(d) trust filter removes rows without refetch.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write integration snapshot tests at src/pages/__tests__/BenchmarkIndex.test.tsx", "wid": "w4"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1", "w2", "w3"], "notes": "Add helper `expectNoAxeViolations(container)` and call it in\nQueryHeatmap, BenchmarkIndex, Compare, and Home page tests.\nWarn on minor; fail only on serious/critical.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate axe-core into the test suite; fail on serious/critical violations", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "scope the config per-call or per-helper", "dont": "change configureAxe to suppress impact levels globally", "why": "that silences violations in all tests"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-heatmap-accessibility-and-tests. Four Required issues\n(broken CSS rule, invisible focus ring, empty columnheader, misleading\naria-sort) were fixed immediately post-review. This item tracks\nthe remaining nit/consider findings.", "id": "explorer-heatmap-accessibility-review-followups", "preserves": ["aria-label remains present on all cells in w1 - only the text format changes", "axe helper continues to suppress minor violations - only the implementation mechanism changes in w2"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from heatmap-accessibility code review", "verifications": [{"command": "cd results-explorer && npm test -- QueryHeatmap", "description": "QueryHeatmap axe tests still pass after axe-helper refactor", "expected": "all tests pass including axe assertions", "item_id": "explorer-heatmap-accessibility-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run typecheck && npm test", "description": "Full test suite green", "expected": "all green", "item_id": "explorer-heatmap-accessibility-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "QueryHeatmap.tsx:278 - aria-label uses ms.toFixed(1) (e.g. \"10.0 ms\")\nwhile the visible cell text uses fmtMs() which emits toFixed(0)\n(\"10 ms\"). Screen readers announce a value that differs from what\nis displayed.\n\nFix: use fmtMs(ms) in the aria-label string for consistency.\n\nAlso: for the fastest cell (ratio=1.0), the label reads \"10 ms, 1.0\u00d7\nfastest in column\" which is grammatically awkward - the cell IS the\nfastest, not a comparison to it. Special-case as \"10 ms, fastest in\ncolumn\" when ratio rounds to 1.0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align aria-label text with visible cell text for timing values", "wid": "w1"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "axe-helper.ts:19-41 - The helper manually splits results.violations\ninto serious/critical vs minor arrays, reconstructs { violations:\ncritical }, and passes the partial object to toHaveNoViolations().\nThis works but relies on jest-axe's private object shape.\n\nThe idiomatic approach:\n import { configureAxe, toHaveNoViolations } from \"jest-axe\";\n const axe = configureAxe({ impactLevels: [\"serious\", \"critical\"] });\n expect(await axe(container)).toHaveNoViolations();\n\nThe helper becomes a clean two-liner; jest-axe handles the filtering\ninternally and is not coupled to the violation object shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace manual violation-split in axe-helper with configureAxe", "wid": "w2"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "QueryHeatmap.tsx:122-149 - handleCellKey handles ArrowLeft/Right/\nUp/Down but not Home/End. The WAI-ARIA 1.2 grid pattern recommends\n(not requires) Home to jump to first cell in row and End to last cell\nin row, Ctrl+Home to first cell in grid.\n\nLow priority: not a WCAG requirement for read-only data grids.\nWorth adding if AT user feedback surfaces the gap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Home/End key support to keyboard grid navigation", "wid": "w3"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "QueryHeatmap.test.tsx currently renders with no onSelectionChange, so\nthe checkbox column header (hasSelection=false) is never in the axe\nDOM. The Required fix (w3 in the main TODO) added aria-label to the\nempty
, but there is no regression test that exercises the\nselection path through axe.\n\nFix: add one axe test case in QueryHeatmap.test.tsx that passes\nselectedIds={new Set()} and onSelectionChange={() => {}} so axe runs\nagainst the full grid including the checkbox column header.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add axe test for QueryHeatmap with selection enabled", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "make the leaderboard the first usable screen.", "dont": "build a marketing landing page inside `/results/`", "why": "users came for benchmark answers"}, {"INSTEAD": "translate labels to platform, workload, latency, throughput, load time, and cost.", "dont": "copy LLM leaderboard language such as model/provider/token throughput", "why": "BenchBox benchmarks database platforms"}, {"INSTEAD": "place them as compact secondary affordances near the data.", "dont": "hide submission CTAs entirely", "why": "leaderboard attention feeds the submission flywheel"}, {"INSTEAD": "keep brand tokens and chart tokens separate.", "dont": "use a single dark/purple palette across charts and brand chrome", "why": "data encodings need semantic contrast"}], "approach": "Treat benchbox.dev as the source of brand expression and the explorer as the\ndata-dense product surface. Start by extracting the shared shell/tokens, then\nrearrange Home so leaderboard data appears before submission/list chrome. Use\ndensity from llm-stats for the table treatment, but keep BenchBox's code-first\ncredibility from the landing page. Test DOM order and labels so later visual\nedits cannot regress the first-impression hierarchy.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Redesign the Results Explorer front door and shell so it feels like part of\nthe BenchBox product rather than a separate light dashboard. The user clarified\nthat the primary audience is benchmark users choosing a database, while the\nleaderboard browser must remain attention-grabbing because it feeds the\nsubmission flywheel. The target frame is therefore:\n\n\"BenchBox Database Leaderboards: reproducible rankings for choosing OLAP\nplatforms by workload, scale, deployment, and normalized cost.\"\n\nThis item covers identity, first impression, first-viewport density, and the\nshared BenchBox narrative. It should preserve the existing home\nMetaLeaderboard work, but make it the first meaningful content rather than\nburying it below generic hero/stat chrome.", "id": "explorer-leaderboard-first-identity-unification", "preserves": ["Chart color encodings remain data-driven and accessible; brand colors must not become categorical chart colors by default.", "Recent Results remains accessible, but not ahead of the primary leaderboard on the home page.", "The MetaLeaderboard keeps Times/Ranks/Speedup modes and URL-backed filters from the completed home leaderboard work.", "The existing `/results/`, `/results/{benchmark}/`, `/results/p/{platform_id}/`, `/results/r/{result_id}`, `/results/compare`, and `/results/query` routes remain stable."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/BrandShell.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/Layout.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/LoadingSpinner.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/BrandShell.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/Layout.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.a11y.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/index.css"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Home.test.tsx"}], "state": "done", "title": "Explorer: unify Results Explorer and benchbox.dev into a leaderboard-first product identity", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test && npm run build", "description": "Frontend typecheck, unit tests, and production build pass.", "expected": "All commands pass with new Home/Layout coverage.", "item_id": "explorer-leaderboard-first-identity-unification", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium -- --grep \"Home\"", "description": "Manual browser smoke at desktop and mobile widths.", "expected": "Chromium browser smoke verifies `/results/` renders the Home shell and leaderboard; DOM-order unit tests cover first-viewport hierarchy.", "item_id": "explorer-leaderboard-first-identity-unification", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": [], "notes": "Read-only audit of `landing/` (tokens are extracted into the explorer;\n`landing/` itself is not edited in this work). Add the following CSS\ncustom properties to `results-explorer/src/index.css`:\n - surfaces: `--bb-bg-primary`, `--bb-bg-panel`, `--bb-bg-elevated`\n - text: `--bb-fg-primary`, `--bb-fg-muted`, `--bb-fg-inverse`\n - borders: `--bb-border-default`, `--bb-border-subtle`\n - accent: `--bb-accent`, `--bb-accent-hover`\n - status: `--bb-status-success`, `--bb-status-warning`,\n `--bb-status-danger`\n - code surfaces: `--bb-code-bg`, `--bb-code-fg`\n - typography: `--bb-font-sans`, `--bb-font-mono`, plus type-scale\n tokens `--bb-text-xs|sm|base|lg|xl|2xl`.\nChart palettes stay in their own `--bb-chart-*` namespace and must not\nbe aliased to brand tokens. Where landing and the existing explorer dark\ntheme disagree, defer to landing for chrome and to the explorer for\nchart/data encodings.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract shared BenchBox brand tokens from landing page into explorer theme variables", "wid": "w1"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w1"], "notes": "In results-explorer/src/components/Layout.tsx, add a compact global nav\nmatching benchbox.dev: BenchBox wordmark/name, Docs, Blog, Results, GitHub,\nand primary submit/run CTA. Below it, keep explorer-specific navigation\nfor Leaderboards, Benchmarks, Platforms, Compare, and Query. Results should\nbe visibly active when on `/results/`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace generic explorer header with shared global BenchBox navigation plus explorer subnav", "wid": "w2"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w1", "w2"], "notes": "In results-explorer/src/pages/Home.tsx, first viewport should contain:\n - H1: \"BenchBox Database Leaderboards\"\n - supporting line: \"Reproducible OLAP benchmark rankings by workload,\n scale, deployment, and normalized cost.\"\n - one dense cohort selector row: benchmark, scale factor, phase,\n deployment/cost coverage.\n - the MetaLeaderboard table immediately visible above the fold.\nUse a dark technical band or code-led surface from benchbox.dev only as\nframing; do not let decorative copy push the leaderboard down.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rewrite Home first viewport around a live leaderboard, not a brochure hero", "wid": "w3"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w3"], "notes": "Replace ambiguous stat cards with explicit labels such as:\n - \"22 supported benchmarks\"\n - \"N public result bundles\"\n - \"N platforms with public results\"\n - \"PR-validated corpus\"\nAvoid implying every supported benchmark has public leaderboard coverage.\nIf a value is unavailable from DuckDB metadata, compute it from existing\nmanifest/cohort tables or label it as \"supported\" rather than \"published\".\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Clarify supported vs published corpus stats in the home proof strip", "wid": "w4"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w3"], "notes": "Add a compact three-step strip or inline CTA cluster near the leaderboard:\n\"Run a benchmark\", \"Compare your result\", \"Submit a bundle\". Each should\nlink to the existing docs/submission flow where available. This should be\nsecondary to the leaderboard, not a marketing section.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Run -> Compare -> Submit flywheel affordance without reducing data density", "wid": "w5"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w1", "w3"], "notes": "This item OWNS the skeleton design system: token names, animation,\nsizing scale, and the Home + Layout consumers. The downstream\n`explorer-performance-perceived-latency` item then applies the same\nskeletons to MetaLeaderboard/BenchmarkIndex/Query/Compare without\nredesigning them. Replace generic spinner-only states on Home and the\nshell with dense skeletons that reserve final table geometry. Loading\nshould communicate that a static DuckDB snapshot is initializing,\nwithout blocking the brand/header render.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define skeleton tokens and Home/Layout loading states", "wid": "w6"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w2", "w3", "w4", "w5"], "notes": "Replace any reliance on manual smoke with mechanical assertions in\n`Home.test.tsx` and `Layout.test.tsx`:\n - The MetaLeaderboard region appears before any \"Recent Results\" or\n submission CTA region in DOM order (`compareDocumentPosition` /\n `getAllByRole` index check).\n - The H1 text is exactly \"BenchBox Database Leaderboards\".\n - Global nav exposes Docs, Blog, Results, GitHub, and the run/submit\n CTA; the Results entry is visibly active when the route is `/`.\n - Stat cards use explicit `supported` / `published` labels (no\n ambiguous \"benchmarks\" copy).\nThese assertions back the manual browser smoke described under\n`verification`; the DOM-order guarantee is what stops a future visual\nedit from regressing first-impression hierarchy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add automated DOM-order tests for first-viewport leaderboard hierarchy", "wid": "w7"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "extend in place", "dont": "fork MetaLeaderboard into a separate component", "why": "the shapes are the same post-widening"}, {"INSTEAD": "call it", "dont": "introduce a new color function", "why": "colorForCell is the shared log-scale palette"}, {"INSTEAD": "keep it local", "dont": "push filter state to a global store", "why": "per-page useUrlState is sufficient"}], "approach": "The pipeline-widening item is the enabling dependency; this item\nis pure frontend wiring on top of the widened MetaRank. Extend in\nplace rather than introducing a sibling MasterLeaderboard component\n(per plan) since the data shape already carries everything\nneeded. Reuse chartMath.colorForCell and QueryHeatmap's a11y\npattern - no new primitives. Default render mode on first visit:\nTimes, so the user immediately sees absolute values (the missing\nsurface this phase exists to restore).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-17T00:00:00Z", "completed_pr": null, "created_at": "2026-04-17T00:00:00Z", "deferrals": [], "deps": ["explorer-pipeline-meta-leaderboard-widening", "explorer-url-synced-filter-state"], "description": "Phase B of the Results Explorer strategy revision\n(plans/purring-jingling-petal-ultraplan).\n\nReplace the current Home page's rank-only `MetaLeaderboard` with a\nClickBench-style Platform \u00d7 Cohort matrix of absolute values, with\ntoggles for Ranks and Speedup-vs-cohort-best, chip-based filters\n(benchmark family, SF, phase, tuning, trust tier, date window), and\nURL-synced state so views are shareable.\n\nThe \"Recent Results\" list stays (or moves below the matrix) - the\ngap being filled is the lack of a cross-benchmark, cross-platform\nhigh-level view with absolute values. This is the first thing a\nuser should see.\n\nDepends on:\n - explorer-pipeline-meta-leaderboard-widening (MetaRank must carry\n metric_value, speedup_vs_best, primary_metric, primary_order)\n - explorer-url-synced-filter-state (useUrlState hook)", "id": "explorer-master-leaderboard-home", "preserves": ["Existing MetaLeaderboard consumers (if any outside Home) continue to work - render-mode toggle defaults to current behavior (Ranks)", "Recent Results data surface remains accessible on Home (just demoted below the matrix)", "colorForCell remains the single log-scale color source shared with QueryHeatmap - no duplicate palettes"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.a11y.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Explorer Home: ClickBench-style master leaderboard (Times/Ranks/Speedup, chip filters, URL state)", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Explorer typecheck + Vitest suites green", "expected": "all green, including MetaLeaderboard.a11y and filter wiring tests", "item_id": "explorer-master-leaderboard-home", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run dev", "description": "End-to-end smoke on /results/", "expected": "/results/ shows master matrix; Times/Ranks/Speedup toggle works; applying benchmark + SF chips narrows columns/rows; URL round-trips; clicking a cohort header goes to /results/{benchmark}/?sf=X&phase=Y; row click goes to /results/p/{platform_id}/", "item_id": "explorer-master-leaderboard-home", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": [], "notes": "results-explorer/src/components/MetaLeaderboard.tsx - extend in\nplace (per plan: data shape already carries what each mode needs\nafter the pipeline widening). Modes:\n - Times: metric_value formatted via fmtGeomean (asc) / fmtScore (desc)\n - Ranks: current behavior (rank / total)\n - Speedup: speedup_vs_best, cohort best = 1.0\u00d7\nLog-scale color shading via existing colorForCell from\nsrc/lib/chartMath.ts (single source of truth, shared with\nQueryHeatmap).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend MetaLeaderboard component with render-mode toggle (Times/Ranks/Speedup)", "wid": "w1"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": [], "notes": "results-explorer/src/types.ts - add metric_value,\nspeedup_vs_best, primary_metric, primary_order to MetaRank.\nAll optional so the type survives older artifacts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Widen MetaRank frontend type in types.ts", "wid": "w2"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w1", "w2"], "notes": "Filters: Benchmark family (multi), SF (multi), Phase\n(power/standard/throughput), Tuning mode, Trust tier, Date\nwindow. Each backed by useUrlState with a suitable serde\n(arraySerde for multis, stringSerde for singles).\nFilter state feeds both the visible cohort columns and the\nvisible platform rows.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sticky filter chip row backed by useUrlState", "wid": "w3"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w3"], "notes": "results-explorer/src/pages/Home.tsx - make the matrix the\nprimary content surface. Do not wire cohort/row navigation yet\n(w6 handles that). Do not change the Recent Results position\nyet (w5 handles that). Scope: layout only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Scaffold Home.tsx around the extended MetaLeaderboard as primary content", "wid": "w4"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w4"], "notes": "Keep the Recent Results component and data surface; move its\nrender position below the matrix on Home.tsx. No behavioral\nchange to the list itself.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Demote Recent Results list below the matrix", "wid": "w5"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w4"], "notes": "Cohort headers \u2192 /results/{benchmark}/?sf=X&phase=Y (preserve\nthe relevant slice of filter state via useUrlState's serde).\nRow click \u2192 /results/p/{platform_id}/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire cohort header links and row-click navigation with preserved filter state", "wid": "w6"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w4"], "notes": "The new master matrix is structurally similar to QueryHeatmap -\nreuse the same a11y scaffolding (role=grid, aria-rowindex/col,\nroving tabindex). Existing tests like\nQueryHeatmap.a11y should continue to pass; author a sibling\nMetaLeaderboard.a11y test covering the equivalent contract.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Accessibility: reuse QueryHeatmap's role=grid + roving tabindex pattern", "wid": "w7"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Vitest coverage for render-mode toggle, filter\u2192matrix wiring, cohort-link state preservation", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "conflate warnings with trust labels", "instead": "separate concerns, separate UI elements", "why": "comparability is about methodology, trust is about provenance"}, {"dont": "hard-block on execution_mode or tuning_mode mismatch", "instead": "warn and allow", "why": "intentional cross-mode comparisons are valid analytical use cases"}, {"dont": "show warnings for equal values", "instead": "only warn when the dimension actually differs", "why": "that would be noise"}], "approach": "Comparability check runs client-side after both DetailResult objects are\nloaded. Returns a list of warnings; each warning renders as a dismissible\nbanner above the chart area.\n\nWarning text should explain the implication, not just state the fact.\n\"Execution mode differs: sql vs dataframe\" is insufficient - add\n\"results may not reflect platform SQL performance differences\".\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": ["explorer-add-tuning-config-visibility", "explorer-extend-manifest-and-pipeline"], "description": "The strategy doc (\u00a7 Trust, Visibility, and Ranking) lists the dimensions\non which two results are comparability-breaking. Phase 1 enforces\nhard-blocks on benchmark and scale factor mismatch. Execution mode and\ntuning mode are also comparability-breaking - a SQL vs DataFrame\ncomparison or a tuned vs notuning comparison can look like a platform\nperformance difference when it is actually a configuration difference.\n\nPhase 1 couldn't enforce these because execution_mode and tuning_mode\nweren't in the manifest. Once explorer-extend-manifest-and-pipeline\nlands, the Compare view can inspect them and warn.\n\nThis TODO adds visible warnings (not hard-blocks) when:\n - The two results differ on execution_mode (SQL vs DataFrame)\n - The two results differ on tuning_mode (tuned vs notuning)\n - The two results use different query subsets (full vs partial)\n\nWarnings are prominent (banner at top of compare view) but not blocking -\nintentional cross-tuning comparisons are valuable.", "id": "explorer-comparability-warnings", "preserves": ["Compare view still renders when warnings are present", "Cross-tuning and cross-mode comparisons are allowed (not blocked) - warning is advisory, not preventive", "Existing hard-block on benchmark mismatch and scale-factor mismatch - these remain blocking"], "prior_art": [], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/ (no pipeline changes)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ComparabilityBanner.tsx (new)"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}], "state": "done", "title": "Add comparability warnings to Compare view for cross-mode and cross-tuning comparisons", "verifications": [{"command": "grep -n 'comparability\\|ComparabilityWarning\\|execution_mode.*warn\\|tuning_mode.*warn' results-explorer/src/pages/Compare.tsx", "description": "Comparability check function present", "expected": "check logic present", "item_id": "explorer-comparability-warnings", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'benchmark.*mismatch\\|scale_factor.*mismatch\\|hard.block\\|Cannot compare' results-explorer/src/pages/Compare.tsx", "description": "Hard-blocks still enforced", "expected": "hard-block logic still present", "item_id": "explorer-comparability-warnings", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles cleanly", "expected": "(no errors)", "item_id": "explorer-comparability-warnings", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-comparability-warnings", "needs": [], "notes": "Dimensions to check:\n - execution_mode: warn if \"sql\" vs \"dataframe\"\n - tuning_mode: warn if \"tuned\" vs \"notuning\"\n - query subset: warn if query_count differs and both are paths", "instead": "keep detail URLs long-form", "why": "that breaks stable detail-page URLs"}, {"dont": "reject unknown IDs", "instead": "unknown IDs fall through to normal 404 handling", "why": "a stale bookmark should still try to resolve gracefully"}, {"dont": "use a random or timestamp-based short ID", "instead": "use a deterministic hash", "why": "it would change on each pipeline run and break shared links"}], "approach": "Hashing is cheap and deterministic; use sha256 so short IDs are\nstable across pipeline runs. The collision-extension rule is the\nonly nuance - implement it as a second pass over the first-pass\ndict: sort entries, find collisions, extend both sides by 2 chars\nuntil unique, repeat.\n\nThe frontend load pattern: fetch short_ids.json on first navigation\nto /results/compare, cache in module scope. For the\nMatrix \u2192 Compare flow, the short_id is already on each PlatformRow\nso no extra fetch is needed.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-matrix-leaderboard-view"], "description": "The current Compare URL format is\n`/results/compare?ids=,,...`.\n\nFull result IDs average ~40 characters (e.g.,\n`tpch-duckdb-sf0.01-20260403-7fe93365`). With 4+ selections plus the\nbase URL, encoded commas, and any future additional query params, the\nURL approaches 2KB. Chrome handles this, but proxies, older browsers,\nand CDN caches have been observed to truncate or reject URLs above\n1024 bytes. Once the matrix leaderboard ships and users can\none-click-compare 6-8 platforms, this becomes a real failure mode.\n\nThis TODO introduces a short-ID scheme: a stable 8-character prefix\nof a deterministic hash of the full result_id. The pipeline emits a\nlookup table `short_ids.json` mapping short \u2192 full; the frontend\nresolves both directions.\n\nCollision handling: if two short IDs collide in the active corpus,\nthe pipeline extends the colliding pair to 10 chars (and so on). This\nis logged at build time. At current corpus scale (12 bundles) and\nforeseeable scale (hundreds), 8 chars is effectively collision-free,\nbut the extend-on-collision rule is required for robustness.", "id": "explorer-compare-url-short-ids", "preserves": ["Direct links to /results/r/ remain unchanged - this TODO only affects Compare query params", "Existing long-form Compare URLs continue to work - backward compatible", "Pipeline remains a single pass - short-ID generation piggybacks on the existing bundle-id enumeration"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/explorer/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/short_ids.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Use short hash IDs in Compare URLs to avoid URL-length limits", "verifications": [{"command": "test -f results-explorer/public/data/short_ids.json && echo exists", "description": "short_ids.json is emitted by the pipeline", "expected": "exists", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import json; d = json.load(open(\"results-explorer/public/data/short_ids.json\")); assert all(len(k) >= 8 for k in d); assert len(set(d.keys())) == len(d)'", "description": "Short IDs are 8 chars by default with no collisions at current corpus size", "expected": "no assertion failure", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'resolveShortId' results-explorer/src/pages/Compare.tsx", "description": "Compare page accepts short IDs", "expected": "function is called", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -n 'short_id' results-explorer/src/components/QueryHeatmap.tsx", "description": "Matrix \u2192 Compare URLs emit short IDs", "expected": "short_id referenced in URL construction", "item_id": "explorer-compare-url-short-ids", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": [], "notes": "For each result_id, compute sha256(result_id)[:8]. Build a dict\n{short: full}; on collision, extend both entries to 10 chars,\nthen 12, up to 16. Emit to\nresults-explorer/public/data/short_ids.json.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add short_id generation to benchbox/core/explorer/pipeline.py with collision extension", "wid": "w1"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w1"], "notes": "Lets the frontend read short_ids from the matrix artifact directly,\nno second lookup needed for the common case.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `short_id` field to each PlatformRow in BenchmarkSummary", "wid": "w2"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w1"], "notes": "Load short_ids.json on first call, cache in module scope. Unknown\nIDs return input unchanged so stale bookmarks fall through to\nnormal 404 handling.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add resolveShortId/resolveFullId helpers in src/lib/manifest.ts", "wid": "w3"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w2", "w3"], "notes": "Long-form URLs remain valid - this is additive.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit short IDs in Compare URLs from QueryHeatmap; accept either form in Compare.tsx", "wid": "w4"}, {"evidence": null, "item_id": "explorer-compare-url-short-ids", "needs": ["w4"], "notes": "When Compare.tsx receives any full IDs, replace the URL with the\ncanonical short-ID form so shared links propagate compact form.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add URL canonicalization in Compare.tsx via history.replaceState", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "hide the Query Workbench View link when result_id is not visible", "instead": "carry row identity separately from displayed columns.", "why": "navigation is a core row action"}, {"dont": "label measurement samples as queries", "instead": "show logical queries and samples as separate counts.", "why": "that overstates benchmark breadth"}, {"dont": "normalize mixed benchmark/scale rows into a single trend line", "instead": "split by cohort or force a cohort selector.", "why": "geomean values are cohort-relative"}], "approach": "Fix each defect in the smallest owning component first, then add tests around\nthe observable behavior. For Query Workbench, avoid making `result_id` visible\nby default just to fix navigation; separate row identity from displayed\ncolumns. For Result Detail, audit all methodology labels that mention query\nscope so the copy cannot reuse sample-count terminology. For Platform trend,\nuse the existing ranking/cohort fields already emitted by\n`bench.benchmark_rankings` rather than inventing a frontend-only cohort key.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-01T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Fix the correctness defects found while auditing the BenchBox Results Explorer\nbefore layering on larger visual and IA changes. These issues directly break\nnavigation or misrepresent benchmark data:\n\n- Query Workbench default `View ->` links can navigate to `/results/r/undefined`\n because the default visible column set omits `result_id` while the row action\n reads `row.result_id`.\n- Result detail methodology can label sample count as query scope, e.g. showing\n \"Query scope: 88 queries\" on a page whose timing section has 22 logical\n queries and 88 individual samples.\n- Platform page \"Performance Trend\" can mix unrelated benchmark/scale cohorts\n such as SSB SF0.01, SSB SF0.1, TPC-H SF0.01, and TPC-H SF0.1 into one\n geomean trend line, which implies a single longitudinal metric where none\n exists.", "id": "explorer-critical-results-ux-defects", "preserves": ["Existing Result Detail route `/results/r/{result_id}` remains unchanged.", "Platform trend charts remain available, but only for comparable cohorts.", "Query Workbench CSV/JSON export continues to emit exactly the columns the user has selected (no extra hidden `result_id`); the projection-only fix keeps `result_id` available to row actions without leaking into export output.", "Query Workbench column picker continues to control visible table columns."], "prior_art": [], "priority": "critical", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/components/MethodologyDisclosure.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/TimeSeries.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MethodologyDisclosure.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryFilters.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Query.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/PlatformIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Query.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/ResultDetail.test.tsx"}], "state": "done", "title": "Explorer: fix critical result-browser defects found in design audit", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Frontend typecheck and unit tests pass.", "expected": "TypeScript and Vitest pass, including new regressions for workbench links, methodology counts, and platform trend grouping.", "item_id": "explorer-critical-results-ux-defects", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run build", "description": "Production explorer bundle still builds.", "expected": "Vite production build completes without errors.", "item_id": "explorer-critical-results-ux-defects", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": [], "notes": "In results-explorer/src/pages/Query.tsx, ensure `result_id` is always\navailable for row actions even when not visible. Preferred fix: keep\n`result_id` in the SELECT projection as a hidden key and render only the\nuser-selected columns. Add a regression test where DEFAULT_COLUMNS omits\n`result_id` but the \"View\" link still points to `/results/r/{id}`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make Query Workbench row actions independent of visible result_id column", "wid": "w1"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": [], "notes": "In results-explorer/src/pages/ResultDetail.tsx and/or\nMethodologyDisclosure, source logical query scope from `detail.query_count`\nor `detail.display_timings.length`, and sample count from\n`detail.queries.length`. The UI should say, for example, \"22 queries,\n88 measurement samples\" rather than \"88 queries\" when samples are the\nlarger number.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Correct Result Detail methodology copy to distinguish logical queries from samples", "wid": "w2"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": [], "notes": "In results-explorer/src/pages/PlatformIndex.tsx and\nresults-explorer/src/components/TimeSeries.tsx call sites, group trend\ndata by `(benchmark, scale_factor, phase, primary_metric)` before drawing.\nEither render cohort-specific small multiples or require an explicit cohort\nselector. Do not plot mixed cohorts on one y-axis.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stop Platform page trend chart from combining different cohorts into one time series", "wid": "w3"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": ["w1"], "notes": "Extend Query.test.tsx with a case where DEFAULT_COLUMNS omits `result_id`\nbut the rendered \"View\" link still resolves to `/results/r/{id}`. Commit\nwith the w1 fix so navigation is locked in by the same change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Query Workbench regression test for default row link without visible result_id", "wid": "w4"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": ["w2"], "notes": "Extend ResultDetail.test.tsx or MethodologyDisclosure.test.tsx to assert\n\"queries\" and \"measurement samples\" render as distinct counts when sample\ncount exceeds logical query count. Commit with the w2 fix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Result Detail regression test for separate query and sample counts", "wid": "w5"}, {"evidence": null, "item_id": "explorer-critical-results-ux-defects", "needs": ["w3"], "notes": "Extend PlatformIndex.test.tsx with rows from at least two distinct\n`(benchmark, scale_factor)` cohorts and assert the trend renderer either\nsplits cohorts into small multiples or refuses to render a single merged\nline. Commit with the w3 fix.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Platform trend regression test against mixed cohorts", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "hide the methodology panel behind a 'why' link", "instead": "show it inline below the table", "why": "the whole point is transparency"}, {"dont": "include cohorts with only 1 platform", "instead": "filter them out at pipeline time", "why": "rank is meaningless there"}, {"dont": "invent a weighted composite score", "instead": "use simple rank aggregation with all inputs visible", "why": "any weighting is indefensible and masks the inputs"}, {"dont": "penalize platforms that didn't run a cohort", "instead": "show N/A and exclude from that platform's average", "why": "missing data is not a loss"}], "approach": "The pipeline already iterates through every bundle once. After the\nper-benchmark artifacts are built, iterate the in-memory accumulators\nagain to compute ranks: for each cohort, sort by the cohort's primary\nmetric and assign 1-based ranks. Aggregate across cohorts per engine.\n\nFrontend is a pure table render. No interactivity in the first cut -\nclicking a cohort header navigates to the matrix leaderboard for that\ncohort.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-emit-benchmark-summary-artifact", "explorer-matrix-leaderboard-view"], "description": "The 2026-04-15 strategy review observed that BenchBox's multi-benchmark\nstory is a real differentiator vs ClickBench but the current Home page\ndoes not lean into it. A cross-benchmark meta-leaderboard answers\n\"which engines consistently do well across benchmarks?\" - a question\nno single-benchmark site can answer.\n\nApproach: compute per-engine performance *rank* within each\n(benchmark, scale, phase) cohort, then surface aggregate rank across\ncohorts. Ranks normalize away the incompatible units (power_score vs\ngeomean_ms) that make direct cross-benchmark comparison impossible.\n\nVisualization: a compact table on the home page with columns per\nbenchmark cohort and a cell showing the engine's rank (e.g., \"1/4\")\nplus a summary \"Average rank: 1.5\" column on the right.\n\nNot included: any claim that the meta-leaderboard is an official or\nweighted score. It is a transparent rank-aggregation across whatever\ncohorts the corpus contains, with all inputs visible and linkable.", "id": "explorer-cross-benchmark-meta-leaderboard", "preserves": ["Home page remains navigable - the meta-leaderboard is additive, not replacing global nav", "No invented composite scores - only transparent rank aggregation", "Recent Results section remains on the home page in a secondary position"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/meta_leaderboard.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}], "state": "done", "title": "Add cross-benchmark meta-leaderboard to the home page", "verifications": [{"command": "test -f results-explorer/public/data/meta_leaderboard.json && echo exists", "description": "meta_leaderboard.json is emitted", "expected": "exists", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -c 'import json; d = json.load(open(\"results-explorer/public/data/meta_leaderboard.json\")); assert all(len(c[\"platforms\"]) >= 2 for c in d[\"cohorts\"])'", "description": "Meta-leaderboard only includes cohorts with \u22652 platforms", "expected": "no assertion failure", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'MetaLeaderboard' results-explorer/src/pages/Home.tsx", "description": "Home page renders MetaLeaderboard", "expected": "imported and rendered", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles", "expected": "no errors", "item_id": "explorer-cross-benchmark-meta-leaderboard", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": [], "notes": "Scan every BenchmarkSummary and emit\nresults-explorer/public/data/meta_leaderboard.json with:\n {cohorts: [{benchmark, scale, phase,\n platforms: [{platform, rank, total}]}],\n summary: [{platform, avg_rank, appearances}]}\nExclude cohorts with only 1 platform.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add MetaLeaderboard computation to pipeline.py; emit meta_leaderboard.json", "wid": "w1"}, {"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": ["w1"], "notes": "Table: one column per cohort (abbreviated header 'TPC-H SF0.01\npwr'), cells show rank fraction. Rightmost column 'Avg Rank';\nrows sorted by avg_rank ascending.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build MetaLeaderboard component at src/components/MetaLeaderboard.tsx", "wid": "w2"}, {"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": ["w2"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate MetaLeaderboard into Home.tsx as hero panel; demote Recent Results", "wid": "w3"}, {"evidence": null, "item_id": "explorer-cross-benchmark-meta-leaderboard", "needs": ["w2"], "notes": "Explain: (a) rank is within-cohort; (b) only cohorts with \u22652\nplatforms count; (c) platforms missing from a cohort are N/A, not\npenalized; (d) avg_rank is a simple mean across appearances;\n(e) link to the full methodology discussion.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add transparency disclosure panel below the meta-leaderboard", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "try to make frame-ancestors work by keeping it in meta", "instead": "(not specified)", "why": "the CSP spec ignores it there; the fix is a header at the host layer, out of this file's scope."}], "approach": null, "blocked_reason": null, "category": "Security", "claimed_at": null, "claimed_by": null, "completed_at": "2026-07-06T00:00:00Z", "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": [], "description": "Source: results-explorer/publication remediation re-review (2026-07-06),\nDefect #5 (de-escalated to cosmetic).\n\nresults-explorer/index.html:15 sets the CSP via ``, and its content includes `frame-ancestors 'none'`.\nPer the CSP spec, `frame-ancestors` is IGNORED when the policy is delivered in\na element \u2014 it is only honoured as an HTTP response header. So that\ndirective provides no clickjacking protection today; it reads as a control\nthat does not exist.\n\nThis is cosmetic, NOT a security hole: the load-bearing directive\n`connect-src 'self'` (which confines all network egress and closes the\nduckdb-wasm remote-fetch/exfil channel) DOES work in a meta CSP, and it is\npresent. The only issue is the misleading inert `frame-ancestors` line.\n\nFix: drop the inert directive and replace it with a comment stating that\nframing/clickjacking protection must be applied as an HTTP response header\n(`X-Frame-Options: DENY` or a header-delivered `frame-ancestors`) at the\nhosting/CDN layer (GitHub Pages), because meta CSP cannot express it.", "id": "explorer-csp-frame-ancestors-meta-cleanup", "preserves": ["The duckdb-wasm-compatible directives (script-src 'wasm-unsafe-eval', worker-src blob:) \u2014 removing these would break the explorer.", "connect-src 'self' \u2014 the load-bearing egress confinement; must not change."], "prior_art": [{"concept": "_project/DONE/main/planning/remediate-explorer-csp-hardening.yaml \u2014 the item that added the CSP; SUPERSEDE its noted 'frame-ancestors inert in meta' caveat by actually removing the directive.", "decision": "supersede", "path": "_project/DONE/main/planning/remediate-explorer-csp-hardening.yaml"}, {"concept": "results-explorer/index.html:6-15 \u2014 the existing CSP meta + head comment; EDIT in place, preserve all non-frame-ancestors directives.", "decision": "reuse", "path": "results-explorer/index.html:6-15"}], "priority": "low", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/"}, {"kind": "only_modify", "path_glob": "results-explorer/index.html"}], "state": "done", "title": "Remove the inert frame-ancestors directive from the explorer CSP meta tag", "verifications": [{"command": "grep -oE 'content=\"default-src[^\"]*\"' results-explorer/index.html | grep -c 'frame-ancestors'", "description": "frame-ancestors is gone from the CSP content= directive string (a comment mentioning it is fine)", "expected": "0 (the directive is removed from the meta content string)", "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -oE 'content=\"default-src[^\"]*\"' results-explorer/index.html | grep -c \"connect-src 'self'\"", "description": "the load-bearing connect-src 'self' directive is preserved", "expected": "1 (egress confinement unchanged)", "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run test:e2e:chromium 2>&1 | tail -5", "description": "the explorer still builds and the browser e2e suite passes (no CSP regression; connect-src still enforced)", "expected": "all specs pass \u2014 removing an inert directive is behaviorally a no-op, so this confirms nothing else broke", "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-csp-frame-ancestors-meta-cleanup", "needs": [], "notes": "In results-explorer/index.html: remove `frame-ancestors 'none'` from the\n`` content string, leaving every\nother directive byte-identical (default-src, script-src incl.\n'wasm-unsafe-eval', worker-src blob:, connect-src 'self', style-src,\nimg-src, font-src, object-src 'none', base-uri 'self'). Add an HTML comment\nnear the tag noting that clickjacking protection needs an HTTP response\nheader at the host/CDN because meta CSP ignores frame-ancestors. If the head\ncomment (index.html:6-7) implies framing is covered, correct it too.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Drop the inert frame-ancestors from the meta CSP; document the header-layer requirement", "wid": "w1"}], "worktree": "main"} +{"anti_patterns": [{"dont": "couple this work to the schema-versioning refactor in TODO `explorer-read-model-schema-versioning`. That work changes the error shape; this TODO changes when the error fires. Both can land in either order.", "instead": "(not specified)", "why": "(unstated)"}, {"dont": "remove the runtime guard in `db.ts` that detects schema mismatch", "instead": "(not specified)", "why": "the schema mismatch is real and should still produce an actionable error, just not as the *default* dev experience."}, {"dont": "track the full-corpus `results-explorer/public/data/results.duckdb` in git. The whole point of (ii) is a tiny dev-only snapshot, not the production artifact.", "instead": "(not specified)", "why": "(unstated)"}], "approach": "Pick one solution shape, implement it minimally, verify with a destructive\nsmoke (rm + npm run dev). Avoid adding configurability that this TODO\ncannot motivate \u2014 if (ii) is chosen, do NOT also implement (i) \"just in\ncase\".\n", "blocked_reason": null, "category": "Developer Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-14T00:00:00Z", "completed_pr": null, "created_at": "2026-07-18T19:50:09Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-dev-snapshot-dx", "id": 164, "reason": "Owned by TODO `explorer-read-model-schema-versioning`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] superseded: Owning TODO explorer-read-model-schema-versioning is now in _project/DONE; this note's fate was resolved there, not left open.", "summary": "Make the error message itself humane (version-diff vs column-diff)."}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-dev-snapshot-dx", "id": 165, "reason": "Owned by TODO `explorer-cli-surface-migration`.", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] obsolete: Owning TODO explorer-cli-surface-migration is DONE and its w2 'Relocate the CLI entry point per ADR' work unit is status:done; the rename already happened.", "summary": "Rename the CLI entry that the docs reference."}], "deps": [], "description": "A fresh-clone or returning developer runs `cd results-explorer && npm run\ndev` and is met with a full-screen error wall:\n\n DuckDB snapshot 'bench.results' is missing required columns: ...\n Regenerate the snapshot via 'benchbox explorer build' or check the\n deployed results.duckdb file.\n\nRoot cause is not a code bug \u2014 it is an architectural papercut:\n - `*.duckdb` is gitignored, so the snapshot is never tracked.\n - CI rebuilds it on every docs deploy\n (`.github/workflows/docs.yml:103`); local devs never do unless they\n know to.\n - `npm run dev` has no predev step that detects staleness.\n - The runtime guard in `results-explorer/src/db.ts:184-199` correctly\n refuses to render against a snapshot whose schema doesn't match the\n frontend's required-column set \u2014 so the user sees a 9-column-diff\n wall the first time the read-model contract bumps.\n\nBecause the read-model contract is changing actively (PRs #344, #354,\n#380 added eligibility columns in the last 30 days), this wall recurs\nevery few weeks for every developer.\n\nThis TODO closes the recurring DX wall. It is separable from the CLI\nsurface relocation (TODO `explorer-cli-surface-migration`) and the\nschema versioning work (TODO `explorer-read-model-schema-versioning`).\n\nThree solution shapes to evaluate:\n\n (i) **`predev` npm script** in `results-explorer/package.json` that\n checks whether `public/data/results.duckdb` is older than the\n most-recent file under `results-data/bundles/`, and rebuilds via\n the canonical entry point if so. Pros: zero new files; works with\n the existing build step. Cons: every `npm run dev` runs an mtime\n check; spawns the Python entry point when rebuilding (3-5s on\n the 528-bundle corpus).\n\n (ii) **Tiny in-tree dev snapshot** under\n `results-explorer/dev-data/results.duckdb` (5-10 cohorts,\n deterministic), NOT gitignored, that the explorer falls back to\n when `public/data/results.duckdb` is missing. Pros: `git clone\n && npm run dev` works instantly with no Python. Cons: adds a\n tracked binary (~500KB-2MB depending on corpus) that needs\n periodic refresh \u2014 but only when the schema bumps, not the data.\n\n (iii) **`make explorer-dev` wrapper target** that runs rebuild then\n vite. Pros: explicit, documented in Makefile. Cons: requires\n developer to know about and use the make target; doesn't help\n someone who runs `npm run dev` from inside `results-explorer/`.\n\nThe right choice depends on how often the schema bumps vs. how often\nthe data bumps; (ii) optimizes for \"schema-stable, data-shifting\" and\n(i) optimizes for \"schema-shifting, data-stable\". (iii) is the most\nconservative and explicit but doesn't fix the default `npm run dev`\npath.", "id": "explorer-dev-snapshot-dx", "preserves": ["Browser-fixture generation (`generate-browser-fixtures.mjs`) is unaffected \u2014 it has its own snapshot regen.", "Production builds (CI docs deploy) continue to use the full corpus, not the dev snapshot or fallback.", "`results-explorer/public/data/results.duckdb` semantics are unchanged when present."], "prior_art": [{"concept": "Makefile \u2014 existing target conventions like `uat-explorer-smoke`; pattern for a possible `explorer-dev` target.", "decision": "reuse", "path": "Makefile"}, {"concept": "results-data/bundles/ \u2014 source-of-truth bundle dir whose mtime can drive a staleness check.", "decision": "reuse", "path": "results-data/bundles/"}, {"concept": "results-explorer/package.json \u2014 npm scripts site; reuse existing `dev` / `build` patterns for any new predev hook.", "decision": "reuse", "path": "results-explorer/package.json"}, {"concept": "results-explorer/public/data/results.duckdb \u2014 current deployed snapshot (gitignored); a dev-data fallback would mirror this layout.", "decision": "reuse", "path": "results-explorer/public/data/results.duckdb"}, {"concept": "results-explorer/scripts/generate-browser-fixtures.mjs \u2014 existing pattern for spawning the Python publish toolchain from Node.", "decision": "reuse", "path": "results-explorer/scripts/generate-browser-fixtures.mjs"}, {"concept": "results-explorer/scripts/serve-browser-tests.mjs \u2014 existing Node-side wrapper around the browser fixture flow; pattern for a Node helper that orchestrates Python + Vite.", "decision": "reuse", "path": "results-explorer/scripts/serve-browser-tests.mjs"}], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": ".gitignore"}, {"kind": "only_modify", "path_glob": "CONTRIBUTING.md"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docs/development/browser-test-architecture.md"}, {"kind": "only_modify", "path_glob": "results-data/dev-corpus/**"}, {"kind": "only_modify", "path_glob": "results-explorer/README.md"}, {"kind": "only_modify", "path_glob": "results-explorer/dev-data/**"}, {"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}], "state": "done", "title": "Eliminate the stale-snapshot wall blocking explorer `npm run dev`", "verifications": [{"command": "rm -f results-explorer/public/data/results.duckdb && cd results-explorer && (timeout 30 npm run dev >/tmp/explorer-dev-smoke.log 2>&1 &) && sleep 8 && curl -fsS -o /dev/null -w '%{http_code}\\n' http://localhost:5173/results/", "description": "Fresh-clone simulation: dev server starts when the public snapshot is missing.", "expected": "200", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 1}, {"command": "curl -fsS -o /dev/null -w '%{http_code}\\n' http://localhost:5173/results/data/results.duckdb", "description": "The snapshot URL the app fetches returns 200 after the predev/fallback resolves.", "expected": "200", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 2}, {"command": null, "description": "Manual smoke (recorded in PR body): the rendered page shows leaderboard rows, not the column-diff Error wall.", "expected": "PR description records the screenshot or the absence of the error string in the HTML body.", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm test", "description": "Frontend tests still pass.", "expected": "exit 0", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm run build", "description": "Production build is unaffected.", "expected": "exit 0, dist/ produced", "item_id": "explorer-dev-snapshot-dx", "last_result": null, "last_run": null, "seq": 5}], "work": [{"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": [], "notes": "The description cites PRs #344 / #354 / #380 as recent schema bumps.\nBefore implementation, capture a compact pinned snapshot so future\nreaders can re-run the check:\n\n - `gh pr view 344 --json title,mergedAt`\n - `gh pr view 354 --json title,mergedAt`\n - `gh pr view 380 --json title,mergedAt`\n - `git log --oneline --since=60.days -- results-explorer/src/db.ts benchbox/core/explorer_pipeline/duckdb_builder.py`\n\nRecord one line per PR (title + merge date) and the schema-bump count\nover the last 60 days in the PR description that lands this TODO's\nwork. Do NOT commit raw stdout to `_project/verification-logs/`;\npinned summary is enough.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pin the schema-bump cadence evidence cited in the description", "wid": "w0"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w0"], "notes": "Capture decision and rationale in the PR description. No separate ADR\nneeded \u2014 this is a DX choice, not an architectural one. Use the\ncadence numbers from w0. Touch on:\n - Schema-change cadence (from w0).\n - Data-change cadence: `git log --oneline --since=60.days -- results-data/bundles/ | wc -l`.\n - Latency budget for `npm run dev` start time.\n - Onboarding signal: do we want git clone -> npm run dev -> working\n UI as a baseline?\n\nThe recommendation absent overriding signal is (ii) \u2014 a tiny tracked\ndev snapshot \u2014 because:\n - It makes git clone -> npm run dev work with no Python toolchain.\n - It survives schema bumps because the developer rebuilding the\n snapshot is part of the same PR that bumps the schema.\n - The binary is small and changes rarely (only when schema bumps).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pick one of (i), (ii), (iii) with a brief decision note", "wid": "w1"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w1"], "notes": "For (ii) specifically:\n - Define a \"dev corpus\" subset under `results-data/dev-corpus/`\n (e.g., 1-2 cohorts of bundles) that produces a small snapshot.\n - Add a `make explorer-dev-snapshot` target that rebuilds it.\n - Track `results-explorer/dev-data/results.duckdb` (with a README\n explaining when to regen).\n - Modify `results-explorer/src/db.ts` snapshot URL resolution to\n try `data/results.duckdb` first, then fall back to\n `dev-data/results.duckdb` (or a build-time constant chosen by\n Vite mode).\n\nFor (i):\n - Add `predev` and `prebuild` npm scripts that check mtime and\n invoke the publish entry point if needed. Make the check\n skippable via `EXPLORER_SKIP_PREDEV=1` env var for CI / offline.\n\nFor (iii):\n - Add the make target and document it in\n `docs/development/browser-test-architecture.md` and\n `results-explorer/README.md`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement the chosen approach", "wid": "w2"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w2"], "notes": "`results-explorer/src/db.ts:196` currently advises running\n`benchbox explorer build`. After this TODO, the recommended remediation\nfor a local dev should be the new workflow (predev / make target /\nfallback note). Update the string accordingly.\n\nNote: this message will also be touched by TODO\n`explorer-cli-surface-migration` (to point at the relocated entry).\nCoordinate via the merge order; whichever lands second resolves the\nfinal text. If `explorer-cli-surface-migration` lands first, this\nTODO updates the dev-workflow phrasing.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update the error-wall remediation message", "wid": "w3"}, {"evidence": null, "item_id": "explorer-dev-snapshot-dx", "needs": ["w3"], "notes": "Smoke commands:\n - `rm -rf results-explorer/public/data && cd results-explorer && npm run dev`\n \u2192 UI loads on first attempt, no error wall.\n - For (i): touch results-data/bundles/.json and re-run\n `npm run dev` \u2192 rebuild fires.\n - For (ii): no rebuild needed; fallback snapshot is served.\n\nDocs:\n - Update `results-explorer/README.md` (if present) with the new\n dev workflow.\n - Add a one-liner to `CONTRIBUTING.md` or\n `docs/development/browser-test-architecture.md` explaining the\n snapshot lifecycle.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Smoke verification + docs update", "wid": "w4"}], "worktree": "explorer-dev-snapshot-dx"} +{"anti_patterns": [{"dont": "collapse tuning modes into a single row per platform", "instead": "each (platform, tuning_mode) gets its own row", "why": "tuned vs notuning is a comparability-breaking dimension"}, {"dont": "default missing power_score to 0", "instead": "use null and document the fallback (display_geomean_ms)", "why": "that would sort broken results to the top"}, {"dont": "fan out into additional bundle loads", "instead": "use in-memory accumulation only", "why": "the pipeline is already O(N) bundles"}, {"dont": "key the artifact on (benchmark, scale) alone", "instead": "include phase in the key", "why": "it silently mixes power and throughput phases"}, {"dont": "populate timings cells from the first raw QueryTiming row", "instead": "use display_ms from QueryDisplayTiming (median of passing runs)", "why": "the first row is the cold-cache outlier"}, {"dont": "use geomean_ms (raw all-sample) for display ranking", "instead": "use display_geomean_ms from the read-model contract reduction", "why": "it averages cold-cache and throughput-stream rows"}, {"dont": "use platform string directly as the matrix row key", "instead": "use platform_id from the read-model contract", "why": "it may include trust suffixes producing spurious duplicate rows"}], "approach": "The pipeline currently loads each bundle exactly once into memory to\nbuild manifest + details. Extend that same pass: maintain a dict\nkeyed by (benchmark, scale_factor, phase) that accumulates platform\nrows as each bundle is processed. At the end of the pass, emit one\nBenchmarkSummary per key.\n\nDo not refactor the pipeline architecture for this change. Add the\naccumulator and the emission step. Nothing else.\n\nFor phase handling: schema-v2 bundles already carry a `test_type`\n(\"power\" or \"throughput\"). Use that field to route each platform row\ninto the right accumulator bucket. Bundles without an explicit\ntest_type default to \"power\" (matches current CLI behavior).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-fix-read-model-contract"], "description": "The strategy review (2026-04-15) identified that the explorer's current\ndata model - manifest.json (aggregate only) + details/{id}.json (one file\nper result) - cannot support a ClickBench-style Platform \u00d7 Query matrix\nleaderboard without N parallel detail fetches assembled client-side. That\napproach is slow, fragile, and silently breaks when platforms run\nheterogeneous query subsets.\n\nThis TODO introduces a third derived artifact shape emitted by the\nexplorer pipeline:\n\n results-explorer/public/data/benchmarks/{benchmark}_sf{sf}_{phase}.json\n\nOne file per (benchmark, scale_factor, phase) tuple, containing the full\ncross-platform query matrix pre-computed from already-loaded bundles (no\nextra I/O - the pipeline already has everything in memory during its\nsingle pass).\n\nKey schema decisions driven by the adversarial review:\n\n1. Key includes **phase** (not just benchmark/scale). The proposal that\n keyed on (benchmark, scale) alone silently collapses power vs\n throughput vs maintenance phases. Making phase part of the filename\n makes the scope of each matrix explicit and URL-addressable.\n\n2. Tuning_mode is a per-platform-row field, not an artifact-level filter.\n Two DuckDB rows with different tuning modes occupy different matrix\n rows; the UI labels them as such.\n\n3. Each row carries `power_score` (for TPC-H/TPC-DS), `geomean_ms`, and\n `cost_usd` so the UI can choose ranking metric per benchmark family\n without a second fetch.\n\n4. Sparse matrices: platforms that ran a subset of queries produce\n `null` in the missing cells, never crash the build.\n\n5. Artifacts are gzipped on write (`*.json.gz`). TPC-DS SF100 with 30\n platforms \u00d7 99 queries \u00d7 throughput streams can reach 200-500KB raw;\n gzip cuts that ~5\u00d7. Loader handles both for local dev convenience.", "id": "explorer-emit-benchmark-summary-artifact", "preserves": ["Existing details/{id}.json per-result pages - detail page consumers must continue to work", "Existing manifest.json schema - new artifact is additive; do not remove or reshape manifest fields", "ExplorerPipeline.run() remains a single pass over bundles - no extra bundle loads introduced"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/cli/commands/explorer.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/explorer/exporter.py"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/**"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer/pipeline.py"}, {"kind": "only_modify", "path_glob": "results-explorer/public/data/benchmarks/**"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer/test_benchmark_summary.py"}], "state": "done", "title": "Emit per-benchmark summary artifact for matrix leaderboard view", "verifications": [{"command": "uv run -- python -m benchbox.cli explorer build --dry-run 2>&1 | grep -c 'benchmarks/.*_sf.*_.*\\.json'", "description": "Pipeline emits a summary artifact per (benchmark, scale, phase) tuple", "expected": "non-zero integer matching unique (benchmark, scale, phase) count", "item_id": "explorer-emit-benchmark-summary-artifact", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer/test_benchmark_summary.py -v", "description": "Unit tests pass", "expected": "all tests pass", "item_id": "explorer-emit-benchmark-summary-artifact", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c 'import gzip, json; d = json.loads(gzip.open(\"results-explorer/public/data/benchmarks/tpch_sf0.01_power.json.gz\").read()); assert \"platforms\" in d and \"query_ids\" in d'", "description": "Generated artifact round-trips through gzip + JSON", "expected": "no assertion failure", "item_id": "explorer-emit-benchmark-summary-artifact", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": [], "notes": "BenchmarkSummary fields: benchmark, scale_factor, phase, query_ids\n(ordered), platforms (list of PlatformRow), cell_reduction (string\ndocumenting the reduction policy, e.g.\n\"median_successful_measurement_ms\"), ranking (from\nRANKING_METRIC_BY_FAMILY, set by explorer-align-ranking-metric-with-tpc-standards).\n\nPlatformRow fields:\n result_id, platform_id (canonical), platform (raw display name),\n platform_version, tuning_mode, tuning_hash, execution_mode,\n trust_label, run_date, is_ranking_eligible (bool),\n power_score (nullable), display_geomean_ms (from read-model contract\n display reduction - NOT raw geomean_ms), sample_geomean_ms (raw,\n for audit), cost_usd (nullable),\n timings (dict query_id \u2192 display_ms from QueryDisplayTiming, null\n per missing cell - NOT the first raw duration_ms row).\n\nThe distinction between display_geomean_ms and sample_geomean_ms is\ncritical: the matrix cells MUST use display_ms values (median of\npassing runs per logical query), not raw per-sample rows. See\nexplorer-fix-read-model-contract for the reduction definition.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define BenchmarkSummary + PlatformRow Pydantic models in benchbox/core/explorer/models.py", "wid": "w1"}, {"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": ["w1"], "notes": "Zero additional disk I/O - accumulate from detail results already\nloaded in the single-pass scan. Phase values: \"power\" and\n\"throughput\" in Phase 1. Use the same sf_str formatting as existing\nmanifest paths.\n\nCanonical-row selection: for each (platform_id, benchmark, scale,\nphase) group, call `select_canonical_row()` from the read-model\ncontract to pick the default ranked row. Non-canonical variants\n(different tuning modes, trust tiers) are included as additional\nPlatformRows marked `is_ranking_eligible: false`. This lets the\nmatrix default view show one row per platform while the \"All\nvariants\" filter reveals the full set.\n\nSort canonical rows first by `is_ranking_eligible` descending, then\nby primary ranking metric (power_score or display_geomean_ms).\nNon-eligible rows follow in the same metric order.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend ExplorerPipeline.run() to emit benchmarks/{b}_sf{sf}_{phase}.json.gz per (benchmark, scale, phase) tuple", "wid": "w2"}, {"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": ["w2"], "notes": "Cases: (a) normal multi-platform; (b) sparse query subset\n(platform missing Q17 \u2192 null cell); (c) mixed tuning modes produce\ndistinct rows; (d) phase separation (power and throughput produce\nseparate files); (e) gzip output is valid and round-trips.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write unit tests in tests/unit/core/explorer/test_benchmark_summary.py", "wid": "w3"}, {"evidence": null, "item_id": "explorer-emit-benchmark-summary-artifact", "needs": ["w2", "w3"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Regenerate artifacts (uv run benchbox explorer build) and commit refreshed data/ tree", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change the `geomean_ms` semantics", "instead": "`display_geomean_ms` is the canonical comparison value, `geomean_ms` is preserved for audit", "why": "legacy consumers depend on the raw-sample geomean"}, {"dont": "compute platform display labels in two places", "instead": "disambiguate once in the pipeline and ship the result in the artifact", "why": "that recreates the parity bug we just fixed"}, {"dont": "emit comparability_warnings as free-form strings", "instead": "use a structured `{kind, severity, message}` shape if extending, but for v1 mirror the existing string output of buildComparabilityWarnings exactly", "why": "the explorer needs to apply formatting/severity"}, {"dont": "emit comparison artifacts for every N-way combination", "instead": "emit on-demand via CLI subcommand, pre-emit only popular combinations", "why": "the combinatorics explode (100 results = millions)"}, {"dont": "remove the per-detail fallback in Compare.tsx during rollout", "instead": "keep the fallback and log when it fires", "why": "pre-emitted artifacts will not exist for every combination users click"}, {"dont": "skip the `schema_version` field on ComparisonArtifact", "instead": "version field is mandatory and explorer hard-fails on mismatch", "why": "explorer must reject artifacts written by an incompatible older pipeline"}], "approach": "Schema-first: define the Pydantic model (w1) before any logic, so the\nexplorer team can stub the consumer in parallel. Build the disambiguator\n(w2) before the artifact builder (w3) because the builder calls it.\nWire the CLI subcommand (w4) before refactoring Compare (w5) so the\nartifact actually exists to fetch. Pipeline pre-emission for popular\ncombinations (w8) layers on top of the subcommand and is independent\nof the explorer refactor.\n\nSpeedup definition: use `slowest / this` per query (matches the current\nCompare.tsx breakdown table at line 392) for the `speedup_vs_slowest_per_row`\nfield. For the per-baseline view (NormalizedSpeedupChart), the artifact\nprovides the raw `display_ms_per_row` and the chart picks the baseline\nindex - this is the one piece of computation that cannot be pre-emitted\nbecause the user picks the baseline interactively.\n\nHOWEVER: the speedup math `baseline_ms / this_ms` becomes a 3-line\nreduce in the chart, calling the parity-tested `speedupRatio` helper.\nDefinition is unified, drift impossible.\n\nThe `ComparisonArtifact` schema field names mirror the explorer types\nexactly to avoid translation layer bugs (Compare.tsx already uses\n`display_timings`, `display_geomean_ms` from DetailResult - the\nartifact uses the same names).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-16T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-emit-comparison-artifact-from-pipeline", "id": 166, "reason": "Combinatorial - 100 results \u00d7 C(100,4) = 3.9M comparisons is not\nreasonable to pre-compute. Address with a smart popularity heuristic\n(top-N most-clicked) once telemetry exists.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] conditional-unmet: Pre-computing all N-way comparisons is explicitly combinatorial-infeasible; deferred pending telemetry to drive a popularity heuristic that doesn't exist yet.", "summary": "Pre-emit comparison artifacts for arbitrary N-way combinations at pipeline time"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-emit-comparison-artifact-from-pipeline", "id": 167, "reason": "Requires turning the static site into an app with a backend.\nOut of scope for the static-hosted Phase 2 explorer. v1 ships\npre-emitted artifacts for popular combinations + browser fallback.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Requires turning the static-hosted explorer into a backend app; explicit architecture decision chose pre-emitted artifacts instead.", "summary": "Server-side computation of comparison artifacts at request time"}], "deps": ["explorer-establish-cli-explorer-parity-tests", "explorer-unify-visualization-foundations"], "description": "The 2026-04-15 review identified that `Compare.tsx` fetches N detail\nJSON files and recomputes everything per render: per-query speedups\n(line 388-393), diverging deltas (DivergingBarChart.tsx:49), normalized\nspeedups (NormalizedSpeedupChart.tsx:57), summary-card metrics\n(Compare.tsx:219-290), and the breakdown table (Compare.tsx:364-429).\n\nThis is the architectural root cause of the parity problem: when the\nCLI and explorer compute independently, drift is inevitable. The\nparity test infrastructure (`explorer-establish-cli-explorer-parity-tests`)\nproves equivalence today but does not eliminate the duplication.\n\nThe fix is to push computation into the pipeline. The pipeline already\nemits per-`(benchmark, sf, phase)` `BenchmarkSummary` artifacts via\n`transformer.py` \u2192 `pipeline.py`. Add a parallel `ComparisonArtifact`\nemitted per ordered set of result_ids: it carries display_timings,\nper-query speedup matrix (one row per (queryId, baselineIdx)),\ndiverging delta matrix, and pre-disambiguated platform display labels.\n\nCompare.tsx becomes a pure renderer that fetches one artifact and\niterates. The chart components stop computing - they receive\npre-computed numeric arrays.\n\nThis work also ports the platform-label disambiguation from\n`result_plotter.py:68-149` (driver version, execution mode, scale\nfactor suffixes) into the artifact, so the explorer no longer renders\nambiguous \"DuckDB / DuckDB\" rows when two results share a platform name.", "id": "explorer-emit-comparison-artifact-from-pipeline", "preserves": ["Comparison URL format `?ids=id1,id2` - short-id resolution behavior unchanged", "Existing `BenchmarkSummary` artifact format and emission - this work adds a parallel artifact, does not modify the summary", "Existing `ComparabilityBanner` UI - only the warning generation moves server-side; the component contract is unchanged", "Existing `DetailResult` model and per-detail JSON file emission - Compare must still work via the fallback path during rollout", "RANKING_METRIC_BY_FAMILY registry - the artifact reads from it but does not modify"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/**"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "benchbox/cli/commands/explorer.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/pipeline.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/DivergingBarChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/NormalizedSpeedupChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryTimingChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "tests/parity/fixtures/comparison_artifact_*.json"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/**"}], "state": "done", "title": "Emit pre-computed comparison artifact from pipeline so Compare page becomes a renderer", "verifications": [{"command": "grep -n 'class ComparisonArtifact' benchbox/core/explorer_pipeline/models.py", "description": "ComparisonArtifact Pydantic model exists with the documented fields", "expected": "class defined with schema_version, rows, query_cells, comparability_warnings, ranking_primary_metric", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_transformer.py::test_disambiguate_platform_labels -v", "description": "Disambiguator produces unique labels", "expected": "passes for single, duplicate, and triple-collision cases", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_comparison_artifact.py -v", "description": "Comparison artifact builder unit tests pass", "expected": "all cases pass", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 3}, {"command": "benchbox explorer build-comparison --ids , --out /tmp/cmp.json && uv run -- python -c 'import json; from benchbox.core.explorer_pipeline.models import ComparisonArtifact; ComparisonArtifact.model_validate(json.load(open(\"/tmp/cmp.json\")))'", "description": "CLI subcommand emits a valid artifact", "expected": "no exception - artifact validates against schema", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- Compare", "description": "Compare.tsx prefers artifact when available", "expected": "tests cover both artifact-present and fallback paths", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 5}, {"command": "grep -n 'slowestPrimary / primary\\|slowest / fastest' results-explorer/src/pages/Compare.tsx", "description": "No two definitions of speedup remain in Compare.tsx", "expected": "zero matches - both replaced with `row.speedup_vs_slowest_per_row` from the artifact", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 6}, {"command": "uv run -- python -m pytest -m fast -q && cd results-explorer && npm test", "description": "Full Python and Vitest suites pass", "expected": "all tests pass", "item_id": "explorer-emit-comparison-artifact-from-pipeline", "last_result": null, "last_run": null, "seq": 7}], "work": [{"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": [], "notes": "Schema (must match what Compare.tsx will consume):\n\n class ComparisonRow(BaseModel):\n result_id: str\n short_id: str\n platform: str # raw platform name\n display_label: str # disambiguated (e.g. \"DuckDB v1.2.0\")\n platform_id: str\n run_date: str\n trust_label: str\n tuning_mode: str | None\n execution_mode: str | None\n test_type: str | None\n geomean_ms: float | None\n display_geomean_ms: float | None\n power_score: float | None\n total_duration_s: float\n driver_version: str | None\n display_timings: list[QueryDisplayTiming]\n\n class ComparisonQueryCell(BaseModel):\n query_id: str\n display_ms_per_row: list[float | None] # one per ComparisonRow\n fastest_ms: float | None\n slowest_ms: float | None\n speedup_vs_slowest_per_row: list[float | None] # slowest/this\n\n class ComparisonArtifact(BaseModel):\n schema_version: int = 1\n benchmark: str\n scale_factor: float\n generated_at: str\n rows: list[ComparisonRow]\n query_cells: list[ComparisonQueryCell]\n comparability_warnings: list[str]\n ranking_primary_metric: str # \"geomean_ms\" or \"power_score\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define `ComparisonArtifact` Pydantic model in `benchbox/core/explorer_pipeline/models.py`", "wid": "w1"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w1"], "notes": "Algorithm (lifted from CLI, adapt for the artifact rows):\n - Group rows by `platform_id`.\n - If a group has size 1: `display_label = platform`.\n - Else: progressively append disambiguators in priority order\n (driver_version, execution_mode, scale_factor, run_date) until\n all labels in the group are unique.\n - Resulting label format: \"DuckDB v1.2.0\" or\n \"DuckDB v1.2.0 (sql)\" or \"DuckDB v1.2.0 SF1.0\".\n\nAdd unit test in `tests/unit/core/explorer_pipeline/test_transformer.py`\nthat mirrors any tests for the CLI version - and add a parity\nfixture (per `explorer-establish-cli-explorer-parity-tests`) so\nboth sides assert the same labels.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement `_disambiguate_platform_labels(rows)` in transformer.py mirroring `result_plotter.py:68-149` logic", "wid": "w2"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w2"], "notes": "Inputs: list of DetailResult (already fetched by caller),\n list of corresponding ManifestEntry (for trust/short_id).\n\nAlgorithm:\n 1. Validate same benchmark + same scale_factor (raise on mismatch).\n 2. Build ComparisonRow per detail using disambiguated labels.\n 3. Compute union of all query_ids (sorted natural).\n 4. For each query_id:\n - display_ms_per_row from row.display_timings\n - fastest_ms = min of non-null, slowest_ms = max\n - speedup_vs_slowest_per_row[i] = slowest_ms / display_ms[i]\n if both present and >0 else None\n 5. Compute comparability_warnings (mirror\n `buildComparabilityWarnings` in ComparabilityBanner.tsx so\n both sides agree).\n 6. Determine ranking_primary_metric from\n `RANKING_METRIC_BY_FAMILY[benchmark]` (already exists).\n\nPure function - no I/O. Caller writes the JSON.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `build_comparison_artifact(detail_results, manifest_entries)` in transformer.py", "wid": "w3"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w3"], "notes": "Strategy decision: v1 emits on-demand. The alternative (pre-emit\nevery N-way combination) is documented in `deferred` as\ncombinatorially infeasible.\n\nSubcommand contract:\n benchbox explorer build-comparison \\\n --ids id1,id2,id3 \\\n --out data/compare/.json\n\nOutput path: `data/compare/.json`.\nHash MUST be deterministic on sorted input ids (the explorer\ncomputes the same hash to fetch). Schema validates against\n`ComparisonArtifact` Pydantic model on emit.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `benchbox explorer build-comparison` CLI subcommand emitting one artifact per --ids set", "wid": "w4"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w4"], "notes": "New explorer flow:\n 1. Resolve short IDs to full IDs (existing).\n 2. Compute deterministic comparison-artifact path:\n `/results/data/compare/${sha256(sortedIds.join(',')).slice(0,16)}.json`.\n 3. Try to fetch it. If 200, render directly from artifact fields.\n 4. If 404, fall back to the current per-detail flow (back-compat\n during rollout) - log a warning so we know when fallback fires.\n\nCharts no longer compute speedups/deltas locally - they receive\npre-computed numeric arrays from the artifact. Per the parity\ntest infrastructure, helper functions in `chartMath.ts` are still\nused for the *renderer* math (axis stops, bar widths) but never\nfor the values.\n\nRemove the dual `slowest/fastest` speedup definition from the\nsummary cards (Compare.tsx:225) and the breakdown table\n(Compare.tsx:392) - both now read `speedup_vs_slowest_per_row`\nfrom the artifact, which is the single source.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Refactor `Compare.tsx` to fetch `ComparisonArtifact` instead of N `DetailResult` files", "wid": "w5"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w3"], "notes": "Cases:\n - Same benchmark/SF passes; different benchmark raises\n - Different SF raises\n - Single result returns artifact with empty speedup_vs_slowest_per_row entries (None)\n - All-failed query produces None entries, not zeros\n - Disambiguation produces unique labels for 2 same-platform-different-driver results\n - speedup_vs_slowest matches `slowest/this` definition exactly\n - comparability_warnings list mirrors what ComparabilityBanner expects (cross-test_type, cross-tuning, query-count mismatch)\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `tests/unit/core/explorer_pipeline/test_comparison_artifact.py` covering edge cases", "wid": "w6"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w5", "w6"], "notes": "Requires the parity infrastructure to exist - declared as a hard\ncross-item dep on `explorer-establish-cli-explorer-parity-tests`\nat the item level.\n\nFixture flow:\n - Python emits `tests/parity/fixtures/comparison_artifact_*.json`\n covering at least: 2-result baseline, 3-result baseline,\n missing-query, all-failed-query, same-platform-different-version\n - Vitest reads each fixture and asserts that mounting Compare\n with the artifact produces the documented row counts, fastest\n highlight positions, and speedup numbers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add a parity fixture pair so the explorer's renderer assertions cover the artifact contract", "wid": "w7"}, {"evidence": null, "item_id": "explorer-emit-comparison-artifact-from-pipeline", "needs": ["w4"], "notes": "Pipeline-level pre-emission for the static site:\n - Config flag in `pipeline.py` (default 0 = off) for the\n number of pre-emitted comparisons per benchmark\n - \"Popular\" defined for v1 as: the N most-recent result\n combinations per benchmark from the manifest. Telemetry-driven\n popularity is deferred.\n - Wired so a single pipeline run produces the artifacts under\n `data/compare/` alongside the manifest.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire pipeline to optionally pre-emit comparison artifacts for top-N popular combinations per benchmark", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "commit auto-regenerated fixtures without inspecting the diff", "instead": "`make parity-check` must be run separately to detect diffs", "why": "the fixture IS the contract and silent regeneration hides drift"}, {"dont": "generate fixtures from the TS side", "instead": "Python (the canonical compute layer) emits, TS verifies", "why": "that makes Python the follower, not the source of truth"}, {"dont": "inline the math back into components after extraction", "instead": "components import helpers, no exceptions", "why": "that defeats the test contract"}, {"dont": "skip cases for missing-data inputs (null/None)", "instead": "every helper must have at least one null-input case", "why": "that is exactly where Python and JS often disagree (NaN vs null)"}, {"dont": "use `toEqual` for floats in the parity suite", "instead": "use `Math.abs(a - b) < 1e-9` with explicit tolerance", "why": "IEEE-754 noise differs between Python and JS"}], "approach": "Build foundation-first: extract pure helpers (w1) before generating\nfixtures (w2), because the fixture generator must mirror the exact\nsignatures the helpers expose.\n\nReuse the project's existing Python visualization functions where they\nexist as pure helpers - `core/visualization/ascii/heatmap.py` and\nsiblings delegate to `textcharts`, but the pre-textcharts color/sort\nmath may live in `result_plotter.py`. If a Python equivalent does not\nexist, ADD it (in Python) rather than picking a TS-only number, so\nparity is bidirectional.\n\nFixture format mirrors the textcharts test pattern (if any) so future\ncontributors recognize the convention. Keep fixtures small and readable\n- explicit `id` on each case is mandatory so failures are debuggable.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [{"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-establish-cli-explorer-parity-tests", "id": 168, "reason": "Overkill for trivial math (log10, ratios, medians). Cost in build\ncomplexity and runtime size is not justified when fixture-pinned\ncontracts achieve the same correctness guarantee.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] non-actionable: Rejected in its own reason as overkill for trivial math; fixture-pinned contracts already provide the correctness guarantee.", "summary": "WASM-compile the Python compute layer and call it from the browser"}, {"created_at": "2026-07-18T19:50:09Z", "from_item": "explorer-establish-cli-explorer-parity-tests", "id": 169, "reason": "Tracked separately in `add-hypothesis-property-tests.yaml`. Should\nbuild on this fixture infrastructure once it is established.\n", "resolution": "dismissed", "resolved_item": null, "resolved_reason": "[G4 sweep] duplicate: Explicitly tracked separately in add-hypothesis-property-tests.yaml, which is open (planning).", "summary": "Property-based parity tests via Hypothesis + fast-check"}], "deps": [], "description": "The 2026-04-15 visualization review identified that the Python CLI\n(`benchbox/core/visualization/`) and the Preact explorer\n(`results-explorer/src/components/`) reimplement the same\nvisualization math in two languages with **zero shared computation\nand zero contract tests**.\n\nConcrete drift surfaces today:\n - Heatmap color: `colorForCell` (QueryHeatmap.tsx:39-45) duplicates\n `textcharts.heatmap` log10-ratio\u2192hue logic.\n - Per-baseline speedup: `NormalizedSpeedupChart.tsx:57` duplicates\n `textcharts.normalized_speedup` (`baseline / this`).\n - Diverging delta: `DivergingBarChart.tsx:49` duplicates the\n `textcharts.diverging_bar` `(this - baseline) / baseline * 100`\n formula, plus an independent magnitude sort at line 63-67.\n - Per-query \"fastest\" highlight: Compare.tsx:388-411 inlines its own\n ranking that has no CLI counterpart and disagrees with Compare's\n summary-card definition (see explorer-unify-visualization-foundations\n for the dual-definition bug).\n\nThe user requirement is explicit: **\"there should be only ONE place\nwhere visualization values are composed, when that is not possible\nwe must have test infrastructure that ENSURES the parallel\nimplementations match.\"** This item builds that test infrastructure.\nCentralizing computation server-side is tracked separately\n(explorer-emit-comparison-artifact-from-pipeline).\n\nStrategy: Python emits canonical fixtures (input data + expected\noutput values) for every chart-type computation. Vitest loads the\nfixtures and asserts that the explorer's pure helper functions produce\nbyte-identical numbers (within float tolerance). Any future drift\nfails CI, regardless of which side changes.", "id": "explorer-establish-cli-explorer-parity-tests", "preserves": ["Existing 36-test Vitest suite continues to pass alongside the new parity suite", "Existing `colorForCell` / `lightnessForCell` exported signatures from QueryHeatmap.tsx - these are imported elsewhere and tested", "Existing chart visual output - extracting helpers must not change pixel-level rendering for the same input data", "Vitest setup at `results-explorer/src/test/setup.ts` - append, do not replace"], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/**"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/ascii/**"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/visualization/result_plotter.py"}, {"kind": "only_modify", "path_glob": ".github/workflows/ci.yml"}, {"kind": "only_modify", "path_glob": "Makefile"}, {"kind": "only_modify", "path_glob": "docs/visualization-parity.md"}, {"kind": "only_modify", "path_glob": "results-explorer/src/__tests__/parity/**"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/DivergingBarChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/NormalizedSpeedupChart.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/QueryHeatmap.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/chartMath.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "tests/parity/**"}], "state": "done", "title": "Establish CLI\u2194explorer visualization parity test infrastructure (fixture-driven)", "verifications": [{"command": "grep -E 'export (function|const) (colorForCell|lightnessForCell|speedupRatio|deltaPct|sortByMagnitudeDesc|perQuerySpeedup|geomeanMs)' results-explorer/src/lib/chartMath.ts", "description": "Pure helper module exists and re-exports the documented functions", "expected": "all 7 named exports present", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python tests/parity/generate_visualization_fixtures.py --out tests/parity/fixtures && ls tests/parity/fixtures/*.json", "description": "Fixture generator runs and produces one file per chart computation", "expected": "at least 7 .json files matching the 7 helpers in w1", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm test -- chartMath.parity", "description": "Vitest parity suite runs all fixture cases and passes", "expected": "parity suite present, all cases pass", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 3}, {"command": "make parity-check", "description": "Make target detects fixture drift", "expected": "exit 0 when fixtures match, exit non-zero with diff when they don't", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 4}, {"command": "cd results-explorer && npm test -- QueryHeatmap", "description": "Existing QueryHeatmap unit tests still pass (refactored to use fixtures)", "expected": "all QueryHeatmap tests pass; no hardcoded color numbers remain in the test file", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 5}, {"command": "cd results-explorer && npm test", "description": "Full Vitest suite still passes (36 + new parity tests)", "expected": "all tests pass", "item_id": "explorer-establish-cli-explorer-parity-tests", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": [], "notes": "Move the math out of the SVG components into pure functions:\n - `colorForCell`, `lightnessForCell` (already pure, re-export from QueryHeatmap)\n - `speedupRatio(baselineMs, thisMs)` from NormalizedSpeedupChart.tsx:57\n - `deltaPct(thisMs, baselineMs)` from DivergingBarChart.tsx:49\n - `sortByMagnitudeDesc(entries)` from DivergingBarChart.tsx:63-67\n - `perQuerySpeedup(timings)` (slowest/fastest) from Compare.tsx:388-393\n - `geomeanMs(samples)` (median-based reduction matching transformer)\nComponents import these helpers; no behavior change. Each helper\nreturns plain numbers - no SVG, no DOM - so Vitest can call them\ndirectly from a fixture loop.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit and extract pure computation functions out of every explorer chart component into testable helpers in `results-explorer/src/lib/chartMath.ts`", "wid": "w1"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w1"], "notes": "Pytest-runnable script (or a regular pytest test using\n`--generate-fixtures` flag) that produces files at\n`tests/parity/fixtures/{chart_name}.json` with shape:\n\n {\n \"chart\": \"heatmap_color\",\n \"version\": 1,\n \"cases\": [\n {\"id\": \"fastest_in_col\", \"input\": {\"ms\": 100, \"minInCol\": 100},\n \"expected\": {\"hue\": 120}},\n {\"id\": \"ten_x_slower\", \"input\": {\"ms\": 1000, \"minInCol\": 100},\n \"expected\": {\"hue\": 0}},\n ...\n ]\n }\n\nOne fixture file per chart computation listed in w1. Cases must\ninclude: identity (ratio=1), boundary (ratio=10 clamp), missing\ndata (None/null), and at least one realistic mid-range value.\n\nFixtures are CHECKED INTO GIT - they are the contract. Regenerating\nthem must be a deliberate act (changes the contract) and require\nthe Vitest suite to accept the new numbers via re-run.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build Python fixture generator at `tests/parity/generate_visualization_fixtures.py` that emits JSON fixtures per chart computation", "wid": "w2"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w2"], "notes": "Pattern: read each `tests/parity/fixtures/*.json` (relative path\ncomputed in vitest config), iterate `cases`, call the corresponding\nTS helper from `chartMath.ts`, assert\n`Math.abs(result - expected) < 1e-9` for floats and strict equality\nfor ints/strings.\n\nUse `describe.each` or generator pattern so each case appears as\nits own test name (debuggable on failure). On a fixture mismatch,\nthe failure message must show: chart name, case id, input, expected\nvs actual, and the suspected source of drift (Python file vs TS file).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Vitest parity suite at `results-explorer/src/__tests__/parity/chartMath.parity.test.ts` that loads each fixture and asserts equivalence", "wid": "w3"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w3"], "notes": "- `make parity-fixtures` regenerates fixtures via the Python script\n- `make parity-check` runs Python generation into a tmpdir, diffs\n against the checked-in fixtures, fails if they differ\n- GitHub Actions: run `make parity-check` AND `cd results-explorer\n && npm test` on every PR\n- Document the workflow in `docs/contributing-to-explorer.md`\n (or new `docs/visualization-parity.md`): \"If you change a chart\n computation, regenerate fixtures with `make parity-fixtures` -\n this changes the contract; reviewers must approve the diff.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire fixture generation into a Make target / CI gate so divergence is detected automatically", "wid": "w4"}, {"evidence": null, "item_id": "explorer-establish-cli-explorer-parity-tests", "needs": ["w3"], "notes": "The current `QueryHeatmap.test.tsx` tests `colorForCell` against\nhardcoded TS expectations only. Replace those assertions with the\nfixture-driven path so QueryHeatmap is the proof-of-concept for\nthe new infrastructure. Delete the duplicated hardcoded numbers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Backfill parity for the existing `colorForCell` and `lightnessForCell` tests so they share the fixture suite", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add UI changes in this TODO", "instead": "UI consumption comes in follow-up TODOs", "why": "this is a pipeline/data-model change only"}, {"dont": "change the tuning_hash algorithm without updating all callers", "instead": "stability matters more than hash quality", "why": "it is used for grouping across results"}, {"dont": "error on missing fields", "instead": "null-safe extraction throughout", "why": "older bundles predate many fields"}, {"dont": "recompute geomean_ms in every browser query", "instead": "add it to the manifest at build time", "why": "it should be precomputed"}], "approach": "Source-map first (w1): walk the bundle schema and write down exactly which\nbundle key maps to which manifest field. Avoids guessing during implementation.\n\nCompute geomean_ms at transform time, not in the browser. The browser DuckDB\ncan do it in a query, but pre-computing it means it's available in the JSON\nmanifest and in sort/filter without DuckDB being loaded.\n\ntuning_hash: short SHA256 prefix of the serialized tuning config dict (or\ntuning mode string if no detail is available). Purpose is grouping, not\ncryptographic identity - 8 hex chars is enough.\n\ntest_type: infer from which phases are present in the bundle. If\nphases.power_test is populated \u2192 \"power\"; if phases.throughput_test \u2192 \"throughput\";\nif both \u2192 \"power\" (primary). If neither, leave null.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-14T00:00:00Z", "completed_pr": null, "created_at": "2026-04-14T00:00:00Z", "deferrals": [], "deps": [], "description": "Post-launch audit (2026-04-14) found that the Phase 1 pipeline transformer\nemits only 11 manifest fields from a bundle that contains 13 blocks and\n50+ fields. This silently drops execution_mode, tuning_mode, platform_version,\ntest_type, validation_status, cost_usd, and the per-query run_type/iter/stream\nfields needed for warmup vs measurement distinction.\n\nThe DuckDB `results` table has 11 columns; all other fidelity TODOs\n(methodology disclosure, tuning visibility, comparability warnings, charts)\ndepend on these fields reaching the manifest and DuckDB schema first.\n\nTarget manifest additions (all sourced from existing bundle fields):\n - geomean_ms - exp(mean(ln(query_ms))) across measurement queries\n - execution_mode - \"sql\" | \"dataframe\" from config.mode\n - tuning_mode - \"tuned\" | \"notuning\" | \"auto\" from config.tuning_mode\n - tuning_hash - stable short hash of tuning config for grouping\n - platform_version - platform.version (distinct from driver_version)\n - test_type - \"power\" | \"throughput\" from phases\n - validation_status - summary.validation.status\n - cost_usd - cost.total_usd (null if absent)\n\nTarget query record additions:\n - run_type - \"warmup\" | \"measurement\" per query timing entry\n - iter, stream - iteration and stream IDs", "id": "explorer-extend-manifest-and-pipeline", "preserves": ["Existing DuckDB column names - extend schema with new columns only", "Existing manifest field names and types - add fields, never rename or remove", "Explorer pages render correctly for bundles that have no tuning config, no cost data, or no run_type on queries", "Null-safety for older bundles that predate any new field - missing \u2192 null, not error"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/results/schema.py (bundle schema is read-only here)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/components/ (no UI changes in this TODO)"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ (no UI changes in this TODO)"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/"}], "state": "done", "title": "Extend explorer pipeline manifest and DuckDB schema with missing bundle fields", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -q", "description": "All pipeline unit tests pass", "expected": "all tests pass", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run python -c \"import json; m=json.load(open('results-explorer/public/data/manifest.json')); r=m['results'][0]; assert 'geomean_ms' in r and 'execution_mode' in r and 'tuning_mode' in r, r.keys()\"", "description": "Regenerated manifest contains new fields for existing bundles", "expected": "(no assertion error)", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run -- python -c \"import duckdb; con=duckdb.connect('results-explorer/public/data/results.duckdb'); print([c[0] for c in con.execute('PRAGMA table_info(results)').fetchall()])\"", "description": "DuckDB results table has new columns", "expected": "list includes geomean_ms, execution_mode, tuning_mode, validation_status", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite unaffected", "expected": "all tests pass", "item_id": "explorer-extend-manifest-and-pipeline", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": [], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Audit transformer.py and models.py: map each target field to its bundle source path; document any fields that require computation (geomean_ms, tuning_hash) vs direct extraction.", "wid": "w1"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add geomean_ms computation to transformer.py: exp(mean(ln(ms))) over measurement-only queries (run_type=measurement or all if run_type absent). Add to ManifestResult and DetailResult models.", "wid": "w2"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add execution_mode, tuning_mode, tuning_hash, platform_version, test_type, validation_status, cost_usd to ManifestResult and DetailResult. Null-safe for older bundles.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w1"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Add run_type, iter, stream to QueryTiming in detail JSON. Update transformer.py query extraction loop.", "wid": "w4"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w2", "w3", "w4"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Extend DuckDB schema in duckdb_builder.py: add all new manifest columns to the results table; add run_type, iter, stream columns to queries table. Regenerate for existing corpus and verify.", "wid": "w5"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w5"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Update TypeScript types.ts in results-explorer to match extended ManifestResult and DetailResult shapes. No UI changes - just type alignment.", "wid": "w6"}, {"evidence": null, "item_id": "explorer-extend-manifest-and-pipeline", "needs": ["w6"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "pending", "summary": "Re-run full pipeline over existing corpus; confirm no regressions in existing explorer pages; confirm new fields appear in manifest and DuckDB.", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "block this item on cosmetic facet improvements", "instead": "file follow-up items instead.", "why": "scope creep is how refactors stall"}, {"dont": "change visible facet behavior or labels", "instead": "keep product surface identical.", "why": "this is a refactor, not a redesign"}, {"dont": "keep adapter-name-conditional fallbacks 'just in case'", "instead": "delete dead branches.", "why": "the whole point of the normalized contract is to remove them"}, {"dont": "reshape the normalized contract from the explorer side", "instead": "raise issues against that item if a field is missing or wrong.", "why": "the contract is owned by results-complete-execution-environment-capture"}], "approach": "Sequence as: audit \u2192 model migration \u2192 query migration \u2192 label reconciliation\n\u2192 coverage re-validation \u2192 test update \u2192 cross-corpus check. Treat the\nmigration table from w1 as the contract for w2-w4; do not let label\ndecisions leak into the model layer. Use the normalized field names from\n`results-complete-execution-environment-capture` w1 as the authoritative\nvocabulary; the explorer is a consumer, not a co-designer of the contract.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-02T00:00:00Z", "deferrals": [], "deps": ["explorer-selection-facets-coverage-at-scale", "results-complete-execution-environment-capture"], "description": "`explorer-selection-facets-coverage-at-scale` ships the explorer facet model\n(`results-explorer/src/lib/facetModel.ts`) against current adapter fields\nbecause the normalized execution-environment contract from\n`results-complete-execution-environment-capture` is not yet available. That\nis the right sequencing call for shipping speed, but it leaves behind a\nknown refactor: facet keys like `deployment_class`, `cloud_provider`,\n`cloud_region`, `instance_or_warehouse`, `storage_format`, and `cost_status`\nmust currently be sourced from adapter-specific raw config fields with\nper-adapter fallbacks. Once `results-complete-execution-environment-capture`\nlands, those fields will be emitted as a stable normalized contract on\nevery result artifact.\n\nThis item is the loop-back: migrate the facet model and its consumers off\nthe per-adapter fallbacks and onto the normalized execution-environment\ncontract. The product surface should not change \u2014 same facet rail, same\nURL contract, same coverage signals \u2014 but the data path becomes uniform\nacross adapters and the fallback scraping code goes away.\n\nScope is intentionally narrow: facet model + DuckDB queries + immediate\nconsumers (FacetRail labels, CoverageBadge grouping). Anything outside\nthe scope_limit is a separate item.", "id": "explorer-facet-model-environment-contract-refactor", "preserves": ["Coverage badges and missing-cell rendering continue to match the pre-refactor explorer for identical inputs.", "Existing explorer URLs (shipped pre-refactor) round-trip identically through useUrlState.", "FacetRail chip labels stay user-stable (no surprise renames mid-session).", "Missing coverage continues to not penalize a platform's average rank."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/docs/facet-environment-migration.md"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/CoverageBadge.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/FacetRail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/facetModel.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/queryFilters.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/facetModel.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryFilters.ts"}], "state": "done", "title": "Explorer: refactor facet model onto normalized execution-environment contract", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Frontend typecheck and tests pass.", "expected": "All facet/coverage tests pass; pre-refactor URL round-trip regression test passes.", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:fixtures && npm run build", "description": "Browser build succeeds against the post-environment-capture corpus fixtures.", "expected": "Fixture generation, fixture verification, and Vite build complete without errors.", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 2}, {"command": "rg -n 'COALESCE|adapter ===|platform ===' results-explorer/src/lib/facetModel.ts results-explorer/src/lib/queryFilters.ts results-explorer/src/lib/duckdbQueries.ts || true", "description": "No fallback-scraping code remains in the facet path.", "expected": "No matches in facet-path files (matches in unrelated files are acceptable).", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 3}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/test_visible_metrics_registry.py -q", "description": "Visible metrics parity remains canonical with Python.", "expected": "Registry parity test passes.", "item_id": "explorer-facet-model-environment-contract-refactor", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": [], "notes": "Walk every `FacetKey` in `results-explorer/src/lib/facetModel.ts` and\nits serializer/parser pairs. For each key sourced from adapter-specific\nraw config (deployment_class, cloud_provider, cloud_region,\ninstance_or_warehouse, storage_format, cost_status), record:\n - current source field(s) and per-adapter quirks;\n - target field on the normalized execution-environment contract\n (cross-reference `results-complete-execution-environment-capture`\n w1 schema);\n - any value-domain shifts (e.g. cloud_provider casing, region\n aliases).\nOutput: a short migration table committed to\n`results-explorer/docs/facet-environment-migration.md`. No code edits\nin this unit; downstream units consume the table.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Audit fallback paths in facetModel.ts and document field-by-field migration", "wid": "w1"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w1"], "notes": "Rewrite serializer/parser pairs and the `facetsToWhereClause` helper\nto read from normalized contract columns. Delete per-adapter fallback\nbranches. Preserve URL round-trip for every existing facet key \u2014 URLs\nshipped against the pre-refactor model must still parse.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Migrate facetModel.ts onto normalized contract fields", "wid": "w2"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w2"], "notes": "Update `results-explorer/src/lib/queryFilters.ts` and\n`results-explorer/src/lib/duckdbQueries.ts` so leaderboard, benchmark,\nand platform queries reference normalized columns. Remove `COALESCE`\nand adapter-name-conditional expressions added as fallbacks during\nthe original facet rollout.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update DuckDB queries to read normalized contract columns", "wid": "w3"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w2"], "notes": "Normalized contract may use canonical labels (e.g. AWS/GCP/Azure for\ncloud_provider) where the adapter-specific path used adapter-native\nstrings. Update FacetRail chip rendering and CoverageBadge grouping\nso labels remain stable for users (no chip text churn) while consuming\nthe canonical values. Add a small label-mapping module if needed; do\nnot embed mappings inline in components.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Reconcile FacetRail and CoverageBadge labels with normalized vocabulary", "wid": "w4"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w3", "w4"], "notes": "Coverage badges, missing-cell rendering, and \"average rank over\ncovered cohorts\" must continue to behave correctly when cohort\nmembership is computed from normalized fields. Confirm: missing\ncoverage still does not penalize average rank; coverage badges\nstill render accurately for the public corpus fixtures.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Re-validate coverage signals against the normalized contract", "wid": "w5"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w2", "w3"], "notes": "Update `lib/__tests__/facetModel.test.ts` and\n`lib/__tests__/queryFilters.test.ts` to assert canonical field names\nand value domains. Add a regression test that pre-refactor URLs still\nround-trip cleanly. Remove tests that exercised adapter-fallback\nbranches.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update tests to assert against normalized contract", "wid": "w6"}, {"evidence": null, "item_id": "explorer-facet-model-environment-contract-refactor", "needs": ["w5", "w6"], "notes": "Run the explorer fixtures end-to-end against the public-corpus shape\nthat `results-complete-execution-environment-capture` w14 establishes.\nConfirm: no facet key falls through to a fallback path; no console\nwarnings about missing normalized fields; coverage and rank surfaces\nmatch the pre-refactor explorer for identical inputs.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Cross-corpus verification: every result bundle facets through the normalized path", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "apply canonical-row selection inside to_manifest_entry", "instead": "selection happens in the pipeline after all entries are loaded", "why": "the transformer processes one bundle at a time"}, {"dont": "hard-code 'maintainer-run' in query logic", "instead": "use is_ranking_eligible()", "why": "the eligibility rules must be the single source of truth"}, {"dont": "rename geomean_ms to sample_geomean_ms", "instead": "add display_geomean_ms alongside it", "why": "it breaks any existing consumers reading that field"}, {"dont": "strip the raw platform string from ManifestEntry", "instead": "add platform_id as a new field", "why": "it is needed for display and debugging"}, {"dont": "use mean instead of median for display_ms reduction", "instead": "use median for robustness", "why": "a single cold-cache outlier inflates the mean"}], "approach": "All changes are additive to the read model - no existing fields are\nremoved or renamed. Add the new functions and fields, populate them\nin the existing `to_manifest_entry` and `to_detail_result` methods.\n\nThe canonical order:\n 1. Define models + constants (w3, w4) - no behavior, just schema\n 2. Define pure functions (_platform_id, _query_display_ms, select_canonical_row)\n 3. Wire into the transformer methods\n 4. Test with fixtures that exercise the edge cases\n\nDo not change how existing fields are computed. The new fields are\nderived from the same bundle data, just with a different reduction.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": [], "description": "The adversarial review (2026-04-15) identified two critical invariants\nthat must hold before any matrix leaderboard can be meaningful:\n\n**Invariant 1 - Platform identity is separate from provenance/visibility.**\nThe current pipeline writes whatever string appears in the bundle's\nplatform.name field directly to manifest.json and details/. In practice\nthis produces distinct rows for \"DuckDB-trust-ci\", \"DuckDB-trust-community\",\n\"DuckDB-trust-local\", etc., even though they are the same engine variant and\nthe trust information is already captured in the separate trust_label field.\nAny cross-platform matrix built on this read model is misleading because the\nplatform axis is polluted with provenance variants.\n\nFix: extract a canonical `platform_id` (e.g., \"duckdb\") alongside the raw\nplatform name, and define explicit ranking-eligibility rules that gate which\nrows may appear in ranked tables.\n\n**Invariant 2 - One canonical display value per logical query.**\nThe transformer preserves every measurement row (iter, stream) for provenance.\n`_geomean_ms()` therefore averages *all* execution samples, including\nthroughput streams. Meanwhile, Compare.tsx calls `find()` on the queries\narray and takes the first row - currently returning 16ms for a query whose\nsamples are [16, 11, 11, 11] (cold-cache outlier, not representative).\n\nNeither approach is wrong for its stated purpose, but neither is appropriate\nas the display value in a comparison matrix. The fix is a canonical reduction:\nmedian of successful measurement runs for each distinct logical query_id,\nexposed as `display_ms` per query and `display_geomean_ms` at the result\nlevel. The raw `geomean_ms` is preserved as an audit/detail field.\n\nThis TODO does **not** implement the matrix UI or the cohort artifact - it\nonly fixes the data contracts that every downstream feature depends on.\nIt replaces the dangling `explorer-extend-manifest-and-pipeline` dependency\nthat previously blocked `explorer-emit-benchmark-summary-artifact`.", "id": "explorer-fix-read-model-contract", "preserves": ["Existing `geomean_ms` field - raw-sample geomean preserved for backward compat; do not rename or remove", "Existing `platform` field on ManifestEntry and DetailResult - raw bundle string, unchanged for display/debugging", "Existing `queries: list[QueryTiming]` on DetailResult - raw per-sample timings preserved for audit; display_timings is additive", "Existing manifest.json schema shape - this adds fields, does not remove or rename any existing keys"], "prior_art": [], "priority": "critical", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/duckdb_builder.py"}, {"kind": "do_not_modify", "path_glob": "benchbox/core/explorer_pipeline/pipeline.py"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/**"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/transformer.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer/test_read_model_contract.py"}], "state": "done", "title": "Fix explorer read-model contract: platform/provenance split, canonical query reduction, ranking eligibility", "verifications": [{"command": "grep -n 'platform_id' benchbox/core/explorer_pipeline/models.py", "description": "platform_id field present in ManifestEntry", "expected": "field defined in ManifestEntry and DetailResult", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'display_' benchbox/core/explorer_pipeline/models.py", "description": "display_timings and display_geomean_ms present in DetailResult", "expected": "display_timings, display_geomean_ms, QueryDisplayTiming all present", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'RANKING_ELIGIBLE' benchbox/core/explorer_pipeline/models.py", "description": "RANKING_ELIGIBLE constants exported", "expected": "constants and is_ranking_eligible defined", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -n 'select_canonical_row' benchbox/core/explorer_pipeline/models.py", "description": "select_canonical_row exported", "expected": "function defined", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 4}, {"command": "uv run -- python -m pytest tests/unit/core/explorer/test_read_model_contract.py -v", "description": "Unit tests pass", "expected": "all tests pass", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 5}, {"command": "uv run -- python -m pytest tests/unit/core/explorer/ -v", "description": "Existing manifest tests still pass", "expected": "all tests pass", "item_id": "explorer-fix-read-model-contract", "last_result": null, "last_run": null, "seq": 6}], "work": [{"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": [], "notes": "`_platform_id(raw: str) -> str` strips known provenance suffixes\nfrom the bundle platform.name before slugifying:\n - Patterns to strip: `-trust-{ci,community,local,unknown}`,\n `-{ci,community,local}` when adjacent to a version segment.\n - After stripping, lowercase and replace spaces with dashes.\nExamples:\n \"DuckDB-trust-ci\" \u2192 \"duckdb\"\n \"DuckDB\" \u2192 \"duckdb\"\n \"polars-df\" \u2192 \"polars-df\"\n \"ClickHouse Cloud\" \u2192 \"clickhouse-cloud\"\n\nAdd `platform_id: str` to both ManifestEntry and DetailResult.\nPopulate from `_platform_id(platform)` in `to_manifest_entry` and\n`to_detail_result`. The raw `platform` field is unchanged - it\nremains the exact string from the bundle for display and debugging.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add `_platform_id()` normalizer and `platform_id` field to ManifestEntry and DetailResult", "wid": "w1"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": [], "notes": "Signature:\n `_query_display_ms(timings: list[QueryTiming], query_id: str) -> float | None`\n\nAlgorithm:\n 1. Filter to timings where `timing.query_id == query_id`\n AND `timing.status == \"pass\"`.\n 2. From passing timings, select `run_type == \"measurement\"` rows;\n if none have explicit run_type, accept all.\n 3. Compute median of `duration_ms` across the filtered set.\n 4. Return None if the filtered set is empty (all failed or query\n not present).\n\nMedian is chosen over mean because a single cold-cache outlier\n(common in power-test runs that include one warmup iteration before\nmeasurement) should not dominate the display value.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define `_query_display_ms()` canonical reduction function in transformer.py", "wid": "w2"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": ["w2"], "notes": "New model in models.py:\n class QueryDisplayTiming(BaseModel):\n query_id: str\n display_ms: float | None # None = all runs failed for this query\n sample_count: int # number of passing runs that contributed\n\nAdd to DetailResult:\n display_timings: list[QueryDisplayTiming] = []\n display_geomean_ms: float | None = None # geomean of display_ms values\n\nThe existing `geomean_ms` field is NOT renamed or removed - it\nremains the raw-sample geomean for backward compatibility and audit.\nManifestEntry also gets `display_geomean_ms: float | None = None`\n(used by the matrix for ranking when power_score is absent).\n\nPopulate in `to_detail_result` and `to_manifest_entry` by calling\n`_query_display_ms()` for each distinct query_id, then computing\ngeomean of the non-None results.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add QueryDisplayTiming model and display_timings + display_geomean_ms fields to DetailResult", "wid": "w3"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": [], "notes": "Per hosted-results-contract.md: mixed self-reported rows are not\neligible for ranked tables. Codify as frozensets:\n\n RANKING_ELIGIBLE_VISIBILITIES: frozenset[str] = frozenset({\n \"public-curated\",\n \"public-verified\",\n })\n\n RANKING_ELIGIBLE_TRUST_LABELS: frozenset[str] = frozenset({\n \"maintainer-run\",\n \"ci-verified\",\n })\n\n def is_ranking_eligible(entry: ManifestEntry) -> bool:\n return (\n entry.visibility in RANKING_ELIGIBLE_VISIBILITIES\n and entry.trust_label in RANKING_ELIGIBLE_TRUST_LABELS\n )\n\nExport all three from models.py `__all__`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define RANKING_ELIGIBLE_VISIBILITIES and RANKING_ELIGIBLE_TRUST_LABELS constants in models.py", "wid": "w4"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": ["w4"], "notes": "Signature:\n `select_canonical_row(entries: list[ManifestEntry]) -> ManifestEntry | None`\n\nGiven multiple ManifestEntries for the same (platform_id, benchmark,\nscale_factor, phase), selects the single canonical row for a ranked\nmatrix display. Returns None if the list is empty.\n\nSelection priority (descending):\n 1. `is_ranking_eligible(entry)` - ranking-eligible rows win\n 2. `run_date` descending - newest run wins within the same tier\n\nThe non-canonical rows are not discarded - they remain in the\nmanifest and detail pages. Only the matrix artifact applies this\nselection when building the default ranked view.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add select_canonical_row() in models.py for matrix default ranking", "wid": "w5"}, {"evidence": null, "item_id": "explorer-fix-read-model-contract", "needs": ["w1", "w2", "w3", "w4", "w5"], "notes": "Cases:\n (a) _platform_id strips known trust suffixes correctly\n (b) _platform_id is idempotent on clean names\n (c) _query_display_ms returns median of passing runs, not first\n (d) _query_display_ms returns None when all runs failed\n (e) _query_display_ms uses measurement-only rows when run_type present\n (f) display_geomean_ms differs from geomean_ms when samples vary\n (g) is_ranking_eligible returns False for community-submission trust\n (h) select_canonical_row picks ranking-eligible over non-eligible\n (i) select_canonical_row picks newest when both are ranking-eligible\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write unit tests in tests/unit/core/explorer/test_read_model_contract.py", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "duplicate the component for high-contrast mode", "instead": "use CSS custom properties that switch on a class", "why": "two components drift out of sync"}, {"dont": "fail the build on minor axe violations", "instead": "fail only on serious/critical", "why": "minor issues include color-contrast edge cases we may accept"}, {"dont": "rely on color alone", "instead": "include text and aria-label on every data-bearing element", "why": "4% of users have color vision deficiencies and screen readers ignore color entirely"}, {"dont": "use tabindex=0 on every cell", "instead": "use roving tabindex so tab exits the grid quickly", "why": "that makes keyboard nav painful (tab through 22\u00d78 cells)"}], "approach": "Accessibility work is most reliably driven by the tests themselves.\nAdd axe-core first, watch the existing tests fail, then fix\nviolations in priority order (keyboard \u2192 aria-label \u2192 contrast \u2192\nfocus visibility).\n\nFor the reduced-color mode: do not build a separate component tree.\nFlip a CSS class on the grid root; cell colors read from CSS custom\nproperties that switch value based on the class. This keeps the\nrender path single and the switch cost zero.\n\nKeyboard nav is the trickiest part. Use a roving tabindex pattern\n(only one cell has tabindex=0 at a time; arrow keys move it).\nReference pattern: WAI-ARIA Authoring Practices grid pattern.\n", "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-matrix-leaderboard-view"], "description": "The 2026-04-15 review flagged that the matrix leaderboard relies heavily\non color to convey information - a known accessibility hazard. Color is\ninvisible to screen readers and insufficient for users with color vision\ndeficiencies. The heatmap must remain readable without color.\n\nThis TODO:\n\n1. Ensures each heat cell carries a descriptive `aria-label` with the\n numeric timing and the relative ratio (\"18ms, 1.6\u00d7 fastest\").\n\n2. Adds a \"reduced color\" visual mode that falls back to monochrome\n lightness steps, controlled by the user's\n `prefers-contrast: more` media query and an explicit toggle.\n\n3. Adds keyboard navigation: tab reaches platform checkboxes,\n arrow keys navigate cells with an `aria-live` announcement.\n\n4. Backfills snapshot tests for the filter bar, empty states, and the\n integration between BenchmarkIndex and QueryHeatmap - the prior TODO\n only specified QueryHeatmap itself.\n\n5. Runs axe-core against the rendered matrix in the test suite and\n fails the build on any serious or critical violations.", "id": "explorer-heatmap-accessibility-and-tests", "preserves": ["Current default color mode - high-contrast is an opt-in, not a default swap", "Existing behavior of the matrix - accessibility is additive layering, not a rewrite", "Snapshot tests from explorer-matrix-leaderboard-view - this TODO extends, does not replace them"], "prior_art": [], "priority": "medium", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/package.json"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/BenchmarkIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/testing/axe-helper.ts"}], "state": "done", "title": "Accessibility hardening and snapshot test coverage for matrix leaderboard", "verifications": [{"command": "cd results-explorer && npm test -- --testPathPattern='(QueryHeatmap|BenchmarkIndex|Compare|Home)'", "description": "axe-core runs clean on the matrix page (no serious/critical violations)", "expected": "all tests pass", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 1}, {"command": "grep -n 'aria-label' results-explorer/src/components/QueryHeatmap.tsx", "description": "Heat cells have aria-label with numeric content", "expected": "aria-label present on cell", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -nE 'high-contrast|reduced-color' results-explorer/src/components/QueryHeatmap.tsx", "description": "Reduced-color mode is CSS-class driven, not a separate component", "expected": "mode is a class modifier", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles", "expected": "no errors", "item_id": "explorer-heatmap-accessibility-and-tests", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": [], "notes": "Cell: aria-label='${ms}ms, ${ratio}\u00d7 fastest in column' (or\n'no data' for null cells). Column headers: aria-sort='none'.\nRow checkboxes: aria-label='Select ${platform} for comparison'.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Enhance QueryHeatmap cells, headers, and checkboxes with descriptive aria labels", "wid": "w1"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1"], "notes": "Activates when prefers-contrast: more matches OR the user toggles\n'High contrast' in the filter bar. Cells use grayscale lightness\n(0-100%) instead of hsl hue. Numeric text remains primary signal.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Implement reduced-color mode via CSS class + custom properties", "wid": "w2"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1"], "notes": "Arrow keys within the matrix move focus between cells; focused\ncell announces itself via aria-live. Tab exits the grid to the\nnext filter control. Focus outline is visible (not just color).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add keyboard navigation (roving tabindex) and visible focus outlines", "wid": "w3"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1"], "notes": "Cases: (a) filter changes re-fetch correct artifact URL;\n(b) empty state renders when artifact has no platforms;\n(c) list-view toggle switches rendered component;\n(d) trust filter removes rows without refetch.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write integration snapshot tests at src/pages/__tests__/BenchmarkIndex.test.tsx", "wid": "w4"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-and-tests", "needs": ["w1", "w2", "w3"], "notes": "Add helper `expectNoAxeViolations(container)` and call it in\nQueryHeatmap, BenchmarkIndex, Compare, and Home page tests.\nWarn on minor; fail only on serious/critical.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Integrate axe-core into the test suite; fail on serious/critical violations", "wid": "w5"}], "worktree": "main"} +{"anti_patterns": [{"dont": "change configureAxe to suppress impact levels globally", "instead": "scope the config per-call or per-helper", "why": "that silences violations in all tests"}], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-heatmap-accessibility-and-tests. Four Required issues\n(broken CSS rule, invisible focus ring, empty columnheader, misleading\naria-sort) were fixed immediately post-review. This item tracks\nthe remaining nit/consider findings.", "id": "explorer-heatmap-accessibility-review-followups", "preserves": ["aria-label remains present on all cells in w1 - only the text format changes", "axe helper continues to suppress minor violations - only the implementation mechanism changes in w2"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from heatmap-accessibility code review", "verifications": [{"command": "cd results-explorer && npm test -- QueryHeatmap", "description": "QueryHeatmap axe tests still pass after axe-helper refactor", "expected": "all tests pass including axe assertions", "item_id": "explorer-heatmap-accessibility-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run typecheck && npm test", "description": "Full test suite green", "expected": "all green", "item_id": "explorer-heatmap-accessibility-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "QueryHeatmap.tsx:278 - aria-label uses ms.toFixed(1) (e.g. \"10.0 ms\")\nwhile the visible cell text uses fmtMs() which emits toFixed(0)\n(\"10 ms\"). Screen readers announce a value that differs from what\nis displayed.\n\nFix: use fmtMs(ms) in the aria-label string for consistency.\n\nAlso: for the fastest cell (ratio=1.0), the label reads \"10 ms, 1.0\u00d7\nfastest in column\" which is grammatically awkward - the cell IS the\nfastest, not a comparison to it. Special-case as \"10 ms, fastest in\ncolumn\" when ratio rounds to 1.0.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align aria-label text with visible cell text for timing values", "wid": "w1"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "axe-helper.ts:19-41 - The helper manually splits results.violations\ninto serious/critical vs minor arrays, reconstructs { violations:\ncritical }, and passes the partial object to toHaveNoViolations().\nThis works but relies on jest-axe's private object shape.\n\nThe idiomatic approach:\n import { configureAxe, toHaveNoViolations } from \"jest-axe\";\n const axe = configureAxe({ impactLevels: [\"serious\", \"critical\"] });\n expect(await axe(container)).toHaveNoViolations();\n\nThe helper becomes a clean two-liner; jest-axe handles the filtering\ninternally and is not coupled to the violation object shape.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace manual violation-split in axe-helper with configureAxe", "wid": "w2"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "QueryHeatmap.tsx:122-149 - handleCellKey handles ArrowLeft/Right/\nUp/Down but not Home/End. The WAI-ARIA 1.2 grid pattern recommends\n(not requires) Home to jump to first cell in row and End to last cell\nin row, Ctrl+Home to first cell in grid.\n\nLow priority: not a WCAG requirement for read-only data grids.\nWorth adding if AT user feedback surfaces the gap.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Home/End key support to keyboard grid navigation", "wid": "w3"}, {"evidence": null, "item_id": "explorer-heatmap-accessibility-review-followups", "needs": [], "notes": "QueryHeatmap.test.tsx currently renders with no onSelectionChange, so\nthe checkbox column header (hasSelection=false) is never in the axe\nDOM. The Required fix (w3 in the main TODO) added aria-label to the\nempty , but there is no regression test that exercises the\nselection path through axe.\n\nFix: add one axe test case in QueryHeatmap.test.tsx that passes\nselectedIds={new Set()} and onSelectionChange={() => {}} so axe runs\nagainst the full grid including the checkbox column header.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add axe test for QueryHeatmap with selection enabled", "wid": "w4"}], "worktree": "main"} +{"anti_patterns": [{"dont": "build a marketing landing page inside `/results/`", "instead": "make the leaderboard the first usable screen.", "why": "users came for benchmark answers"}, {"dont": "copy LLM leaderboard language such as model/provider/token throughput", "instead": "translate labels to platform, workload, latency, throughput, load time, and cost.", "why": "BenchBox benchmarks database platforms"}, {"dont": "hide submission CTAs entirely", "instead": "place them as compact secondary affordances near the data.", "why": "leaderboard attention feeds the submission flywheel"}, {"dont": "use a single dark/purple palette across charts and brand chrome", "instead": "keep brand tokens and chart tokens separate.", "why": "data encodings need semantic contrast"}], "approach": "Treat benchbox.dev as the source of brand expression and the explorer as the\ndata-dense product surface. Start by extracting the shared shell/tokens, then\nrearrange Home so leaderboard data appears before submission/list chrome. Use\ndensity from llm-stats for the table treatment, but keep BenchBox's code-first\ncredibility from the landing page. Test DOM order and labels so later visual\nedits cannot regress the first-impression hierarchy.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": [], "description": "Redesign the Results Explorer front door and shell so it feels like part of\nthe BenchBox product rather than a separate light dashboard. The user clarified\nthat the primary audience is benchmark users choosing a database, while the\nleaderboard browser must remain attention-grabbing because it feeds the\nsubmission flywheel. The target frame is therefore:\n\n\"BenchBox Database Leaderboards: reproducible rankings for choosing OLAP\nplatforms by workload, scale, deployment, and normalized cost.\"\n\nThis item covers identity, first impression, first-viewport density, and the\nshared BenchBox narrative. It should preserve the existing home\nMetaLeaderboard work, but make it the first meaningful content rather than\nburying it below generic hero/stat chrome.", "id": "explorer-leaderboard-first-identity-unification", "preserves": ["Chart color encodings remain data-driven and accessible; brand colors must not become categorical chart colors by default.", "Recent Results remains accessible, but not ahead of the primary leaderboard on the home page.", "The MetaLeaderboard keeps Times/Ranks/Speedup modes and URL-backed filters from the completed home leaderboard work.", "The existing `/results/`, `/results/{benchmark}/`, `/results/p/{platform_id}/`, `/results/r/{result_id}`, `/results/compare`, and `/results/query` routes remain stable."], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "landing/"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/BrandShell.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/Layout.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/LoadingSpinner.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/BrandShell.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/Layout.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.a11y.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/index.css"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Home.test.tsx"}], "state": "done", "title": "Explorer: unify Results Explorer and benchbox.dev into a leaderboard-first product identity", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test && npm run build", "description": "Frontend typecheck, unit tests, and production build pass.", "expected": "All commands pass with new Home/Layout coverage.", "item_id": "explorer-leaderboard-first-identity-unification", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium -- --grep \"Home\"", "description": "Manual browser smoke at desktop and mobile widths.", "expected": "Chromium browser smoke verifies `/results/` renders the Home shell and leaderboard; DOM-order unit tests cover first-viewport hierarchy.", "item_id": "explorer-leaderboard-first-identity-unification", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": [], "notes": "Read-only audit of `landing/` (tokens are extracted into the explorer;\n`landing/` itself is not edited in this work). Add the following CSS\ncustom properties to `results-explorer/src/index.css`:\n - surfaces: `--bb-bg-primary`, `--bb-bg-panel`, `--bb-bg-elevated`\n - text: `--bb-fg-primary`, `--bb-fg-muted`, `--bb-fg-inverse`\n - borders: `--bb-border-default`, `--bb-border-subtle`\n - accent: `--bb-accent`, `--bb-accent-hover`\n - status: `--bb-status-success`, `--bb-status-warning`,\n `--bb-status-danger`\n - code surfaces: `--bb-code-bg`, `--bb-code-fg`\n - typography: `--bb-font-sans`, `--bb-font-mono`, plus type-scale\n tokens `--bb-text-xs|sm|base|lg|xl|2xl`.\nChart palettes stay in their own `--bb-chart-*` namespace and must not\nbe aliased to brand tokens. Where landing and the existing explorer dark\ntheme disagree, defer to landing for chrome and to the explorer for\nchart/data encodings.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract shared BenchBox brand tokens from landing page into explorer theme variables", "wid": "w1"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w1"], "notes": "In results-explorer/src/components/Layout.tsx, add a compact global nav\nmatching benchbox.dev: BenchBox wordmark/name, Docs, Blog, Results, GitHub,\nand primary submit/run CTA. Below it, keep explorer-specific navigation\nfor Leaderboards, Benchmarks, Platforms, Compare, and Query. Results should\nbe visibly active when on `/results/`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Replace generic explorer header with shared global BenchBox navigation plus explorer subnav", "wid": "w2"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w1", "w2"], "notes": "In results-explorer/src/pages/Home.tsx, first viewport should contain:\n - H1: \"BenchBox Database Leaderboards\"\n - supporting line: \"Reproducible OLAP benchmark rankings by workload,\n scale, deployment, and normalized cost.\"\n - one dense cohort selector row: benchmark, scale factor, phase,\n deployment/cost coverage.\n - the MetaLeaderboard table immediately visible above the fold.\nUse a dark technical band or code-led surface from benchbox.dev only as\nframing; do not let decorative copy push the leaderboard down.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rewrite Home first viewport around a live leaderboard, not a brochure hero", "wid": "w3"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w3"], "notes": "Replace ambiguous stat cards with explicit labels such as:\n - \"22 supported benchmarks\"\n - \"N public result bundles\"\n - \"N platforms with public results\"\n - \"PR-validated corpus\"\nAvoid implying every supported benchmark has public leaderboard coverage.\nIf a value is unavailable from DuckDB metadata, compute it from existing\nmanifest/cohort tables or label it as \"supported\" rather than \"published\".\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Clarify supported vs published corpus stats in the home proof strip", "wid": "w4"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w3"], "notes": "Add a compact three-step strip or inline CTA cluster near the leaderboard:\n\"Run a benchmark\", \"Compare your result\", \"Submit a bundle\". Each should\nlink to the existing docs/submission flow where available. This should be\nsecondary to the leaderboard, not a marketing section.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Run -> Compare -> Submit flywheel affordance without reducing data density", "wid": "w5"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w1", "w3"], "notes": "This item OWNS the skeleton design system: token names, animation,\nsizing scale, and the Home + Layout consumers. The downstream\n`explorer-performance-perceived-latency` item then applies the same\nskeletons to MetaLeaderboard/BenchmarkIndex/Query/Compare without\nredesigning them. Replace generic spinner-only states on Home and the\nshell with dense skeletons that reserve final table geometry. Loading\nshould communicate that a static DuckDB snapshot is initializing,\nwithout blocking the brand/header render.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define skeleton tokens and Home/Layout loading states", "wid": "w6"}, {"evidence": null, "item_id": "explorer-leaderboard-first-identity-unification", "needs": ["w2", "w3", "w4", "w5"], "notes": "Replace any reliance on manual smoke with mechanical assertions in\n`Home.test.tsx` and `Layout.test.tsx`:\n - The MetaLeaderboard region appears before any \"Recent Results\" or\n submission CTA region in DOM order (`compareDocumentPosition` /\n `getAllByRole` index check).\n - The H1 text is exactly \"BenchBox Database Leaderboards\".\n - Global nav exposes Docs, Blog, Results, GitHub, and the run/submit\n CTA; the Results entry is visibly active when the route is `/`.\n - Stat cards use explicit `supported` / `published` labels (no\n ambiguous \"benchmarks\" copy).\nThese assertions back the manual browser smoke described under\n`verification`; the DOM-order guarantee is what stops a future visual\nedit from regressing first-impression hierarchy.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add automated DOM-order tests for first-viewport leaderboard hierarchy", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "fork MetaLeaderboard into a separate component", "instead": "extend in place", "why": "the shapes are the same post-widening"}, {"dont": "introduce a new color function", "instead": "call it", "why": "colorForCell is the shared log-scale palette"}, {"dont": "push filter state to a global store", "instead": "keep it local", "why": "per-page useUrlState is sufficient"}], "approach": "The pipeline-widening item is the enabling dependency; this item\nis pure frontend wiring on top of the widened MetaRank. Extend in\nplace rather than introducing a sibling MasterLeaderboard component\n(per plan) since the data shape already carries everything\nneeded. Reuse chartMath.colorForCell and QueryHeatmap's a11y\npattern - no new primitives. Default render mode on first visit:\nTimes, so the user immediately sees absolute values (the missing\nsurface this phase exists to restore).\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-17T00:00:00Z", "completed_pr": null, "created_at": "2026-04-17T00:00:00Z", "deferrals": [], "deps": ["explorer-pipeline-meta-leaderboard-widening", "explorer-url-synced-filter-state"], "description": "Phase B of the Results Explorer strategy revision\n(plans/purring-jingling-petal-ultraplan).\n\nReplace the current Home page's rank-only `MetaLeaderboard` with a\nClickBench-style Platform \u00d7 Cohort matrix of absolute values, with\ntoggles for Ranks and Speedup-vs-cohort-best, chip-based filters\n(benchmark family, SF, phase, tuning, trust tier, date window), and\nURL-synced state so views are shareable.\n\nThe \"Recent Results\" list stays (or moves below the matrix) - the\ngap being filled is the lack of a cross-benchmark, cross-platform\nhigh-level view with absolute values. This is the first thing a\nuser should see.\n\nDepends on:\n - explorer-pipeline-meta-leaderboard-widening (MetaRank must carry\n metric_value, speedup_vs_best, primary_metric, primary_order)\n - explorer-url-synced-filter-state (useUrlState hook)", "id": "explorer-master-leaderboard-home", "preserves": ["Existing MetaLeaderboard consumers (if any outside Home) continue to work - render-mode toggle defaults to current behavior (Ranks)", "Recent Results data surface remains accessible on Home (just demoted below the matrix)", "colorForCell remains the single log-scale color source shared with QueryHeatmap - no duplicate palettes"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.a11y.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Explorer Home: ClickBench-style master leaderboard (Times/Ranks/Speedup, chip filters, URL state)", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Explorer typecheck + Vitest suites green", "expected": "all green, including MetaLeaderboard.a11y and filter wiring tests", "item_id": "explorer-master-leaderboard-home", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run dev", "description": "End-to-end smoke on /results/", "expected": "/results/ shows master matrix; Times/Ranks/Speedup toggle works; applying benchmark + SF chips narrows columns/rows; URL round-trips; clicking a cohort header goes to /results/{benchmark}/?sf=X&phase=Y; row click goes to /results/p/{platform_id}/", "item_id": "explorer-master-leaderboard-home", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": [], "notes": "results-explorer/src/components/MetaLeaderboard.tsx - extend in\nplace (per plan: data shape already carries what each mode needs\nafter the pipeline widening). Modes:\n - Times: metric_value formatted via fmtGeomean (asc) / fmtScore (desc)\n - Ranks: current behavior (rank / total)\n - Speedup: speedup_vs_best, cohort best = 1.0\u00d7\nLog-scale color shading via existing colorForCell from\nsrc/lib/chartMath.ts (single source of truth, shared with\nQueryHeatmap).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extend MetaLeaderboard component with render-mode toggle (Times/Ranks/Speedup)", "wid": "w1"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": [], "notes": "results-explorer/src/types.ts - add metric_value,\nspeedup_vs_best, primary_metric, primary_order to MetaRank.\nAll optional so the type survives older artifacts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Widen MetaRank frontend type in types.ts", "wid": "w2"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w1", "w2"], "notes": "Filters: Benchmark family (multi), SF (multi), Phase\n(power/standard/throughput), Tuning mode, Trust tier, Date\nwindow. Each backed by useUrlState with a suitable serde\n(arraySerde for multis, stringSerde for singles).\nFilter state feeds both the visible cohort columns and the\nvisible platform rows.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Sticky filter chip row backed by useUrlState", "wid": "w3"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w3"], "notes": "results-explorer/src/pages/Home.tsx - make the matrix the\nprimary content surface. Do not wire cohort/row navigation yet\n(w6 handles that). Do not change the Recent Results position\nyet (w5 handles that). Scope: layout only.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Scaffold Home.tsx around the extended MetaLeaderboard as primary content", "wid": "w4"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w4"], "notes": "Keep the Recent Results component and data surface; move its\nrender position below the matrix on Home.tsx. No behavioral\nchange to the list itself.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Demote Recent Results list below the matrix", "wid": "w5"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w4"], "notes": "Cohort headers \u2192 /results/{benchmark}/?sf=X&phase=Y (preserve\nthe relevant slice of filter state via useUrlState's serde).\nRow click \u2192 /results/p/{platform_id}/.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire cohort header links and row-click navigation with preserved filter state", "wid": "w6"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w4"], "notes": "The new master matrix is structurally similar to QueryHeatmap -\nreuse the same a11y scaffolding (role=grid, aria-rowindex/col,\nroving tabindex). Existing tests like\nQueryHeatmap.a11y should continue to pass; author a sibling\nMetaLeaderboard.a11y test covering the equivalent contract.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Accessibility: reuse QueryHeatmap's role=grid + roving tabindex pattern", "wid": "w7"}, {"evidence": null, "item_id": "explorer-master-leaderboard-home", "needs": ["w5", "w6", "w7"], "notes": null, "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Vitest coverage for render-mode toggle, filter\u2192matrix wiring, cohort-link state preservation", "wid": "w8"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Non-required nits surfaced by the code review of explorer-matrix-leaderboard-view.\nNo Required issues were found; all critical implementation items are correct.\nThis item tracks four minor polish improvements.", "id": "explorer-matrix-leaderboard-review-followups", "preserves": ["Compare.tsx grouped chart rendering - only adding a comment in w1", "QueryHeatmap heat coloring behavior - only adding a comment in w2 and guarding an unused call in w4"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from matrix-leaderboard code review", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Full test suite green", "expected": "all green", "item_id": "explorer-matrix-leaderboard-review-followups", "last_result": null, "last_run": null, "seq": 1}], "work": [{"evidence": null, "item_id": "explorer-matrix-leaderboard-review-followups", "needs": [], "notes": "Compare.tsx:196-198 - grouped bar chart uses `?? 0` for null display_ms\nvalues (queries where all runs failed). Zero-height bars correctly\ndisappear visually, but the intent is non-obvious: a reader might wonder\nif 0 misrepresents failed queries as \"fastest.\"\n\nFix: add a one-line comment before the `?? 0`:\n // Null means all runs failed; render as 0 so bar disappears rather than\n // crashing the chart formatter.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add comment explaining ?? 0 fallback in Compare.tsx grouped chart", "wid": "w1"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-review-followups", "needs": [], "notes": "QueryHeatmap.tsx:74 - filter `v > 0` silently excludes timing=0ms\n(physically impossible but could occur with rounding artifacts).\n\nFix: add a comment:\n // Exclude zero/null cells; sub-ms values round to 0 in some runners.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Document zero-timing filter intent in QueryHeatmap.tsx", "wid": "w2"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-review-followups", "needs": [], "notes": "manifest.ts:79-80 - HTML detection uses `contentType.includes(\"text/html\")`.\nIf a real 5xx error page uses `application/json` content-type, the gzip\nDecompressionStream would throw with a cryptic \"DecompressionStream error\"\nrather than a meaningful HTTP error.\n\nFix: check `!res.ok` first:\n if (!res.ok) throw new Error(`HTTP ${res.status} fetching ${url}`);\nThen proceed to content-type check for Vite SPA fallback detection.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Improve getBenchmarkSummary to also check !res.ok before content-type", "wid": "w3"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-review-followups", "needs": [], "notes": "QueryHeatmap.tsx:272 - lightnessForCell is called unconditionally even\nwhen suppressHeat=true (single-platform view). The result is immediately\ndiscarded. This is a negligible waste but could confuse a reader.\n\nFix: guard the call:\n const lightness = suppressHeat ? null : lightnessForCell(ms, minInCol);\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Suppress lightnessForCell call when suppressHeat is true", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "use the benchmark-aware choice from explorer-align-ranking-metric-with-tpc-standards", "dont": "default the ranking metric to geomean_ms for TPC-H/DS", "why": "those benchmarks define power_score as the standard metric"}, {"INSTEAD": "use CSS custom properties", "dont": "emit color via hard-coded hex", "why": "dark mode and high-contrast accessibility settings need to override"}, {"INSTEAD": "render them with a visible tier badge instead", "dont": "hide community-tier rows by default", "why": "hiding them defeats the trust model"}, {"INSTEAD": "always include numeric + relative label", "dont": "omit aria-label on heat cells", "why": "colored cells alone are invisible to screen readers"}, {"INSTEAD": "use log10(ratio-to-fastest) with a 10x clamp", "dont": "use linear min-max coloring", "why": "linear compresses sub-2x gaps when a 80x outlier is present"}], "approach": "The heatmap is a single component with no internal state beyond\nselection. All data comes from the summary prop. Filter controls\nlive in BenchmarkIndex and drive the (sf, phase) \u2192 artifact URL\nselection + client-side filtering of rows by tuning/trust.\n\nFor coloring: write a pure function colorForCell(ms, minInCol,\nmaxInCol) \u2192 CSSValue. Unit-test it directly with known inputs before\nwiring it to the component. The color math is where a proposal most\neasily goes wrong; keep it isolated and snapshot-tested.\n\nFor the log-ratio color: clamp the ratio at 10x so a 100x-slower\noutlier doesn't compress everything else. Values above the clamp\nstill render red but don't stretch the scale further.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-align-ranking-metric-with-tpc-standards", "explorer-emit-benchmark-summary-artifact"], "description": "The strategy review (2026-04-15) concluded that the explorer's primary\nsurface per benchmark must be a Platform \u00d7 Query matrix leaderboard,\nmodelled on ClickBench but adapted to BenchBox's multi-dimensional data\n(trust tiers, tuning modes, phases). The current BenchmarkIndex.tsx\nrenders a sorted list - this TODO replaces it with the matrix.\n\nThe matrix is the product definition: arriving at /results/tpch/ should\nimmediately answer \"which engine wins, and by how much, on each query?\"\nwithout navigating further.\n\nLayout:\n\n [Scale \u25be] [Tuning \u25be] [Phase \u25be] [Trust \u25be] [View: Matrix | List]\n\n Platform Trust Power/Geomean Cost Q1 Q2 ... Q22\n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n DuckDB maintainer 3184.9 / 11.5ms $0 18ms 4ms ... 6ms\n Polars maintainer -- / 8.2ms $0 12ms 3ms ... 4ms\n DataFusion community 2847.1 / 13.7ms $0 22ms 6ms ... 9ms\n\nHeat coloring rules (corrected from the original proposal):\n\n- Per-column normalization (each query independently).\n- **Log scale**, not linear - SQLite at 892ms vs DuckDB at 11ms would\n collapse the DuckDB/Polars/DataFusion gradation under linear.\n- Scale is **ratio-to-fastest-in-column** (ClickBench-style),\n not min-max.\n- Null cells render \"-\" in muted gray with `aria-label=\"no data\"`.\n\nRow selection: checkboxes on each row feed the existing Compare page\nvia URL-encoded IDs. See explorer-compare-url-short-ids for the ID\nencoding concern.", "id": "explorer-matrix-leaderboard-view", "preserves": ["BenchmarkIndex list view remains available via a toggle for mobile users", "Compare page contract - selection emits URLs the Compare page already understands", "Existing /results/:benchmark/ route - do not change URL pattern, only the rendered content", "Existing manifest.json consumers - this TODO adds a new loader, does not remove the existing one"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/**"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/QueryHeatmap.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryReduction.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Build ClickBench-style matrix leaderboard as primary benchmark landing view", "verifications": [{"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles", "expected": "no errors", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm test -- QueryHeatmap", "description": "Snapshot tests pass", "expected": "all tests pass", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'QueryHeatmap' results-explorer/src/pages/BenchmarkIndex.tsx", "description": "BenchmarkIndex renders QueryHeatmap by default", "expected": "imported and rendered as default view", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -nE 'export.*colorForCell' results-explorer/src/components/QueryHeatmap.tsx", "description": "Color function is pure and exported for testing", "expected": "function is exported", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": [], "notes": "Mirror the Pydantic model in src/types.ts. Loader fetches and\ndecompresses the gzipped artifact via DecompressionStream API.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add BenchmarkSummary TS type and getBenchmarkSummary(benchmark, sf, phase) loader in src/lib/manifest.ts", "wid": "w1"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w1"], "notes": "Props: summary (BenchmarkSummary), rankingMetric ('power_score' |\n'geomean_ms'). Sticky left-column (platform + trust badge +\nmetric), horizontally scrollable query cells, row-level checkbox.\n\nColor math: per column, log10(ms / min_in_column) mapped to hsl\nhue 120\u00b0 (green) \u2192 0\u00b0 (red) across [0, log10(10)] clamped.\nCSS custom properties for the color so high-contrast and dark\nmodes can override. Each cell emits numeric text AND\n`aria-label=\"${ms}ms, ${ratio}\u00d7 slowest\"` for screen readers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build QueryHeatmap component at src/components/QueryHeatmap.tsx with log-ratio heat coloring", "wid": "w2"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w2"], "notes": "Cases: (a) known timings produce stable color classes; (b) null\ncells render '-'; (c) single-platform matrix degrades gracefully;\n(d) empty matrix shows empty state; (e) ratio calculation matches\nexpected values for a 10x-range fixture.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write snapshot tests at src/components/__tests__/QueryHeatmap.test.tsx", "wid": "w3"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w2"], "notes": "Load BenchmarkSummary and render QueryHeatmap. Keep existing\nsorted-list behind a 'List view' toggle for mobile. Add\nScale/Tuning/Phase/Trust filter controls; Scale and Phase are\nrequired (one value each); Tuning and Trust default to 'All'.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rewrite BenchmarkIndex.tsx to render QueryHeatmap as default view with filter controls", "wid": "w4"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w4"], "notes": "Selecting \u22652 rows reveals a sticky button that navigates to\n/results/compare?ids=. Uses short IDs once\nexplorer-compare-url-short-ids lands; until then falls back to\nfull result_ids.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire row checkboxes to Compare URL with sticky 'Compare N selected' button", "wid": "w5"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w4"], "notes": "When only 1 platform has run a (benchmark, scale, phase) tuple,\nhide heat coloring and show 'Only 1 result in this cohort - add\nmore with benchbox run ...'. When 0 query overlap across\nplatforms, show 'No shared queries to compare'.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Empty-state + single-platform handling with guidance banners", "wid": "w6"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": [], "notes": "The current Compare.tsx calls find() to get the first QueryTiming\nrow for each query (Compare.tsx:107, :350). This returns the\ncold-cache outlier when multiple measurement iterations exist (e.g.,\n[16, 11, 11, 11] \u2192 16ms displayed).\n\nFix: read `display_timings` (list[QueryDisplayTiming]) from the\ndetail JSON instead of scanning the raw `queries` array. The\n`display_ms` value per query_id is the median of successful runs,\ncomputed by the pipeline (see explorer-fix-read-model-contract).\n\nAdd a helper in src/lib/queryReduction.ts:\n `getDisplayMs(detail: DetailResult, queryId: string): number | null`\nIt reads `detail.display_timings.find(t => t.query_id === queryId)?.display_ms ?? null`.\nReplace both find() call-sites in Compare.tsx with this helper.\n\nThis makes the matrix and the Compare page agree on what a query's\nperformance is - they will now both use the same canonical reduced\nvalue from the pipeline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix Compare.tsx to use display_ms from DetailResult instead of first-row raw duration_ms", "wid": "w7"}], "worktree": "main"} +{"anti_patterns": [{"dont": "default the ranking metric to geomean_ms for TPC-H/DS", "instead": "use the benchmark-aware choice from explorer-align-ranking-metric-with-tpc-standards", "why": "those benchmarks define power_score as the standard metric"}, {"dont": "emit color via hard-coded hex", "instead": "use CSS custom properties", "why": "dark mode and high-contrast accessibility settings need to override"}, {"dont": "hide community-tier rows by default", "instead": "render them with a visible tier badge instead", "why": "hiding them defeats the trust model"}, {"dont": "omit aria-label on heat cells", "instead": "always include numeric + relative label", "why": "colored cells alone are invisible to screen readers"}, {"dont": "use linear min-max coloring", "instead": "use log10(ratio-to-fastest) with a 10x clamp", "why": "linear compresses sub-2x gaps when a 80x outlier is present"}], "approach": "The heatmap is a single component with no internal state beyond\nselection. All data comes from the summary prop. Filter controls\nlive in BenchmarkIndex and drive the (sf, phase) \u2192 artifact URL\nselection + client-side filtering of rows by tuning/trust.\n\nFor coloring: write a pure function colorForCell(ms, minInCol,\nmaxInCol) \u2192 CSSValue. Unit-test it directly with known inputs before\nwiring it to the component. The color math is where a proposal most\neasily goes wrong; keep it isolated and snapshot-tested.\n\nFor the log-ratio color: clamp the ratio at 10x so a 100x-slower\noutlier doesn't compress everything else. Values above the clamp\nstill render red but don't stretch the scale further.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-15T00:00:00Z", "completed_pr": null, "created_at": "2026-04-15T00:00:00Z", "deferrals": [], "deps": ["explorer-align-ranking-metric-with-tpc-standards", "explorer-emit-benchmark-summary-artifact"], "description": "The strategy review (2026-04-15) concluded that the explorer's primary\nsurface per benchmark must be a Platform \u00d7 Query matrix leaderboard,\nmodelled on ClickBench but adapted to BenchBox's multi-dimensional data\n(trust tiers, tuning modes, phases). The current BenchmarkIndex.tsx\nrenders a sorted list - this TODO replaces it with the matrix.\n\nThe matrix is the product definition: arriving at /results/tpch/ should\nimmediately answer \"which engine wins, and by how much, on each query?\"\nwithout navigating further.\n\nLayout:\n\n [Scale \u25be] [Tuning \u25be] [Phase \u25be] [Trust \u25be] [View: Matrix | List]\n\n Platform Trust Power/Geomean Cost Q1 Q2 ... Q22\n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n DuckDB maintainer 3184.9 / 11.5ms $0 18ms 4ms ... 6ms\n Polars maintainer -- / 8.2ms $0 12ms 3ms ... 4ms\n DataFusion community 2847.1 / 13.7ms $0 22ms 6ms ... 9ms\n\nHeat coloring rules (corrected from the original proposal):\n\n- Per-column normalization (each query independently).\n- **Log scale**, not linear - SQLite at 892ms vs DuckDB at 11ms would\n collapse the DuckDB/Polars/DataFusion gradation under linear.\n- Scale is **ratio-to-fastest-in-column** (ClickBench-style),\n not min-max.\n- Null cells render \"-\" in muted gray with `aria-label=\"no data\"`.\n\nRow selection: checkboxes on each row feed the existing Compare page\nvia URL-encoded IDs. See explorer-compare-url-short-ids for the ID\nencoding concern.", "id": "explorer-matrix-leaderboard-view", "preserves": ["BenchmarkIndex list view remains available via a toggle for mobile users", "Compare page contract - selection emits URLs the Compare page already understands", "Existing /results/:benchmark/ route - do not change URL pattern, only the rendered content", "Existing manifest.json consumers - this TODO adds a new loader, does not remove the existing one"], "prior_art": [], "priority": "high", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/**"}, {"kind": "do_not_modify", "path_glob": "results-explorer/src/pages/ResultDetail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/QueryHeatmap.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/manifest.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryReduction.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/types.ts"}], "state": "done", "title": "Build ClickBench-style matrix leaderboard as primary benchmark landing view", "verifications": [{"command": "cd results-explorer && npm run typecheck", "description": "TypeScript compiles", "expected": "no errors", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm test -- QueryHeatmap", "description": "Snapshot tests pass", "expected": "all tests pass", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 2}, {"command": "grep -n 'QueryHeatmap' results-explorer/src/pages/BenchmarkIndex.tsx", "description": "BenchmarkIndex renders QueryHeatmap by default", "expected": "imported and rendered as default view", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 3}, {"command": "grep -nE 'export.*colorForCell' results-explorer/src/components/QueryHeatmap.tsx", "description": "Color function is pure and exported for testing", "expected": "function is exported", "item_id": "explorer-matrix-leaderboard-view", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": [], "notes": "Mirror the Pydantic model in src/types.ts. Loader fetches and\ndecompresses the gzipped artifact via DecompressionStream API.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add BenchmarkSummary TS type and getBenchmarkSummary(benchmark, sf, phase) loader in src/lib/manifest.ts", "wid": "w1"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w1"], "notes": "Props: summary (BenchmarkSummary), rankingMetric ('power_score' |\n'geomean_ms'). Sticky left-column (platform + trust badge +\nmetric), horizontally scrollable query cells, row-level checkbox.\n\nColor math: per column, log10(ms / min_in_column) mapped to hsl\nhue 120\u00b0 (green) \u2192 0\u00b0 (red) across [0, log10(10)] clamped.\nCSS custom properties for the color so high-contrast and dark\nmodes can override. Each cell emits numeric text AND\n`aria-label=\"${ms}ms, ${ratio}\u00d7 slowest\"` for screen readers.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build QueryHeatmap component at src/components/QueryHeatmap.tsx with log-ratio heat coloring", "wid": "w2"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w2"], "notes": "Cases: (a) known timings produce stable color classes; (b) null\ncells render '-'; (c) single-platform matrix degrades gracefully;\n(d) empty matrix shows empty state; (e) ratio calculation matches\nexpected values for a 10x-range fixture.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Write snapshot tests at src/components/__tests__/QueryHeatmap.test.tsx", "wid": "w3"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w2"], "notes": "Load BenchmarkSummary and render QueryHeatmap. Keep existing\nsorted-list behind a 'List view' toggle for mobile. Add\nScale/Tuning/Phase/Trust filter controls; Scale and Phase are\nrequired (one value each); Tuning and Trust default to 'All'.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Rewrite BenchmarkIndex.tsx to render QueryHeatmap as default view with filter controls", "wid": "w4"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w4"], "notes": "Selecting \u22652 rows reveals a sticky button that navigates to\n/results/compare?ids=. Uses short IDs once\nexplorer-compare-url-short-ids lands; until then falls back to\nfull result_ids.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Wire row checkboxes to Compare URL with sticky 'Compare N selected' button", "wid": "w5"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": ["w4"], "notes": "When only 1 platform has run a (benchmark, scale, phase) tuple,\nhide heat coloring and show 'Only 1 result in this cohort - add\nmore with benchbox run ...'. When 0 query overlap across\nplatforms, show 'No shared queries to compare'.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Empty-state + single-platform handling with guidance banners", "wid": "w6"}, {"evidence": null, "item_id": "explorer-matrix-leaderboard-view", "needs": [], "notes": "The current Compare.tsx calls find() to get the first QueryTiming\nrow for each query (Compare.tsx:107, :350). This returns the\ncold-cache outlier when multiple measurement iterations exist (e.g.,\n[16, 11, 11, 11] \u2192 16ms displayed).\n\nFix: read `display_timings` (list[QueryDisplayTiming]) from the\ndetail JSON instead of scanning the raw `queries` array. The\n`display_ms` value per query_id is the median of successful runs,\ncomputed by the pipeline (see explorer-fix-read-model-contract).\n\nAdd a helper in src/lib/queryReduction.ts:\n `getDisplayMs(detail: DetailResult, queryId: string): number | null`\nIt reads `detail.display_timings.find(t => t.query_id === queryId)?.display_ms ?? null`.\nReplace both find() call-sites in Compare.tsx with this helper.\n\nThis makes the matrix and the Compare page agree on what a query's\nperformance is - they will now both use the same canonical reduced\nvalue from the pipeline.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Fix Compare.tsx to use display_ms from DetailResult instead of first-row raw duration_ms", "wid": "w7"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-cross-benchmark-meta-leaderboard. Three Required issues were\nfixed immediately post-review:\n - _get_val() now uses getattr(row, primary_metric) instead of hardcoded field names\n - href benchmark name is URL-encoded\n - platform_records sort is stable (secondary sort by platform_id)\nThis item tracks the remaining nit/consider findings.", "id": "explorer-meta-leaderboard-review-followups", "preserves": ["MetaLeaderboard rendering for all current artifact shapes", "avg_rank sort order - only adding a comment in w5, no logic change"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from cross-benchmark meta-leaderboard code review", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -n 0", "description": "Pipeline unit tests pass", "expected": "all passed", "item_id": "explorer-meta-leaderboard-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run typecheck && npm test", "description": "TypeScript test suite green", "expected": "all green", "item_id": "explorer-meta-leaderboard-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-meta-leaderboard-review-followups", "needs": [], "notes": "Home.tsx:80-82 - guard checks `metaLeaderboard && metaLeaderboard.cohorts.length > 0`\nbefore rendering MetaLeaderboard. But MetaLeaderboard.tsx:23-25 already handles\nempty data (returns null when platforms.length === 0 or cohorts.length === 0).\nThe outer check is redundant.\n\nFix: simplify to:\n {metaLeaderboard && }\n\nMetaLeaderboard handles the empty-data case internally.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Simplify Home.tsx meta-leaderboard guard to trust MetaLeaderboard.tsx internal guard", "wid": "w1"}, {"evidence": null, "item_id": "explorer-meta-leaderboard-review-followups", "needs": [], "notes": "MetaLeaderboard.tsx:102 - avg_rank displayed with `.toFixed(1)` (1 decimal).\nPipeline emits 2 decimal places (pipeline.py:244).\nResult: pipeline emits 1.25, frontend shows \"1.2\".\n\nFix: either display 2 decimals in UI (.toFixed(2)) or reduce pipeline precision to 1\n(round(..., 1)). The 2-decimal pipeline value is the authority.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align avg_rank decimal display in MetaLeaderboard.tsx with pipeline precision", "wid": "w2"}, {"evidence": null, "item_id": "explorer-meta-leaderboard-review-followups", "needs": [], "notes": "MetaLeaderboard.tsx:23 - `const { cohorts, platforms } = data;`\nIf a corrupted or partial meta-leaderboard.json.gz is served, `platforms`\ncould be undefined and `.length` would crash.\n\nFix:\n const { cohorts = [], platforms = [] } = data ?? {};\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add defensive destructuring defaults in MetaLeaderboard.tsx", "wid": "w3"}, {"evidence": null, "item_id": "explorer-meta-leaderboard-review-followups", "needs": [], "notes": "pipeline.py:185-188 - cohort_key, label, and href derivation is repeated\ninline inside the loop. Could be a small helper:\n def _cohort_metadata(benchmark, scale_factor, phase):\n sf = _sf_str(scale_factor)\n return {\"key\": f\"...\", \"label\": f\"...\", \"href\": f\"...\"}\n\nLow value until the loop needs modification; extract when next touched.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Extract _cohort_metadata helper in pipeline.py", "wid": "w4"}, {"evidence": null, "item_id": "explorer-meta-leaderboard-review-followups", "needs": [], "notes": "pipeline.py:244 - avg_rank is rounded to 2 decimal places. Two platforms\nwith identical averages (e.g., both 1.25) are ordered by platform_id\n(secondary key added in the Required fix above). Add a comment documenting\nthe tie-breaking rule so future readers don't wonder:\n # Platforms with equal avg_rank are ordered alphabetically by platform_id.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add avg_rank rounding comment explaining tie-breaking", "wid": "w5"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "show active filters and result count first, full filters in a drawer.", "dont": "put all filters above all data on mobile", "why": "it hides the leaderboard answer"}, {"INSTEAD": "use responsive variants.", "dont": "reduce desktop table density to solve mobile", "why": "desktop is the primary analysis surface"}, {"INSTEAD": "add sticky labels and visible overflow affordance or a card fallback.", "dont": "silently clip heatmaps", "why": "users cannot tell more columns exist"}], "approach": "Implement responsive primitives once, then use them on Home, BenchmarkIndex,\nQuery, and Compare. Start with CSS/layout and DOM ordering before adding new\ncomponents. Validate with Playwright at representative widths because visual\nregressions here are hard to catch with unit tests alone.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": ["explorer-chart-correctness-and-navigation", "explorer-leaderboard-first-identity-unification", "explorer-selection-facets-coverage-at-scale"], "description": "Make the Results Explorer usable below 768px without simply stacking all\nfilters before all data. The audit found that the mobile first viewport can\nshow hero/stat/submission chrome before the leaderboard, benchmark heatmaps can\nclip horizontally, and Query Workbench filters can dominate the screen before\nusers see any rows. The mobile design should preserve the leaderboard-first\nidentity while adapting dense tables into scrollable, card, or tabular\npatterns with explicit affordances.", "id": "explorer-mobile-density-responsive-tables", "preserves": ["Desktop density does not regress: at 1440px, MetaLeaderboard renders at least 8 rows above the fold (asserted by w8 Playwright check).", "FacetDrawer prop API and the underlying facet model from `explorer-selection-facets-coverage-at-scale` w6 are not modified by this work.", "Horizontal scroll, where used, is explicit and keeps row/header context visible (sticky first column + visible scroll-hint element).", "Keyboard navigation and accessible labels remain available for tables, drawers, and charts.", "URL state remains the source of active filters across responsive layouts (no mobile-only state shadow)."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/e2e/responsive.spec.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/playwright.config.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ChartPanel.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/FacetRail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MobileFilterDrawer.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ResponsiveDataTable.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/ChartPanel.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/FacetRail.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MobileFilterDrawer.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/QueryHeatmap.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/ResponsiveDataTable.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/index.css"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Query.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/BenchmarkIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Compare.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Home.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Query.test.tsx"}], "state": "done", "title": "Explorer: mobile-responsive dense leaderboard, filters, and charts", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test && npm run build", "description": "Frontend typecheck, unit tests, and build pass.", "expected": "All commands pass after responsive component changes.", "item_id": "explorer-mobile-density-responsive-tables", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium", "description": "Responsive Playwright smoke passes.", "expected": "Chromium e2e tests pass at configured desktop and mobile viewports.", "item_id": "explorer-mobile-density-responsive-tables", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "Add `--bb-bp-mobile`, `--bb-bp-tablet`, `--bb-bp-desktop` to\n`index.css` (consistent with the brand tokens added by the upstream\nidentity unification work). Document the rule \"use horizontal table\nscroll above the mobile breakpoint when the table has a stable sticky\nfirst column; below it, fall back to compact platform cards\" once, in\na CSS comment near the breakpoint tokens. No page edits.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define responsive breakpoint tokens and table-vs-card decision rule", "wid": "w1"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w1"], "notes": "At 390px width, the Home page must render in this DOM order: headline,\nactive cohort/filter summary, first leaderboard rows (or the compact\nResponsiveDataTable card variant), and only after that any Recent\nResults or submission CTA strip. Pull leaderboard ordering from the\ndesktop rules established in `explorer-leaderboard-first-identity-\nunification` w7 \u2014 do not redefine them.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply mobile DOM order on Home: headline, cohort summary, leaderboard, then chrome", "wid": "w2"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "The FacetDrawer COMPONENT is owned by `explorer-selection-facets-\ncoverage-at-scale` w6. This work unit ONLY adds narrow-width styling\n(full-screen sheet behavior, swipe-to-dismiss optional, focus trap when\nopen). It must not change the drawer's prop shape or the facet model.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Style the FacetDrawer for narrow widths", "wid": "w3"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w3"], "notes": "The drawer trigger button shows active filter chips and live result\ncount BEFORE the user opens it. Result count comes from the same\n`facetsToWhereClause` count query the desktop rail uses; do not run a\nseparate count query just for the mobile trigger.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add closed-state summary chip strip and live result count to drawer trigger", "wid": "w4"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "QueryHeatmap/benchmark matrices should use sticky row labels, stable cell\ndimensions, and an explicit horizontal overflow affordance. On very narrow\nscreens, provide a list/card fallback grouped by platform with query\noutliers and aggregate metric.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make benchmark matrices and heatmaps responsive without silent clipping", "wid": "w5"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w4"], "notes": "Query page mobile order should be: title/result count, active filters,\ntable rows or SQL panel tabs, then full filters in a drawer. The default\nshould not require scrolling past every facet before seeing data.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Put Query Workbench results before deep filter controls on mobile", "wid": "w6"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "Chart group controls should remain tappable and not overflow. Use icon or\nshort labels where necessary, with accessible names. Do not let chart chips\nresize the chart area on selection.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make chart controls wrap into stable segmented controls at small widths", "wid": "w7"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w2", "w4", "w5", "w6", "w7"], "notes": "Mechanical assertions, not screenshot-only smokes. For `/results/`,\n`/results/star_schema/?phase=standard`, `/results/query`, and\n`/results/compare?ids=...`, at 390px AND 1440px:\n - Headline + cohort summary + first leaderboard row are within the\n first 800px of scroll on desktop and within the first 1200px on\n mobile (use `boundingBox().y`).\n - At 1440px, MetaLeaderboard renders >= 8 rows above the fold (desktop\n density assertion \u2014 fails if mobile responsive rules accidentally\n truncate the desktop layout).\n - QueryHeatmap container reports `scrollWidth > clientWidth` \u21d2 a\n visible scroll-hint element exists in the DOM.\n - FacetDrawer trigger shows the live result count attribute before the\n drawer is opened.\nScreenshots stay as a secondary regression net.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Playwright responsive assertions (not just screenshots) for DOM order, density, and overflow", "wid": "w8"}], "worktree": "main"} +{"anti_patterns": [{"dont": "put all filters above all data on mobile", "instead": "show active filters and result count first, full filters in a drawer.", "why": "it hides the leaderboard answer"}, {"dont": "reduce desktop table density to solve mobile", "instead": "use responsive variants.", "why": "desktop is the primary analysis surface"}, {"dont": "silently clip heatmaps", "instead": "add sticky labels and visible overflow affordance or a card fallback.", "why": "users cannot tell more columns exist"}], "approach": "Implement responsive primitives once, then use them on Home, BenchmarkIndex,\nQuery, and Compare. Start with CSS/layout and DOM ordering before adding new\ncomponents. Validate with Playwright at representative widths because visual\nregressions here are hard to catch with unit tests alone.\n", "blocked_reason": null, "category": "User Experience", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": ["explorer-chart-correctness-and-navigation", "explorer-leaderboard-first-identity-unification", "explorer-selection-facets-coverage-at-scale"], "description": "Make the Results Explorer usable below 768px without simply stacking all\nfilters before all data. The audit found that the mobile first viewport can\nshow hero/stat/submission chrome before the leaderboard, benchmark heatmaps can\nclip horizontally, and Query Workbench filters can dominate the screen before\nusers see any rows. The mobile design should preserve the leaderboard-first\nidentity while adapting dense tables into scrollable, card, or tabular\npatterns with explicit affordances.", "id": "explorer-mobile-density-responsive-tables", "preserves": ["Desktop density does not regress: at 1440px, MetaLeaderboard renders at least 8 rows above the fold (asserted by w8 Playwright check).", "FacetDrawer prop API and the underlying facet model from `explorer-selection-facets-coverage-at-scale` w6 are not modified by this work.", "Horizontal scroll, where used, is explicit and keeps row/header context visible (sticky first column + visible scroll-hint element).", "Keyboard navigation and accessible labels remain available for tables, drawers, and charts.", "URL state remains the source of active filters across responsive layouts (no mobile-only state shadow)."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/e2e/responsive.spec.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/playwright.config.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ChartPanel.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/FacetRail.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MobileFilterDrawer.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/QueryHeatmap.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/ResponsiveDataTable.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/ChartPanel.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/FacetRail.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MobileFilterDrawer.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/QueryHeatmap.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/ResponsiveDataTable.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/index.css"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Query.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/BenchmarkIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Compare.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Home.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Query.test.tsx"}], "state": "done", "title": "Explorer: mobile-responsive dense leaderboard, filters, and charts", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test && npm run build", "description": "Frontend typecheck, unit tests, and build pass.", "expected": "All commands pass after responsive component changes.", "item_id": "explorer-mobile-density-responsive-tables", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium", "description": "Responsive Playwright smoke passes.", "expected": "Chromium e2e tests pass at configured desktop and mobile viewports.", "item_id": "explorer-mobile-density-responsive-tables", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "Add `--bb-bp-mobile`, `--bb-bp-tablet`, `--bb-bp-desktop` to\n`index.css` (consistent with the brand tokens added by the upstream\nidentity unification work). Document the rule \"use horizontal table\nscroll above the mobile breakpoint when the table has a stable sticky\nfirst column; below it, fall back to compact platform cards\" once, in\na CSS comment near the breakpoint tokens. No page edits.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Define responsive breakpoint tokens and table-vs-card decision rule", "wid": "w1"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w1"], "notes": "At 390px width, the Home page must render in this DOM order: headline,\nactive cohort/filter summary, first leaderboard rows (or the compact\nResponsiveDataTable card variant), and only after that any Recent\nResults or submission CTA strip. Pull leaderboard ordering from the\ndesktop rules established in `explorer-leaderboard-first-identity-\nunification` w7 \u2014 do not redefine them.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply mobile DOM order on Home: headline, cohort summary, leaderboard, then chrome", "wid": "w2"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "The FacetDrawer COMPONENT is owned by `explorer-selection-facets-\ncoverage-at-scale` w6. This work unit ONLY adds narrow-width styling\n(full-screen sheet behavior, swipe-to-dismiss optional, focus trap when\nopen). It must not change the drawer's prop shape or the facet model.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Style the FacetDrawer for narrow widths", "wid": "w3"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w3"], "notes": "The drawer trigger button shows active filter chips and live result\ncount BEFORE the user opens it. Result count comes from the same\n`facetsToWhereClause` count query the desktop rail uses; do not run a\nseparate count query just for the mobile trigger.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add closed-state summary chip strip and live result count to drawer trigger", "wid": "w4"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "QueryHeatmap/benchmark matrices should use sticky row labels, stable cell\ndimensions, and an explicit horizontal overflow affordance. On very narrow\nscreens, provide a list/card fallback grouped by platform with query\noutliers and aggregate metric.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make benchmark matrices and heatmaps responsive without silent clipping", "wid": "w5"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w4"], "notes": "Query page mobile order should be: title/result count, active filters,\ntable rows or SQL panel tabs, then full filters in a drawer. The default\nshould not require scrolling past every facet before seeing data.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Put Query Workbench results before deep filter controls on mobile", "wid": "w6"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": [], "notes": "Chart group controls should remain tappable and not overflow. Use icon or\nshort labels where necessary, with accessible names. Do not let chart chips\nresize the chart area on selection.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Make chart controls wrap into stable segmented controls at small widths", "wid": "w7"}, {"evidence": null, "item_id": "explorer-mobile-density-responsive-tables", "needs": ["w2", "w4", "w5", "w6", "w7"], "notes": "Mechanical assertions, not screenshot-only smokes. For `/results/`,\n`/results/star_schema/?phase=standard`, `/results/query`, and\n`/results/compare?ids=...`, at 390px AND 1440px:\n - Headline + cohort summary + first leaderboard row are within the\n first 800px of scroll on desktop and within the first 1200px on\n mobile (use `boundingBox().y`).\n - At 1440px, MetaLeaderboard renders >= 8 rows above the fold (desktop\n density assertion \u2014 fails if mobile responsive rules accidentally\n truncate the desktop layout).\n - QueryHeatmap container reports `scrollWidth > clientWidth` \u21d2 a\n visible scroll-hint element exists in the DOM.\n - FacetDrawer trigger shows the live result count attribute before the\n drawer is opened.\nScreenshots stay as a secondary regression net.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Playwright responsive assertions (not just screenshots) for DOM order, density, and overflow", "wid": "w8"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Testing and Quality Assurance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-16T00:00:00Z", "deferrals": [], "deps": [], "description": "Non-required nits and improvements surfaced by the code review of\nexplorer-establish-cli-explorer-parity-tests. One Required finding\n(missing vsSlowestRatio fixture coverage) was fixed immediately post-review.\nThis item tracks the remaining nit/consider findings.", "id": "explorer-parity-tests-review-followups", "preserves": [], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Address follow-up findings from parity-tests code review", "verifications": [{"command": "cd results-explorer && npm test -- chartMath.parity", "description": "Parity suite still passes after path fix", "expected": "all tests pass", "item_id": "explorer-parity-tests-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm test", "description": "Full fast test suite green", "expected": "all green", "item_id": "explorer-parity-tests-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-parity-tests-review-followups", "needs": [], "notes": "chartMath.parity.test.ts:43-45 - FIXTURES_DIR resolves via a\nfour-level `../../../../` relative path from __dirname:\n path.resolve(__dirname, \"../../../../tests/parity/fixtures\")\n\nThis silently breaks if either directory moves. Options:\n (a) Use an environment variable (PARITY_FIXTURES_DIR) set in\n vitest.config.ts so the path is defined in one place.\n (b) Use `path.resolve(process.cwd(), \"tests/parity/fixtures\")`\n since vitest runs from the project root.\n (c) Expose the fixtures path via a vite define constant.\n\nOption (b) is simplest and matches how the Makefile already\ninvokes vitest from the repo root. Also improve the loadFixture\nerror message to say \"Run `make parity-fixtures` to regenerate.\"\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Harden FIXTURES_DIR path in chartMath.parity.test.ts", "wid": "w1"}, {"evidence": null, "item_id": "explorer-parity-tests-review-followups", "needs": [], "notes": "docs/visualization-parity.md describes how to ADD new parity tests\nand how to regenerate fixtures, but does not explain what to do\nwhen `make parity-check` fails in CI:\n - Which file changed? (Python or TS)\n - Should I regenerate or revert?\n - How do I run a single failing fixture case locally?\n\nAdd a \"Failure Modes\" section with decision tree:\n 1. If the Python function changed intentionally \u2192 regenerate\n fixtures and update the TS helper to match.\n 2. If the TS function changed intentionally \u2192 regenerate\n fixtures and verify Python is still the source of truth.\n 3. If neither changed (CI flake) \u2192 re-run; if persistent,\n check for float platform differences.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add failure-modes section to docs/visualization-parity.md", "wid": "w2"}, {"evidence": null, "item_id": "explorer-parity-tests-review-followups", "needs": [], "notes": "chartMath.ts - computePercentile(values: number[], p: number)\nreturns undefined when values is empty, but this behavior is not\ndocumented. Callers (computeBoxStats, elsewhere) depend on this\nsilently.\n\nAdd a one-line JSDoc comment:\n /** Returns undefined for empty input - callers must handle null. */\n\nLow risk: behavior already tested by percentile_ladder fixture's\nnull-result cases.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add empty-array contract comment to computePercentile", "wid": "w3"}, {"evidence": null, "item_id": "explorer-parity-tests-review-followups", "needs": [], "notes": "generate_visualization_fixtures.py - builder functions follow\ntwo naming conventions:\n - Most: build__fixture() e.g. build_heatmap_color_fixture\n - Added: build_vs_slowest_ratio_fixture() - consistent\n\nBut the BUILDERS list mixes positional and named patterns with\nno enforced contract. Consider:\n - A dataclass or TypedDict: FixtureBuilder(name, fn) so the\n name used in the file matches the function's chart field.\n - Or a decorator: @fixture(\"vs_slowest_ratio\") def build(): ...\n\nLow urgency; the current list has 13 builders and is readable.\nOnly worth addressing if builders grow past ~20.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Standardize fixture builder naming in generate_visualization_fixtures.py", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "gate it.", "dont": "add noisy production console timing", "why": "public visitors should not see debug instrumentation"}, {"INSTEAD": "optimize queries and rendering instead.", "dont": "optimize by removing methodology/trust data from the first page", "why": "trust context is part of the product"}, {"INSTEAD": "use limits and explicit expansion.", "dont": "render every row by default just because DuckDB can return it", "why": "DOM rendering is the likely bottleneck"}], "approach": "Measure first, then optimize the specific hot paths. Prefer simple memoization,\nquery batching, and bounded rendering before adding complex virtualization.\nUse skeletons to improve perceived performance even when cold-load time cannot\nbe eliminated.\n", "blocked_reason": null, "category": "Performance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": ["explorer-leaderboard-first-identity-unification", "explorer-selection-facets-coverage-at-scale"], "description": "Improve the performance feel of the static DuckDB-WASM explorer as the corpus\ngrows. The audit called out cold-load, query response, loading skeletons, and\npagination behavior as risks. This item focuses on perceived latency and\nruntime cost, not visual redesign: initialize predictably, avoid avoidable\nrepeated scans, keep table rendering bounded, and instrument the expensive\npaths so regressions are visible.", "id": "explorer-performance-perceived-latency", "preserves": ["DuckDB-WASM remains loaded client-side from the static explorer bundle.", "Generated browser fixtures remain deterministic.", "Production UI does not show debug timing unless explicitly enabled.", "Query Workbench retains an intentional unlimited mode for users who request it."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/e2e/performance.spec.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/generate-browser-fixtures.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/verify-browser-fixtures.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/LoadingSpinner.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/LoadingSpinner.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/duckdbQueries.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/queryFilters.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/performanceMarks.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryFilters.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Query.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/BenchmarkIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Compare.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Home.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/PlatformIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Query.test.tsx"}], "state": "done", "title": "Explorer: DuckDB-WASM load performance and perceived-latency hardening", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test && npm run test:e2e:fixtures && npm run build", "description": "Frontend typecheck, tests, fixture generation, and build pass.", "expected": "All commands pass with performance instrumentation and bounded rendering.", "item_id": "explorer-performance-perceived-latency", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium", "description": "Performance smoke passes within budgets defined in w5.", "expected": "Chromium performance smoke passes; P95 budgets in w5 are met or exceeded with margin. Test output reports both P50 and P95 for each mark.", "item_id": "explorer-performance-perceived-latency", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run test:e2e:fixtures", "description": "Fixture generation is deterministic across two consecutive runs.", "expected": "Fixture invariant and determinism checks pass; output includes the documented DuckDB logical digest allow-list.", "item_id": "explorer-performance-perceived-latency", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Add lightweight browser performance marks around DB initialization in\n`results-explorer/src/db.ts`, first Home leaderboard data resolution, and\nfirst rendered table/chart. Log only in development or expose a debug panel\nbehind an explicit flag so production UI stays clean.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Measure DuckDB-WASM cold load and first meaningful leaderboard render", "wid": "w1"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Skeleton TOKENS and the Home/Layout skeleton treatment are owned by\n`explorer-leaderboard-first-identity-unification` w6. This work unit\nonly consumes those tokens to reserve final-geometry skeletons for\nMetaLeaderboard, benchmark matrices, Query Workbench rows, and Compare\nsummary. Do not redefine skeleton primitives here; if a new skeleton\nshape is needed, add it to the upstream identity work and consume it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply identity-owned skeletons to MetaLeaderboard, BenchmarkIndex, Query, and Compare", "wid": "w2"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Query Workbench already has default/unlimited row limit. Audit all table\nsurfaces for unbounded rendering and add explicit limits where corpus\ngrowth can create thousands of rows. Show visible result counts and\n\"show more\" affordances rather than rendering everything by default.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Bound large table rendering with pagination, limits, or virtualization", "wid": "w3"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Review `duckdbQueries.ts`, `queryFilters.ts`, Home, BenchmarkIndex,\nPlatformIndex, and Query for repeated COUNT/DISTINCT queries. Cache stable\nmetadata for the lifetime of the loaded DuckDB snapshot and invalidate\nonly when filters change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch or cache facet/count queries that repeat across pages", "wid": "w4"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": ["w1", "w3", "w4"], "notes": "Provisional budgets, measured against the generated fixture corpus on a\nGitHub Actions Ubuntu runner with the project's standard Chromium build:\n - DuckDB-WASM cold init (`db-init` performance mark): P50 < 4000 ms,\n P95 < 6000 ms\n - First Home leaderboard data resolved (`leaderboard-data-ready`):\n P50 < 1500 ms after `db-init`, P95 < 2500 ms\n - First leaderboard render (`leaderboard-rendered`):\n P50 < 500 ms after `leaderboard-data-ready`, P95 < 1000 ms\n - First Query Workbench paint with default DEFAULT_COLUMNS:\n P50 < 600 ms, P95 < 1200 ms after `db-init`\nRun the smoke 10 times, fail on the P95 threshold, surface the P50 in\nthe test report. Tolerance is built into the gap between P50 and P95;\nno extra slack. If a budget is genuinely wrong the fix is to revise it\nin this file with rationale, not to widen tolerance silently.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Playwright performance smoke with concrete budgets", "wid": "w5"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": ["w5"], "notes": "`npm run test:e2e:fixtures` already generates fixtures. Add a check\nthat re-running fixture generation produces byte-identical output (or\na documented allow-list of timestamp fields). `data/results.duckdb`\nis an allowed binary-container exception only if its logical table\ndigest is identical, so the perf smoke is not measuring regenerated\nfixtures with shifting row counts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add fixture determinism guard", "wid": "w6"}], "worktree": "main"} +{"anti_patterns": [{"dont": "add noisy production console timing", "instead": "gate it.", "why": "public visitors should not see debug instrumentation"}, {"dont": "optimize by removing methodology/trust data from the first page", "instead": "optimize queries and rendering instead.", "why": "trust context is part of the product"}, {"dont": "render every row by default just because DuckDB can return it", "instead": "use limits and explicit expansion.", "why": "DOM rendering is the likely bottleneck"}], "approach": "Measure first, then optimize the specific hot paths. Prefer simple memoization,\nquery batching, and bounded rendering before adding complex virtualization.\nUse skeletons to improve perceived performance even when cold-load time cannot\nbe eliminated.\n", "blocked_reason": null, "category": "Performance", "claimed_at": null, "claimed_by": null, "completed_at": "2026-05-02T00:00:00Z", "completed_pr": null, "created_at": "2026-05-01T00:00:00Z", "deferrals": [], "deps": ["explorer-leaderboard-first-identity-unification", "explorer-selection-facets-coverage-at-scale"], "description": "Improve the performance feel of the static DuckDB-WASM explorer as the corpus\ngrows. The audit called out cold-load, query response, loading skeletons, and\npagination behavior as risks. This item focuses on perceived latency and\nruntime cost, not visual redesign: initialize predictably, avoid avoidable\nrepeated scans, keep table rendering bounded, and instrument the expensive\npaths so regressions are visible.", "id": "explorer-performance-perceived-latency", "preserves": ["DuckDB-WASM remains loaded client-side from the static explorer bundle.", "Generated browser fixtures remain deterministic.", "Production UI does not show debug timing unless explicitly enabled.", "Query Workbench retains an intentional unlimited mode for users who request it."], "prior_art": [], "priority": "medium-high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/e2e/performance.spec.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/generate-browser-fixtures.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/scripts/verify-browser-fixtures.mjs"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/LoadingSpinner.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/MetaLeaderboard.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/LoadingSpinner.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/components/__tests__/MetaLeaderboard.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/db.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/duckdbQueries.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/__tests__/queryFilters.test.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/duckdbQueries.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/performanceMarks.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryFilters.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/BenchmarkIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Compare.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Home.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/PlatformIndex.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Query.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/BenchmarkIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Compare.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Home.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/PlatformIndex.test.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Query.test.tsx"}], "state": "done", "title": "Explorer: DuckDB-WASM load performance and perceived-latency hardening", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test && npm run test:e2e:fixtures && npm run build", "description": "Frontend typecheck, tests, fixture generation, and build pass.", "expected": "All commands pass with performance instrumentation and bounded rendering.", "item_id": "explorer-performance-perceived-latency", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run test:e2e:chromium", "description": "Performance smoke passes within budgets defined in w5.", "expected": "Chromium performance smoke passes; P95 budgets in w5 are met or exceeded with margin. Test output reports both P50 and P95 for each mark.", "item_id": "explorer-performance-perceived-latency", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npm run test:e2e:fixtures", "description": "Fixture generation is deterministic across two consecutive runs.", "expected": "Fixture invariant and determinism checks pass; output includes the documented DuckDB logical digest allow-list.", "item_id": "explorer-performance-perceived-latency", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Add lightweight browser performance marks around DB initialization in\n`results-explorer/src/db.ts`, first Home leaderboard data resolution, and\nfirst rendered table/chart. Log only in development or expose a debug panel\nbehind an explicit flag so production UI stays clean.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Measure DuckDB-WASM cold load and first meaningful leaderboard render", "wid": "w1"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Skeleton TOKENS and the Home/Layout skeleton treatment are owned by\n`explorer-leaderboard-first-identity-unification` w6. This work unit\nonly consumes those tokens to reserve final-geometry skeletons for\nMetaLeaderboard, benchmark matrices, Query Workbench rows, and Compare\nsummary. Do not redefine skeleton primitives here; if a new skeleton\nshape is needed, add it to the upstream identity work and consume it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Apply identity-owned skeletons to MetaLeaderboard, BenchmarkIndex, Query, and Compare", "wid": "w2"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Query Workbench already has default/unlimited row limit. Audit all table\nsurfaces for unbounded rendering and add explicit limits where corpus\ngrowth can create thousands of rows. Show visible result counts and\n\"show more\" affordances rather than rendering everything by default.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Bound large table rendering with pagination, limits, or virtualization", "wid": "w3"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": [], "notes": "Review `duckdbQueries.ts`, `queryFilters.ts`, Home, BenchmarkIndex,\nPlatformIndex, and Query for repeated COUNT/DISTINCT queries. Cache stable\nmetadata for the lifetime of the loaded DuckDB snapshot and invalidate\nonly when filters change.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Batch or cache facet/count queries that repeat across pages", "wid": "w4"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": ["w1", "w3", "w4"], "notes": "Provisional budgets, measured against the generated fixture corpus on a\nGitHub Actions Ubuntu runner with the project's standard Chromium build:\n - DuckDB-WASM cold init (`db-init` performance mark): P50 < 4000 ms,\n P95 < 6000 ms\n - First Home leaderboard data resolved (`leaderboard-data-ready`):\n P50 < 1500 ms after `db-init`, P95 < 2500 ms\n - First leaderboard render (`leaderboard-rendered`):\n P50 < 500 ms after `leaderboard-data-ready`, P95 < 1000 ms\n - First Query Workbench paint with default DEFAULT_COLUMNS:\n P50 < 600 ms, P95 < 1200 ms after `db-init`\nRun the smoke 10 times, fail on the P95 threshold, surface the P50 in\nthe test report. Tolerance is built into the gap between P50 and P95;\nno extra slack. If a budget is genuinely wrong the fix is to revise it\nin this file with rationale, not to widen tolerance silently.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add Playwright performance smoke with concrete budgets", "wid": "w5"}, {"evidence": null, "item_id": "explorer-performance-perceived-latency", "needs": ["w5"], "notes": "`npm run test:e2e:fixtures` already generates fixtures. Add a check\nthat re-running fixture generation produces byte-identical output (or\na documented allow-list of timestamp fields). `data/results.duckdb`\nis an allowed binary-container exception only if its logical table\ndigest is identical, so the perf smoke is not measuring regenerated\nfixtures with shifting row counts.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add fixture determinism guard", "wid": "w6"}], "worktree": "main"} {"anti_patterns": [], "approach": null, "blocked_reason": null, "category": "Code Quality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-19T00:00:00Z", "completed_pr": null, "created_at": "2026-04-18T00:00:00Z", "deferrals": [], "deps": ["explorer-master-leaderboard-home", "explorer-pipeline-meta-leaderboard-widening", "explorer-query-workbench", "explorer-result-detail-median-first", "explorer-unified-chart-panel", "explorer-url-synced-filter-state"], "description": "Non-required nits surfaced during code review of the six explorer-*\nTODOs completed 2026-04-17 (ultraplan phases A-F):\n\n - explorer-result-detail-median-first (Phase A)\n - explorer-master-leaderboard-home (Phase B)\n - explorer-query-workbench (Phase C)\n - explorer-unified-chart-panel (Phase D)\n - explorer-url-synced-filter-state (Phase E)\n - explorer-pipeline-meta-leaderboard-widening (Phase F)\n\nAll typecheck and Vitest tests are green; post-feature fix commits\n(59aa46440, a79af024c) already addressed required issues. These are\ncleanup items only.", "id": "explorer-phase-a-f-review-followups", "preserves": ["All existing Vitest suites continue to pass (229 tests across 23 files as of 2026-04-18)", "Python explorer_pipeline tests continue to pass", "meta_leaderboard.json artifact shape remains backward-compatible for any frontend older than w5"], "prior_art": [], "priority": "low", "scope": [], "state": "done", "title": "Fix nit-level findings from six explorer phase A-F code reviews (2026-04-17 TODOs)", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Explorer typecheck + Vitest", "expected": "all green", "item_id": "explorer-phase-a-f-review-followups", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -q", "description": "Pipeline tests", "expected": "all green", "item_id": "explorer-phase-a-f-review-followups", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "Home.tsx line 98 rounds avg_rank to 2 decimals; MetaLeaderboard.tsx\nline 218 renders via `.toFixed(1)`. Pick one (1 decimal is enough\nfor visual scan) and drop the more precise round at the source.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Align avg_rank decimal precision across Home.tsx and MetaLeaderboard.tsx", "wid": "w1"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "ResultDetail.tsx uses without\nrole=\"button\", tabIndex, or onKeyDown. Add keyboard affordances\nmatching the MetaLeaderboard a11y pattern: role=\"button\",\ntabIndex=0, Enter/Space activation.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Keyboard-enable sortable table headers in ResultDetail.tsx", "wid": "w10"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "`sortedMedians` / `sortedRawQueries` (lines 68-86) sort on every\nrender. Low impact but easy useMemo win.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Memoize sorted arrays in ResultDetail.tsx", "wid": "w11"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "ResultDetail.tsx:282 and BenchmarkIndex/Compare/PlatformIndex\npass `context={{ kind: \"...\", ... }}` as inline literals. Each\nrender produces a new object identity, invalidating the\napplicableCharts/summary/historical/compareRows useMemos inside\nChartPanel. Wrap each site's context in useMemo or let\nChartPanel accept discrete props (kind, detail, summary, ...)\nand build the discriminated union internally.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Memoize ChartPanel context objects at call sites", "wid": "w12"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "chartRegistry.parity.test.ts regex-matches _CHART_SPECS in\nchart_types.py. Any Python syntax change (formatting, comments\ninside the tuple) risks silently breaking the extraction.\nGenerate `tests/parity/fixtures/chart_ids.json` from Python\nduring make parity-fixtures and have the test read that file\ninstead. Related to follow-up nit w11 on\nexplorer-visualization-review-followup-nits.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit chart spec IDs as a JSON fixture for the parity test", "wid": "w13"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "ChartPanel.tsx:419 hardcodes `benchmark === \"tpch\" || benchmark === \"tpcds\"`\nas the Power@Size carve-out when summary.ranking is missing. The\nranking config emitted by the pipeline should always be present\n- if it isn't, error/log rather than guess. Alternatively read\na shared RANKING_METRIC_BY_FAMILY mirror on the frontend.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Broaden SummaryBoxPanel primary-metric fallback", "wid": "w14"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "Call sites like Home.tsx:23-24 pass fresh `[]` as `initial`. The\npopstate useEffect at useUrlState.ts:116 depends on `initial`,\nso it re-subscribes on every render. Behavior is correct\n(cleanup runs) but wasteful. Either hoist the empty-array\nliterals to module-level constants or ignore `initial` in the\neffect dep array (it is semantically a constant after mount).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stabilize useUrlState's initial/value references at call sites", "wid": "w15"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "useUrlState.ts:126 returns a new setValue function each render.\nDownstream memoized subtrees re-render when it is passed as a\nprop. useCallback((v) => ..., [key, initial]) is a minor but\neasy win.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Memoize useUrlState's setValue via useCallback", "wid": "w16"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "Home.tsx buildCohortHref (lines 161-168) only carries `tuning` to\nthe cohort landing page. Passing `trust` and `window` (date) as\nwell would keep the filter slice consistent when navigating to a\ncohort detail - verify BenchmarkIndex reads the same param names.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Preserve trust and date-window filters in MetaLeaderboard cohort hrefs", "wid": "w2"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "MetaLeaderboard.tsx line 304 falls back to `${rank.rank}/${rank.total}`\nwhen renderCellValue returns JSX (\"ranks\" mode). Factor the a11y\nannouncement text out of renderCellValue so describeCell doesn't\nmirror the formatting logic.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Clean up describeCell's JSX-vs-string coupling in MetaLeaderboard.tsx", "wid": "w3"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "pipeline.py _build_meta_leaderboard (line 242) uses\n`len(sorted_rows)` for `total`, which counts platforms whose\nprimary metric is None. A cohort of 3 where 1 platform has no\nmetric currently shows \"1/3\" and \"2/3\" - arguably it should be\n\"1/2\" and \"2/2\". Consider `sum(1 for r in sorted_rows if _get_val(r) is not None)`.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Clarify MetaRank `total` - count only rankable rows, not null-metric rows", "wid": "w4"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "pipeline.py:240-247 duplicates primary_metric/primary_order into\nevery rank cell when it's already on the parent cohort record.\nFrontend reads cohort.primary_order anyway - we can trim the\nper-cell copies to keep meta_leaderboard.json smaller and remove\nthe divergence risk. Requires follow-up on frontend: drop the\noptional fields from MetaRank and update MetaLeaderboard.tsx\ncellShadeMetric to pull from the cohort.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Drop duplicated primary_metric/primary_order from per-cell MetaRank", "wid": "w5"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "Query.tsx:285-295 hardcodes `{count: 0}` for each date-window\noption. Add a dedicated facet query that buckets rows by\n(now \u2212 30d, 90d, 365d) so users see the impact before clicking.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Populate Query workbench date-window facet counts", "wid": "w6"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "Query.tsx:192 uses\n COPY (...) TO '${exportName}' (FORMAT CSV, ...)\nexportName is currently built from Date.now() so injection is\nimpossible today, but any future change that weaves user input\ninto the filename would silently become injectable. Either\nruntime-assert `exportName` matches `/^benchbox-query-export-\\d+\\.csv$/`\nor use DuckDB parameterization if the driver supports it.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Defensively sanitize or parameterize exportName in CSV COPY TO", "wid": "w7"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "Query.tsx:399 uses `Object.keys(sqlRows[0]!)` to derive headers.\nHeterogeneous rows (UNION ALL over schema-drifted views) would\nmiss columns only present in later rows. Compute the union of\nkeys across all rows, or warn the user.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Stabilize Advanced SQL result column derivation", "wid": "w8"}, {"evidence": null, "item_id": "explorer-phase-a-f-review-followups", "needs": [], "notes": "Query.tsx line 232 uses literal backticks in prose - they render\nas visible characters rather than code styling. Replace with\n`results.duckdb` spans.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Render code-marker backticks as inline in Query.tsx copy", "wid": "w9"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "(not specified)", "dont": "add funding to a view as a computed expression", "why": "bare column reference only (G-11)."}, {"INSTEAD": "(not specified)", "dont": "conflate the funding chip with the trust badge", "why": "they are orthogonal (a vendor result can be employer-funded)."}, {"INSTEAD": "(not specified)", "dont": "introduce a chip color outside the approved design tokens (token scan will fail).", "why": "(unstated)"}, {"INSTEAD": "(not specified)", "dont": "re-list the funding enum or its display text in TypeScript", "why": "source it from the read model / a single generated table."}], "approach": "The pipeline + base-table column already exist, so this is purely the read\nsurface: (1) project funding through the view(s) the frontend actually SELECTs\n(result_detail_metrics for the detail page; platform_index_rows only if the\nindex card shows funding) - keep it a bare column reference, and mirror the\nchange in both the builder DDL and the browser-duckdb-schema.sql doc plus the\nEXPECTED_VIEWS contract test; (2) add the funding chip component reading that\ncolumn; (3) add a legend row; (4) e2e. Frontend unit/e2e tests require\n`npm install` in results-explorer/ (not available in every checkout) - run them\nwhere node is available.\n", "blocked_reason": null, "category": "Feature", "claimed_at": null, "claimed_by": null, "completed_at": null, "completed_pr": null, "created_at": "2026-07-06T00:00:00Z", "deferrals": [], "deps": ["corpus-inventory-and-read-model"], "description": "Correction (2026-07-07): the explorer pipeline (`_project/scripts/explorer_pipeline/`)\nand the frontend (`results-explorer/`) ARE present in this checkout (they were\nmaterialized after the plan was written), so this is no longer a hand-off spec \u2014\nit is real in-repo code. The pipeline half is already done and landed in PR\n#1021: ManifestEntry.funding + transformer read, the results.funding column in\nboth DuckDBSnapshotBuilder DDLs, EXPLORER_READ_MODEL_VERSION 1->2, the vendor\nsubtree -> vendor-supplied/public-vendor-reported derivation, and the\nvendor-supplied TrustBadge mapping.\n\nWhat remains is the funding *disclosure surface* the frontend renders. The\nread-model review found that `funding` is only in the base `results` table, NOT\nprojected into the read-model VIEWS the frontend queries\n(`result_detail_metrics`, `platform_index_rows`). So the funding chip cannot\nread it yet. Remaining work: project funding into the relevant view(s), render\na funding chip on cards/detail, add a legend row, and add e2e.\n", "id": "explorer-pipeline-frontend-handoff", "preserves": ["Design-token / stale-theme scans pass \u2014 no ad-hoc chip colors.", "Existing trust-badge rendering (maintainer/community/vendor) is unchanged; the funding chip is ADDITIVE and orthogonal to the trust badge.", "The DuckDB-in-browser read-only attach + bare-column view rule (G-11) is unchanged \u2014 no computation in views.", "test_explorer_build_contract.py + test_duckdb_browser_contract.py stay green."], "prior_art": [{"concept": "_project/scripts/explorer_pipeline/duckdb_builder.py result_detail_metrics + platform_index_rows view DDLs \u2014 extend (add funding projection; the base results.funding column already exists)", "decision": "extend", "path": "_project/scripts/explorer_pipeline/duckdb_builder.py"}, {"concept": "benchbox/core/results/provenance.py FUNDING_SOURCES + display strings \u2014 reference (the chip's label/tooltip text; do not re-list the enum in TS)", "decision": "reuse", "path": "benchbox/core/results/provenance.py"}, {"concept": "docs/development/browser-duckdb-schema.sql result_detail_metrics (~169) + platform_index_rows (~263) views \u2014 extend (mirror the builder view change)", "decision": "extend", "path": "docs/development/browser-duckdb-schema.sql"}, {"concept": "docs/development/results-explorer-brand-ownership.md + results-explorer-token-scan.md \u2014 reference (chip color must use an approved design token; the token scan gate at tests/unit/scripts/test_scan_explorer_tokens.py will fail on ad-hoc colors)", "decision": "reuse", "path": "docs/development/results-explorer-brand-ownership.md"}, {"concept": "results-explorer/src/components/TrustBadge.tsx + StatusBadge.tsx \u2014 reference (done vendor badge; reuse the StatusBadge/chip pattern for the funding chip)", "decision": "reuse", "path": "results-explorer/src/components/TrustBadge.tsx"}, {"concept": "tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py EXPECTED_VIEWS \u2014 extend (add funding to whichever view(s) now project it; the results-table set already asserts funding)", "decision": "extend", "path": "tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py"}], "priority": "medium", "scope": [{"kind": "do_not_modify", "path_glob": "benchbox/ (producer + CLI are items 1-3, done)"}, {"kind": "do_not_modify", "path_glob": "scripts/generate_corpus_inventory.py (item 4, done)"}, {"kind": "do_not_modify", "path_glob": "the results base-table DDL / read_model_version (already at v2; the view work adds no base column)"}, {"kind": "only_modify", "path_glob": "_project/scripts/explorer_pipeline/duckdb_builder.py (view DDLs only)"}, {"kind": "only_modify", "path_glob": "docs/development/browser-duckdb-schema.sql (view definitions only)"}, {"kind": "only_modify", "path_glob": "results-explorer/ (funding chip component, legend, e2e)"}, {"kind": "only_modify", "path_glob": "tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py"}], "state": "active", "title": "Project funding into read-model views + render the funding chip/legend + e2e", "verifications": [{"command": "uv run -- python -m pytest tests/unit/scripts/explorer_pipeline/ tests/unit/scripts/test_explorer_build_contract.py -q", "description": "Pipeline + build-contract + browser-contract tests pass (correct in-repo paths)", "expected": "all pass; read_model_version == 2", "item_id": "explorer-pipeline-frontend-handoff", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py -q", "description": "funding is projected into the detail view", "expected": "EXPECTED_VIEWS asserts funding in result_detail_metrics", "item_id": "explorer-pipeline-frontend-handoff", "last_result": null, "last_run": null, "seq": 2}, {"command": "cd results-explorer && npx vitest run src/components; uv run -- python -m pytest tests/unit/scripts/test_scan_explorer_tokens.py -q", "description": "Frontend chip + token scan (where node is available)", "expected": "chip renders; token scan passes", "item_id": "explorer-pipeline-frontend-handoff", "last_result": null, "last_run": null, "seq": 3}, {"command": "cd results-explorer && npm run test:e2e -- --grep 'funding'", "description": "e2e for funding chip + legend", "expected": "chip + legend render", "item_id": "explorer-pipeline-frontend-handoff", "last_result": null, "last_run": null, "seq": 4}], "work": [{"evidence": null, "item_id": "explorer-pipeline-frontend-handoff", "needs": [], "notes": "DONE in PR #1021. Kept here as the completed foundation the remaining units build on.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pipeline populate + vendor derivation + read_model_version bump (results.funding column, transformer read, vendor subtree label, contract v2).", "wid": "w1"}, {"evidence": null, "item_id": "explorer-pipeline-frontend-handoff", "needs": ["w1"], "notes": "Add funding to the view DDL in duckdb_builder.py AND browser-duckdb-schema.sql (bare-column projection, G-11 rule), and add funding to the matching EXPECTED_VIEWS set in test_duckdb_browser_contract.py. No new base-table column, so no read_model_version bump.\nDONE in PR #1080: r.funding added to result_detail_metrics in duckdb_builder.py + browser-duckdb-schema.sql; EXPECTED_VIEWS guards it.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Project results.funding into the read-model view(s) the frontend queries (result_detail_metrics, and platform_index_rows if the index shows it).", "wid": "w2"}, {"evidence": null, "item_id": "explorer-pipeline-frontend-handoff", "needs": ["w2"], "notes": "DONE: funding threaded through duckdbQueries.ts (RESULT_DETAIL_METRICS_COLUMNS + ResultDetailMetricsRow + the DetailResult mapping) and types.ts; new FundingChip.tsx renders via the StatusBadge primitive with a new StatusRole \"funding\" and a uniform `neutral` tone (a tone gradient would re-encode funding as a trust signal); 'unspecified' renders NO chip; formatFunding() added to displayLabels.ts. New ProvenanceLegend.tsx satisfies the hosted-results-contract legend rule and pulls its prose from trustLabelDescription()/fundingDescription() so it cannot drift from the badges. SCOPE: detail page only - see w5 for the card-level chip.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Render a funding chip on the detail page (ResultDetail.tsx), reading funding from result_detail_metrics; add a legend row explaining chip + vendor badge.", "wid": "w3"}, {"evidence": null, "item_id": "explorer-pipeline-frontend-handoff", "needs": ["w3"], "notes": "DONE: results-explorer/e2e/routes/funding-disclosure.spec.ts covers chip render, chip omission on 'unspecified', neutral tone, trust/funding orthogonality, and the legend (reachable + expands to explain both axes). The browser fixture corpus gained its first funding-disclosing bundle: the `community` variant in scripts/generate-browser-fixtures.mjs now declares provenance.funding=employer, so community-submission + employer-funded proves the two axes are independent. Unit coverage in FundingChip.test.tsx, ProvenanceLegend.test.tsx, ResultDetail.test.tsx.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add results-explorer e2e (results-explorer/e2e/) for the funding chip + legend; update any legend-set smoke. Needs a node env.", "wid": "w4"}, {"evidence": null, "item_id": "explorer-pipeline-frontend-handoff", "needs": ["w4"], "notes": "Done. Two corrections to the plan as written, both found during implementation. (1) The cards do NOT all read the views this note named: PlatformIndex reads `bench.results` directly via loadPlatformIndexRows, and the frontend PlatformRow is mapped from benchmark_rankings - so the real changes were a bare `funding` column in the platform_index_rows view (kept so the view still mirrors its base table), a new `funding` column on the benchmark_rankings TABLE (not a view; populated from ManifestEntry.funding, since PlatformRow is serialized into the BenchmarkSummary artifact and widening that is a separate contract change), and `r.funding` added to the SELECT lists in duckdbQueries.ts. benchmark_matrix_cells was left alone - it is per-query heatmap data and carries no badges. (2) read_model_version WAS bumped, 2 -> 3, contrary to this note. A v2 snapshot has results.funding but not the projections the cards read, and the version guard is exact-match, so without the bump a stale snapshot passed the guard and then failed deep inside a card query as a Binder Error. The guard tracks the shape of every view the UI reads, not just the shape of `results`. Bumped in contract.py, db.ts, and explorer-build-contract.mjs (all three are pinned together by db-remediation-pin.test.ts). Chip added to PlatformIndex, BenchmarkIndex, MetaLeaderboard, Compare, and QueryHeatmap; ProvenanceLegend now renders on ResultDetail, BenchmarkIndex, PlatformIndex, Compare, and Home.", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Project funding into the card-level views and render the chip on the index/leaderboard/compare surfaces.", "wid": "w5"}], "worktree": "results-labels-funding"} -{"anti_patterns": [{"INSTEAD": "read it", "dont": "hardcode latency-vs-throughput logic", "why": "RankingConfig.primary_order already encodes the direction"}, {"INSTEAD": "fold speedup into that loop", "dont": "recompute best per cohort in a second pass", "why": "the cohort iteration at pipeline.py:217 already has all values"}, {"INSTEAD": "only add fields", "dont": "remove {rank, total}", "why": "existing frontends depend on them"}], "approach": "w1/w2 form a pair: compute new fields in the pipeline, then widen\nthe Pydantic model to surface them. w3 is independent (sidecar\nemission in duckdb_builder.py) so it can land in parallel. w4\ngates both with unit tests; regenerate any snapshot fixtures under\ntests/ that encode the narrower MetaRank shape.\n\nFor speedup math, follow the plan's direction convention:\n - latency (asc): speedup = best_metric / cell_metric \u2192 > 1 means slower\n - throughput/score (desc): speedup = cell_metric / best_metric \u2192 < 1 means worse\nBoth forms make the cohort best = 1.0, which is what the UI renders.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-17T00:00:00Z", "completed_pr": null, "created_at": "2026-04-17T00:00:00Z", "deferrals": [], "deps": [], "description": "Phase F of the Results Explorer strategy revision\n(plans/purring-jingling-petal-ultraplan).\n\nThe master leaderboard (Phase B) and the query workbench (Phase C)\nboth need pipeline-side surface additions before the UI work can\nland.\n\n1) `_build_meta_leaderboard` in\n `benchbox/core/explorer_pipeline/pipeline.py` already computes\n `metric_value` per platform per cohort (line 217) but drops it\n before the meta artifact is flattened (line 223, which keeps only\n `{rank, total}`). The Home master leaderboard needs the absolute\n value, the within-cohort speedup, and the primary metric/order so\n it can format cells in the three render modes (Times / Ranks /\n Speedup) without a second cohort lookup.\n\n2) The query workbench's column picker shouldn't hardcode the\n column list. `DuckDBSnapshotBuilder._COLUMNS` is already\n authoritative - emit it as `results_schema.json` next to\n `results.duckdb` so the frontend can consume it directly.", "id": "explorer-pipeline-meta-leaderboard-widening", "preserves": ["DuckDB snapshot output (results.duckdb) byte-for-byte identical - this item only adds a sidecar", "Existing frontend consumers that read only {rank, total} from MetaRank continue to work - extra keys are additive", "primary_metric / primary_order values match get_ranking_config() for every registered benchmark family"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/duckdb_builder.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/models.py"}, {"kind": "only_modify", "path_glob": "benchbox/core/explorer_pipeline/pipeline.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/test_duckdb_builder.py"}, {"kind": "only_modify", "path_glob": "tests/unit/core/explorer_pipeline/test_meta_leaderboard.py"}], "state": "done", "title": "Explorer pipeline: widen MetaRank cells and emit results_schema.json", "verifications": [{"command": "uv run -- python -m pytest tests/unit/core/explorer_pipeline/ -v", "description": "Explorer pipeline unit tests green", "expected": "all pass, including new MetaRank and results_schema tests", "item_id": "explorer-pipeline-meta-leaderboard-widening", "last_result": null, "last_run": null, "seq": 1}, {"command": "uv run -- python -m pytest -m fast -q", "description": "Fast suite green (no regressions elsewhere)", "expected": "all green", "item_id": "explorer-pipeline-meta-leaderboard-widening", "last_result": null, "last_run": null, "seq": 2}, {"command": "uv run benchbox explorer build --data-dir results-data/ --output /tmp/explorer-out", "description": "End-to-end pipeline run emits widened artifacts", "expected": "/tmp/explorer-out/meta_leaderboard.json cells include metric_value/speedup_vs_best/primary_metric/primary_order; /tmp/explorer-out/results_schema.json exists", "item_id": "explorer-pipeline-meta-leaderboard-widening", "last_result": null, "last_run": null, "seq": 3}], "work": [{"evidence": null, "item_id": "explorer-pipeline-meta-leaderboard-widening", "needs": [], "notes": "benchbox/core/explorer_pipeline/pipeline.py:_build_meta_leaderboard\ncurrently flattens each cell to {rank, total} at line 223. Keep\nrank/total and add:\n - metric_value: float | None (already computed at line 217)\n - speedup_vs_best: float | None\n * for latency-style metrics (primary_order == \"asc\"):\n best / value\n * for throughput/score metrics (primary_order == \"desc\"):\n value / best\n best = min or max of non-null metric_values in the cohort\n - primary_metric: str (e.g. \"geomean_ms\", \"power_score\")\n - primary_order: \"asc\" | \"desc\"\nPull primary_metric/primary_order from get_ranking_config(benchmark)\n- pipeline.py line 142 already calls this for the summary build,\nreuse the result.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Widen MetaRank cell shape to {rank, total, metric_value, speedup_vs_best, primary_metric, primary_order}", "wid": "w1"}, {"evidence": null, "item_id": "explorer-pipeline-meta-leaderboard-widening", "needs": ["w1"], "notes": "Widen MetaRank to include the new fields. Mark them Optional to\nremain backward-compatible with any cached artifacts (current\nconsumers only read {rank, total}, so extra keys are safe).\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Update MetaRank Pydantic model in explorer_pipeline/models.py", "wid": "w2"}, {"evidence": null, "item_id": "explorer-pipeline-meta-leaderboard-widening", "needs": [], "notes": "benchbox/core/explorer_pipeline/duckdb_builder.py - after the\nDuckDB file is written, serialize _COLUMNS (name + type) as\nresults_schema.json in the same output directory. No schema\nchange; just persisting the already-authoritative column list\nso the frontend's column picker (Phase C) doesn't have to\nhardcode it.\n\nShape:\n {\"columns\": [{\"name\": \"...\", \"type\": \"...\"}, ...]}\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Emit results_schema.json next to results.duckdb", "wid": "w3"}, {"evidence": null, "item_id": "explorer-pipeline-meta-leaderboard-widening", "needs": ["w2", "w3"], "notes": "- tests/unit/core/explorer_pipeline/test_meta_leaderboard.py (new):\n assert each cell has metric_value, speedup_vs_best,\n primary_metric, primary_order; check speedup math for a\n latency cohort (best = min) and a throughput cohort\n (best = max)\n- tests/unit/core/explorer_pipeline/test_duckdb_builder.py\n (extended): assert results_schema.json exists next to\n results.duckdb and lists the expected columns\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Pipeline unit tests for widened MetaRank and results_schema.json emission", "wid": "w4"}], "worktree": "main"} -{"anti_patterns": [{"INSTEAD": "fetch it at load", "dont": "duplicate _COLUMNS on the frontend", "why": "results_schema.json is emitted by the pipeline specifically to avoid this"}, {"INSTEAD": "plain textarea is sufficient", "dont": "introduce CodeMirror or a SQL syntax highlighter", "why": "scope says 'parity with benchbox results --sql, no new deps'"}, {"INSTEAD": "register it first", "dont": "register the /results/query route AFTER /results/:benchmark", "why": "the wildcard will capture it"}, {"INSTEAD": "parameterise every user value", "dont": "string-concat filter values into SQL", "why": "DuckDB-WASM supports parameters"}], "approach": "Land as a single consumer of the already-scaffolded db.ts - no\nrefactor of db.ts itself. Note the out-of-date docstring reference\nto \"bench.manifest\" in db.ts; the actual attached name from\nduckdb_builder.py is `bench.results` (benign to leave for now, but\nworth a drive-by comment fix). Facet counts via a single GROUP BY\nprobe per load keeps the cold-start snappy - do not push counts to\nserver-side aggregation. All user input in the SQL path is\nuntrusted: use parameterised queries for the filter\u2192SQL path, and\nrely on READ_ONLY for the textarea path.\n", "blocked_reason": null, "category": "Core Functionality", "claimed_at": null, "claimed_by": null, "completed_at": "2026-04-17T00:00:00Z", "completed_pr": null, "created_at": "2026-04-17T00:00:00Z", "deferrals": [], "deps": ["explorer-pipeline-meta-leaderboard-widening"], "description": "Phase C of the Results Explorer strategy revision\n(plans/purring-jingling-petal-ultraplan).\n\nAdd a new `/results/query` page: faceted filter panel + flat sortable\ntable + optional SQL textarea, backed by `results.duckdb` via\nDuckDB-WASM. Models the CloudSpecs (cloudspecs.fyi) pattern of very\nfast filter + pivot over a canonical result set.\n\nThe pipeline already emits `results.duckdb`, and `src/db.ts` already\ninitialises DuckDB-WASM in the frontend - no consumer exists yet.\nThis item is the consumer.\n\nDepends on explorer-pipeline-meta-leaderboard-widening for the\n`results_schema.json` sidecar so the column picker doesn't hardcode\nthe list.", "id": "explorer-query-workbench", "preserves": ["Existing routes remain reachable - /results/query must be registered before /results/:benchmark so it isn't shadowed", "queryRows() helper at db.ts:78 continues to be the sole query entry point from this page - no parallel DuckDB client", "src/db.ts initialises DuckDB-WASM with ATTACH READ_ONLY - do not weaken this; SQL textarea relies on it for safety"], "prior_art": [], "priority": "high", "scope": [{"kind": "only_modify", "path_glob": "results-explorer/src/App.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/lib/queryFilters.ts"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/Query.tsx"}, {"kind": "only_modify", "path_glob": "results-explorer/src/pages/__tests__/Query.test.tsx"}], "state": "done", "title": "Explorer: CloudSpecs-style /results/query workbench over results.duckdb", "verifications": [{"command": "cd results-explorer && npm run typecheck && npm test", "description": "Explorer typecheck + Vitest green (including Query.test.tsx)", "expected": "all green", "item_id": "explorer-query-workbench", "last_result": null, "last_run": null, "seq": 1}, {"command": "cd results-explorer && npm run dev", "description": "End-to-end smoke", "expected": "/results/query loads; applying platform=DuckDB filter narrows rows; sort by geomean_ms works; CSV export row count matches corpus-inventory filter count; SQL textarea runs SELECTs; DDL/DML errors out", "item_id": "explorer-query-workbench", "last_result": null, "last_run": null, "seq": 2}], "work": [{"evidence": null, "item_id": "explorer-query-workbench", "needs": [], "notes": "results-explorer/src/App.tsx - register\n`` BEFORE the benchmark-wildcard\nroute so it isn't captured by /results/:benchmark.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Add /results/query route in App.tsx", "wid": "w1"}, {"evidence": null, "item_id": "explorer-query-workbench", "needs": [], "notes": "New file: results-explorer/src/lib/queryFilters.ts. Given a\nFilterState object, produce a safe SQL string (or prepared\nstatement args) against `bench.results`. All user-provided\nvalues parameterised - never string-concatenated. This module\nis also what the SQL textarea's \"build from current facets\"\nbutton emits.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Build queryFilters.ts: filter state \u2192 parameterised DuckDB SQL", "wid": "w2"}, {"evidence": null, "item_id": "explorer-query-workbench", "needs": ["w1", "w2"], "notes": "Left sticky panel. Facets: benchmark, platform, SF, tuning\nmode, trust tier, date range, has_cost, validation_status.\nOn load, run one DuckDB GROUP BY probe per facet (or a single\nmulti-column CUBE) and display counts next to each option.\nEach facet is a checkbox group backed by useUrlState so the\nfull filter is shareable.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Facet panel with counts (single GROUP BY probe on load)", "wid": "w3"}, {"evidence": null, "item_id": "explorer-query-workbench", "needs": ["w2", "w3"], "notes": "Center column. One row per bundle from bench.results. Column\npicker reads results_schema.json (emitted by the pipeline -\nsee explorer-pipeline-meta-leaderboard-widening w3) so the\nfrontend never hardcodes _COLUMNS. Default visible columns:\nbenchmark, platform, SF, date, power_score, geomean_ms,\ntrust. Sort by any column, ascending/descending.\nRow \u2192 /results/r/{result_id}.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Flat sortable results table + column picker (reads results_schema.json)", "wid": "w4"}, {"evidence": null, "item_id": "explorer-query-workbench", "needs": ["w4"], "notes": "Button: \"Download CSV\". Exports the currently-filtered set\n(not the full table). Uses the DuckDB-WASM Blob path already\nscaffolded in db.ts - no new deps.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "CSV export via DuckDBDataProtocol.BROWSER_FILEREADER \u2192 Blob", "wid": "w5"}, {"evidence": null, "item_id": "explorer-query-workbench", "needs": ["w4"], "notes": "Collapsed
under the table. Plain textarea (no\nCodeMirror - parity with `benchbox results --sql`, no new\ndeps). Runs via `queryRows()` (db.ts line 78). Read-only\nguarantee comes from DuckDB's existing\n`ATTACH ... (READ_ONLY)` at db.ts:57.\n", "started_at": null, "started_branch": null, "started_worktree": null, "status": "done", "summary": "Advanced mode: plain