fix(validation): exclude TPC-H Q11/16/18/20 from EXACT row-count checks under non-reference seeds

_project/config/fast_test_lane_policy.json

@@ -1,7 +1,7 @@
 {
   "enabled": true,
-  "max_fast_tests": 25190,
-  "_ceiling_note": "Bumped 13200 -> 22000 on 2026-04-13 to reflect grown fast-lane (20,184 collected). Bumped 22000 -> 22100 on 2026-05-10 for joinorder-canonical-foundation Phase-1 (data_fetch + scale-factor + surface-field tests; 22038 collected). Bumped 22100 -> 22400 on 2026-05-11 for joinorder-canonical-cutover canonical SQL/DataFrame, MCP, publishing, and corpus-inventory guards; 22317 collected. Bumped 22400 -> 22500 on 2026-05-13 for UAT enabled-platform remediation coverage; pr-preflight collected 22419. Bumped 22500 -> 22510 on 2026-05-15 after rebasing onto develop at 36ab01058; pr-preflight collected 22501 before this branch's new public theme contract tests, which are medium-marked to avoid fast-lane growth. Bumped 22510 -> 22530 on 2026-05-14 for pr-review-followup batch coverage (joinorder dataframe family, pg_mooncake adapter, landing quickstart validation, explorer read-model retry); pr-preflight collected 22520. Bumped 22530 -> 22540 on 2026-05-15 for results-explorer-post-theme-reconcile count-aware Home/script coverage; CI collected 22537. Bumped 22540 -> 22543 on 2026-05-16 for pr-review-followups coverage on UAT CLI dispatch, UAT release-gate envelope scoping, and develop-post-merge orphan ordering; ci-lint collected 22543. Bumped 22543 -> 22550 on 2026-05-16 for prompts landing Phase 1 CLI hygiene validator/prompt-shape coverage; pr-preflight collected 22550. Bumped 22550 -> 22553 on 2026-05-16 for prompts landing Phase 2 cost-class and registry prompt-safety coverage; timing policy collected 22553. Bumped 22553 -> 22558 on 2026-05-16 for prompts landing Phase 3 platform-option and TPC-DS dsdgen validator coverage; timing_policy_check collected 22558. Bumped 22558 -> 22561 on 2026-05-16 for prompts landing Phase 4 provenance template and capture-plan footer coverage; timing_policy_check collected 22561. Bumped 22561 -> 22563 on 2026-05-16 for prompts landing review fixes covering MCP cost-gating parity and compare summary discipline; targeted lane count increased by two fast tests. Bumped 22563 -> 22568 on 2026-05-16 for prompts landing Phase 5 MCP parity prompt-surface coverage; timing_policy_check collected 22568. Bumped 22568 -> 22570 on 2026-05-16 for prompts landing MCP real-tool parity and platform-option gap follow-up coverage; timing_policy_check collected 22570. Bumped 22570 -> 22572 on 2026-05-19 for pr-review-followups prompt regressions covering credential-before-smoke ordering and smoke-scale dsdgen warning behavior; timing_policy_check collected 22572. Set 22572 -> 25000 on 2026-05-19 by maintainer direction during pr-review-followups cleanup. Ratchet down if the lane contracts; never raise without justification. Bumped 25000 -> 25050 on 2026-07-16 for tuning-mode-vocabulary-and-facet-implementation-20260712 (ADR-2 canonical_mode/tuned-fallback/official-refusal coverage, the PACKAGED_RESOURCE composition pin added when merging develop's #1188 packaged-template tier, and the physical_mechanisms unknown-vs-empty ingest-pipeline regression tests); develop alone collected 24995, this branch's merge collected 25022. Bumped 25050 -> 25060 on 2026-07-17 for tuning-capability-registry-coverage-20260716 (alias-resolution invariant tests and derived generator-coverage drift guards for the 9 newly-registered platforms); CI collected 25052 on the merge ref. Bumped 25060 -> 25080 on 2026-07-17 composing #1198's registry-coverage guards with #1176's provenance/hash export coverage (test_tuning_provenance_export.py + test_requested_config_hash.py); both branches independently bumped from 25050 (#1198: 25052 collected; #1176 pre-compose: 25062 collected); composed merge tree collects 25077. Bumped 25080 -> 25140 on 2026-07-17 for tuning-from-config-forwarding-sweep-20260716 (test_tuning_config_forwarding.py's registry-driven parametrized test, one case per registered platform adapter, verifying from_config forwards tuning_enabled/tuning_config/unified_tuning_configuration/tuning_source/tuning_source_file); timing_policy_check collected 25128. Bumped 25140 -> 25180 on 2026-07-18 for the todo-db-tracker local-SQLite spike (tests/unit/scripts/test_todo_db.py: 49 fast unit tests covering the enforced lifecycle invariants, archive import, and concurrency guards); develop collected 25128, this branch collects 25177. Bumped 25180 -> 25190 on 2026-07-18 composing the todo-db spike branch with develop's post-25128 growth: the branch alone collects 25177 (49 fast tracker tests; the 11 wrapper tests are medium-marked and collect 0 under -m fast), but the PR merge ref collects 25187.",
+  "max_fast_tests": 25220,
+  "_ceiling_note": "Bumped 13200 -> 22000 on 2026-04-13 to reflect grown fast-lane (20,184 collected). Bumped 22000 -> 22100 on 2026-05-10 for joinorder-canonical-foundation Phase-1 (data_fetch + scale-factor + surface-field tests; 22038 collected). Bumped 22100 -> 22400 on 2026-05-11 for joinorder-canonical-cutover canonical SQL/DataFrame, MCP, publishing, and corpus-inventory guards; 22317 collected. Bumped 22400 -> 22500 on 2026-05-13 for UAT enabled-platform remediation coverage; pr-preflight collected 22419. Bumped 22500 -> 22510 on 2026-05-15 after rebasing onto develop at 36ab01058; pr-preflight collected 22501 before this branch's new public theme contract tests, which are medium-marked to avoid fast-lane growth. Bumped 22510 -> 22530 on 2026-05-14 for pr-review-followup batch coverage (joinorder dataframe family, pg_mooncake adapter, landing quickstart validation, explorer read-model retry); pr-preflight collected 22520. Bumped 22530 -> 22540 on 2026-05-15 for results-explorer-post-theme-reconcile count-aware Home/script coverage; CI collected 22537. Bumped 22540 -> 22543 on 2026-05-16 for pr-review-followups coverage on UAT CLI dispatch, UAT release-gate envelope scoping, and develop-post-merge orphan ordering; ci-lint collected 22543. Bumped 22543 -> 22550 on 2026-05-16 for prompts landing Phase 1 CLI hygiene validator/prompt-shape coverage; pr-preflight collected 22550. Bumped 22550 -> 22553 on 2026-05-16 for prompts landing Phase 2 cost-class and registry prompt-safety coverage; timing policy collected 22553. Bumped 22553 -> 22558 on 2026-05-16 for prompts landing Phase 3 platform-option and TPC-DS dsdgen validator coverage; timing_policy_check collected 22558. Bumped 22558 -> 22561 on 2026-05-16 for prompts landing Phase 4 provenance template and capture-plan footer coverage; timing_policy_check collected 22561. Bumped 22561 -> 22563 on 2026-05-16 for prompts landing review fixes covering MCP cost-gating parity and compare summary discipline; targeted lane count increased by two fast tests. Bumped 22563 -> 22568 on 2026-05-16 for prompts landing Phase 5 MCP parity prompt-surface coverage; timing_policy_check collected 22568. Bumped 22568 -> 22570 on 2026-05-16 for prompts landing MCP real-tool parity and platform-option gap follow-up coverage; timing_policy_check collected 22570. Bumped 22570 -> 22572 on 2026-05-19 for pr-review-followups prompt regressions covering credential-before-smoke ordering and smoke-scale dsdgen warning behavior; timing_policy_check collected 22572. Set 22572 -> 25000 on 2026-05-19 by maintainer direction during pr-review-followups cleanup. Ratchet down if the lane contracts; never raise without justification. Bumped 25000 -> 25050 on 2026-07-16 for tuning-mode-vocabulary-and-facet-implementation-20260712 (ADR-2 canonical_mode/tuned-fallback/official-refusal coverage, the PACKAGED_RESOURCE composition pin added when merging develop's #1188 packaged-template tier, and the physical_mechanisms unknown-vs-empty ingest-pipeline regression tests); develop alone collected 24995, this branch's merge collected 25022. Bumped 25050 -> 25060 on 2026-07-17 for tuning-capability-registry-coverage-20260716 (alias-resolution invariant tests and derived generator-coverage drift guards for the 9 newly-registered platforms); CI collected 25052 on the merge ref. Bumped 25060 -> 25080 on 2026-07-17 composing #1198's registry-coverage guards with #1176's provenance/hash export coverage (test_tuning_provenance_export.py + test_requested_config_hash.py); both branches independently bumped from 25050 (#1198: 25052 collected; #1176 pre-compose: 25062 collected); composed merge tree collects 25077. Bumped 25080 -> 25140 on 2026-07-17 for tuning-from-config-forwarding-sweep-20260716 (test_tuning_config_forwarding.py's registry-driven parametrized test, one case per registered platform adapter, verifying from_config forwards tuning_enabled/tuning_config/unified_tuning_configuration/tuning_source/tuning_source_file); timing_policy_check collected 25128. Bumped 25140 -> 25180 on 2026-07-18 for the todo-db-tracker local-SQLite spike (tests/unit/scripts/test_todo_db.py: 49 fast unit tests covering the enforced lifecycle invariants, archive import, and concurrency guards); develop collected 25128, this branch collects 25177. Bumped 25180 -> 25190 on 2026-07-18 composing the todo-db spike branch with develop's post-25128 growth: the branch alone collects 25177 (49 fast tracker tests; the 11 wrapper tests are medium-marked and collect 0 under -m fast), but the PR merge ref collects 25187. Bumped 25190 -> 25220 on 2026-07-18 for the post-CODEOWNERS-retirement merge queue, composed once so the queued branches carry byte-identical policy content instead of five conflicting bumps (#1142 seed-validation exclusion coverage +10; #1116 bot-finding sweep regressions +4; #1202 capability-registry/report regressions ~+8; #1206 object-schema loader regressions ~+5; #1114 branch-rename test moves +0): develop alone collects 25187; the queue's final merge ref is expected to collect ~25214.",
   "forbidden_marker_expressions": [
     "resource_heavy",
     "stress",

benchbox/core/expected_results/tpch_results.py

@@ -77,6 +77,26 @@
 
 logger = logging.getLogger(__name__)
 
+# Queries whose result cardinality is sensitive to the specific substitution
+# parameters used (not just the scale factor) -- TPC-H's "answer-set boundary"
+# queries. Only the pinned reference-seed answer set is guaranteed to match;
+# a different seed produces valid-but-different query parameters that
+# legitimately shift these queries' row counts, so an EXACT-mode mismatch on
+# a non-reference-seed run is not a correctness regression.
+#
+# The bounded correctness gate (Makefile CORRECTNESS_GATE_QUERY_IDS, see
+# docs/operations/release-guide.md "Q11/Q16/Q18/Q20 ... excluded for
+# answer-set boundary sensitivity") already excludes this same set for the
+# same reason. This constant is the production-code mirror that
+# benchbox.core.validation.query_validation.QueryValidator consults so
+# non-reference-seed runs (custom --seed, throughput streams whose derived
+# seed is base_seed + stream_id*1000 + position) report an exclusion instead
+# of a false EXACT-mode row-count failure. Not the same list object as the
+# Makefile's (that one is test-only env-var plumbing feeding a single
+# integration test, not importable production code) -- kept in sync by
+# convention; a change to either should be reviewed against the other.
+PARAMETER_SENSITIVE_QUERY_IDS: frozenset[str] = frozenset({"11", "16", "18", "20"})
+
 
 def get_tpch_expected_results(scale_factor: float = 1.0) -> BenchmarkExpectedResults | None:
     """Get expected results for TPC-H queries at a given scale factor.