diff --git a/.github/workflows/nim-benchmark.yml b/.github/workflows/nim-benchmark.yml index 48e279548..243dbfbbc 100644 --- a/.github/workflows/nim-benchmark.yml +++ b/.github/workflows/nim-benchmark.yml @@ -72,6 +72,9 @@ jobs: --task-manifest examples/nim_task_manifest.json \ --output-dir benchmark_artifacts \ --max-total-requests "$MAX_REQUESTS" \ + --bootstrap-resample-count 2000 \ + --confidence-level 0.95 \ + --comparison-pair conduct_bounded,route_once \ --max-workflow-depth 5 \ --max-output-tokens 264 \ --git-sha "$PROVENANCE_GIT_SHA" \ @@ -140,6 +143,9 @@ jobs: --task-manifest examples/nim_task_manifest.json \ --output-dir benchmark_artifacts \ --max-total-requests "$MAX_REQUESTS" \ + --bootstrap-resample-count 2000 \ + --confidence-level 0.95 \ + --comparison-pair conduct_bounded,route_once \ --max-workflow-depth 5 \ --max-output-tokens 264 \ --git-sha "$PROVENANCE_GIT_SHA" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index f47d10f54..911455f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,15 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Changed +- The psychometric held-out harness now requires a declared resample count, + percentile coverage, and seed for paired intervals. Hidden 2,000-sample + 95% defaults are removed. The script entry still writes 2,000, 0.95, and + seed 568 as this run's choices. +- NIM paired comparisons now require a declared resample count, percentile + coverage, and policy-pair list. Hidden 2,000-resample 95% defaults and the + baked-in conduct/route/cheapest/hindsight subset are removed. Report schema + 4.0.0 records those declarations; older reports must be regenerated. + - NIM benchmark workflow depth and per-call output-token budgets are required declarations (`max_workflow_depth` / `max_output_tokens`); omitted values fail closed instead of inventing five steps or 264 tokens (ADR 0043). diff --git a/contextual_orchestrator/nim_benchmark.py b/contextual_orchestrator/nim_benchmark.py index cd7cc7f4c..92372b455 100644 --- a/contextual_orchestrator/nim_benchmark.py +++ b/contextual_orchestrator/nim_benchmark.py @@ -85,7 +85,7 @@ def estimate_tokens(text: str) -> int: return (len(text) + 3) // 4 if text else 0 -BENCHMARK_SCHEMA_VERSION = "3.0.0" +BENCHMARK_SCHEMA_VERSION = "4.0.0" NIM_DEFAULT_ENDPOINT = "https://integrate.api.nvidia.com/v1" NIM_CREDENTIAL_NAME = "NVIDIA_NIM_API_KEY" DRY_RUN_PROVENANCE_PLACEHOLDER = "dry_run" @@ -2391,32 +2391,125 @@ def _require_declared_positive_int(value: object, field_name: str) -> int: # -------------------------------------------------------------------------- +def _require_declared_positive_int(value: object, field_name: str) -> int: + """Reject missing, boolean, or non-positive integer declarations.""" + if type(value) is not int or value < 1: + raise BenchmarkContractError(f"{field_name} must be a declared positive integer") + return value + + +def _require_declared_confidence_level(value: object) -> float: + """Reject missing or non-exclusive-unit-interval coverage declarations.""" + if type(value) is not float or not math.isfinite(value) or not 0.0 < value < 1.0: + raise BenchmarkContractError( + "confidence_level must be a declared finite exclusive unit interval" + ) + return value + + +def _require_declared_seed(value: object) -> int: + """Reject missing or boolean bootstrap seeds.""" + if type(value) is not int: + raise BenchmarkContractError("seed must be a declared integer") + return value + + +def _require_declared_comparison_pairs( + value: object, +) -> tuple[tuple[str, str], ...]: + """Reject missing, empty, malformed, or duplicate policy-pair declarations.""" + if isinstance(value, (str, bytes)) or not isinstance(value, (list, tuple)): + raise BenchmarkContractError( + "comparison_pairs must declare a sequence of policy pairs" + ) + if not value: + raise BenchmarkContractError( + "comparison_pairs must declare at least one policy pair" + ) + normalized: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for pair in value: + if not isinstance(pair, (list, tuple)) or len(pair) != 2: + raise BenchmarkContractError( + "each comparison pair must contain two policy names" + ) + policy_a, policy_b = pair + if ( + not isinstance(policy_a, str) + or not isinstance(policy_b, str) + or not policy_a + or not policy_b + or policy_a == policy_b + ): + raise BenchmarkContractError( + "comparison pair policies must be distinct nonempty names" + ) + key = (policy_a, policy_b) + if key in seen: + raise BenchmarkContractError("duplicate comparison pair") + seen.add(key) + normalized.append(key) + return tuple(normalized) + + +def _comparison_pairs_from_cli(values: list[str] | None) -> tuple[tuple[str, str], ...]: + """Parse repeated ``policy_a,policy_b`` flags into declared comparison pairs.""" + if not values: + raise BenchmarkContractError( + "comparison_pairs must declare at least one policy pair" + ) + parsed: list[tuple[str, str]] = [] + for raw in values: + parts = raw.split(",") if isinstance(raw, str) else () + if len(parts) != 2: + raise BenchmarkContractError( + "each comparison pair must contain two policy names" + ) + parsed.append((parts[0], parts[1])) + return _require_declared_comparison_pairs(parsed) + + def paired_bootstrap_mean_difference( paired_scores: list[tuple[float, float]], - iterations: int = 2000, - seed: int = 7, + *, + resample_count: int | None = None, + confidence_level: float | None = None, + seed: int | None = None, ) -> dict[str, Any]: - """Paired bootstrap CI for mean(score_a - score_b) over shared tasks.""" + """Paired percentile interval for mean(score_a - score_b) on shared tasks. + + ``resample_count``, ``confidence_level``, and ``seed`` are required + declarations. ``None`` is a fail-closed sentinel, not a statistical default. + """ if not paired_scores: raise BenchmarkContractError( "paired bootstrap requires at least one score pair" ) + iterations = _require_declared_positive_int(resample_count, "resample_count") + coverage = _require_declared_confidence_level(confidence_level) + declared_seed = _require_declared_seed(seed) differences = [a - b for a, b in paired_scores] - rng = random.Random(seed) + rng = random.Random(declared_seed) resampled_means = sorted( sum(rng.choice(differences) for _ in differences) / len(differences) for _ in range(iterations) ) - lower_index = int(0.025 * (iterations - 1)) - upper_index = int(0.975 * (iterations - 1)) + tail_mass = (1.0 - coverage) / 2.0 + lower_index = int(tail_mass * (iterations - 1)) + upper_index = int((1.0 - tail_mass) * (iterations - 1)) + if lower_index >= upper_index: + raise BenchmarkContractError( + "declared coverage cannot be represented with the resample count" + ) return { "mean_difference": round(sum(differences) / len(differences), 6), "ci_low": round(resampled_means[lower_index], 6), "ci_high": round(resampled_means[upper_index], 6), "iterations": iterations, - "seed": seed, + "confidence_level": coverage, + "seed": declared_seed, "pair_count": len(differences), - "method": "paired_bootstrap_percentile_95", + "method": "paired_bootstrap_percentile", } @@ -2515,9 +2608,22 @@ def best_single_worker_hindsight( def paired_policy_comparisons( - cells: list[dict[str, Any]], seed: int + cells: list[dict[str, Any]], + *, + seed: int | None = None, + comparison_pairs: object = None, + resample_count: int | None = None, + confidence_level: float | None = None, ) -> list[dict[str, Any]]: - """Compare delivered score and terminal-outcome time on all shared tasks.""" + """Compare delivered score and terminal-outcome time on declared policy pairs. + + Comparison pairs, resample count, coverage, and seed are required + declarations. Unobserved or disjoint pairs are omitted rather than imputed. + """ + declared_pairs = _require_declared_comparison_pairs(comparison_pairs) + declared_seed = _require_declared_seed(seed) + _require_declared_positive_int(resample_count, "resample_count") + _require_declared_confidence_level(confidence_level) policy_cells: dict[str, dict[str, dict[str, Any]]] = {} locked_cells = [cell for cell in cells if cell["task_split"] == "locked"] for cell in locked_cells: @@ -2538,17 +2644,13 @@ def paired_policy_comparisons( if type(task_score) not in (int, float) or not 0 <= task_score <= 1: raise BenchmarkContractError("invalid successful task_score observation") task_cells[cell["task_id"]] = cell - summaries = summarize_policies(locked_cells) - hindsight = best_single_worker_hindsight(summaries) - comparison_pairs = [ - ("conduct_bounded", "route_once"), - ("cheapest_eligible_worker", "route_once"), - ] - if hindsight is not None: - comparison_pairs.append(("route_once", hindsight["policy_name"])) - comparison_pairs.append(("conduct_bounded", hindsight["policy_name"])) comparisons = [] - for policy_a, policy_b in comparison_pairs: + bootstrap_declaration = { + "resample_count": resample_count, + "confidence_level": confidence_level, + "seed": declared_seed, + } + for policy_a, policy_b in declared_pairs: tasks_a, tasks_b = policy_cells.get(policy_a), policy_cells.get(policy_b) if not tasks_a or not tasks_b: continue @@ -2581,9 +2683,9 @@ def paired_policy_comparisons( ), "policy_a_unpaired_task_count": len(tasks_a) - len(shared_tasks), "policy_b_unpaired_task_count": len(tasks_b) - len(shared_tasks), - **paired_bootstrap_mean_difference(score_pairs, seed=seed), + **paired_bootstrap_mean_difference(score_pairs, **bootstrap_declaration), "end_to_end_latency_ms": paired_bootstrap_mean_difference( - latency_pairs, seed=seed + latency_pairs, **bootstrap_declaration ), } ) @@ -2799,6 +2901,11 @@ def _validate_live_provenance(git_sha: str, workflow_run_id: str) -> None: "evaluation.worker_count", "evaluation.cheapest_worker_skip_reason", "provenance.benchmark_parameters.max_eval_models", + "provenance.benchmark_parameters.max_output_tokens", + "provenance.benchmark_parameters.max_workflow_depth", + "provenance.benchmark_parameters.bootstrap_resample_count", + "provenance.benchmark_parameters.confidence_level", + "provenance.benchmark_parameters.comparison_pairs", "evaluation.policy_summaries", "evaluation.paired_comparisons", "evaluation.pareto_frontiers", @@ -2868,6 +2975,16 @@ def validate_report_schema(report: dict[str, Any]) -> None: model_limit = report["provenance"]["benchmark_parameters"]["max_eval_models"] if type(model_limit) is not int or model_limit < 1: raise BenchmarkContractError("evaluation model limit must be a positive integer") + parameters = report["provenance"]["benchmark_parameters"] + _require_declared_positive_int( + parameters["bootstrap_resample_count"], "resample_count" + ) + _require_declared_confidence_level(parameters["confidence_level"]) + _require_declared_comparison_pairs(parameters["comparison_pairs"]) + _require_declared_positive_int(parameters["max_output_tokens"], "max_output_tokens") + _require_declared_positive_int( + parameters["max_workflow_depth"], "max_workflow_depth" + ) workers = build_worker_agents( report["catalog_snapshot"]["probed_models"], "mock://plan-validation", model_limit ) @@ -2980,7 +3097,11 @@ def render_markdown_summary(report: dict[str, Any]) -> str: ) lines += [ "", - "## Paired comparisons (95% bootstrap CI)", + ( + "## Paired comparisons " + f"({report['provenance']['benchmark_parameters']['confidence_level']:g} " + "percentile bootstrap interval)" + ), "", ( "Differences are A minus B on all shared locked tasks. Failed delivery " @@ -3105,7 +3226,6 @@ def assemble_benchmark_report( evaluation: dict[str, Any], request_budget: RequestBudget, provenance_inputs: dict[str, Any], - seed: int, ) -> dict[str, Any]: """Assemble and validate the complete evidence-grade benchmark report.""" cells = evaluation["evaluation_cells"] @@ -3147,7 +3267,19 @@ def assemble_benchmark_report( "planned_evaluation_cells": evaluation["planned_evaluation_cells"], "policy_summaries": summaries, "best_single_worker_hindsight": best_single_worker_hindsight(summaries), - "paired_comparisons": paired_policy_comparisons(cells, seed=seed), + "paired_comparisons": paired_policy_comparisons( + cells, + seed=provenance_inputs["benchmark_parameters"]["seed"], + comparison_pairs=provenance_inputs["benchmark_parameters"][ + "comparison_pairs" + ], + resample_count=provenance_inputs["benchmark_parameters"][ + "bootstrap_resample_count" + ], + confidence_level=provenance_inputs["benchmark_parameters"][ + "confidence_level" + ], + ), "pareto_frontiers": build_pareto_frontiers(summaries), "cheapest_worker_skip_reason": evaluation["cheapest_worker_skip_reason"], "locked_task_count": evaluation["locked_task_count"], @@ -3323,6 +3455,9 @@ def run_benchmark( max_output_tokens: int | None = None, max_eval_models: int = 7, seed: int = 7, + resample_count: int | None = None, + confidence_level: float | None = None, + comparison_pairs: object = None, max_workflow_depth: int | None = None, git_sha: str = "", workflow_run_id: str = "", @@ -3346,6 +3481,9 @@ def run_benchmark( max_output_tokens: Declared per-provider-call output-token cap. max_eval_models: Maximum chat-eligible workers in policy evaluation. seed: Deterministic bootstrap seed. + resample_count: Declared paired-bootstrap resample count. + confidence_level: Declared exclusive-unit-interval percentile coverage. + comparison_pairs: Declared ordered policy pairs to compare. max_workflow_depth: Declared equal-call workflow envelope. The equal cell-wide prompt-plus-completion budget is ``max_output_tokens`` multiplied by this value. @@ -3364,6 +3502,12 @@ def run_benchmark( raise BenchmarkContractError( f"run_mode must be 'dry_run' or 'live', not {run_mode!r}" ) + declared_resample_count = _require_declared_positive_int( + resample_count, "resample_count" + ) + declared_confidence_level = _require_declared_confidence_level(confidence_level) + declared_comparison_pairs = _require_declared_comparison_pairs(comparison_pairs) + declared_seed = _require_declared_seed(seed) declared_max_output_tokens = _require_declared_positive_int( max_output_tokens, "max_output_tokens" ) @@ -3432,7 +3576,10 @@ def dry_run_probe_timer() -> float: "policy_maximum_calls": declared_max_workflow_depth, "minimum_paired_task_count": None, "required_completion_fraction": None, - "seed": seed, + "seed": declared_seed, + "bootstrap_resample_count": declared_resample_count, + "confidence_level": declared_confidence_level, + "comparison_pairs": [list(pair) for pair in declared_comparison_pairs], "task_manifest_version": manifest["manifest_version"], "pricing_scenario_version": ( pricing_scenario["scenario_version"] if pricing_scenario else None @@ -3512,7 +3659,6 @@ def dry_run_probe_timer() -> float: "benchmark_parameters": benchmark_parameters, "request_plan": request_plan, }, - seed, ) report["artifact_paths"] = write_benchmark_artifacts(report, output_dir) return report @@ -3571,6 +3717,25 @@ def run_benchmark_cli(argv: list[str]) -> int: ) parser.add_argument("--max-eval-models", type=int, default=7) parser.add_argument("--seed", type=int, default=7) + parser.add_argument( + "--bootstrap-resample-count", + type=int, + default=None, + help="Declared paired-bootstrap resample count. Required; there is no hidden default.", + ) + parser.add_argument( + "--confidence-level", + type=float, + default=None, + help="Declared exclusive-unit-interval percentile coverage. Required.", + ) + parser.add_argument( + "--comparison-pair", + action="append", + dest="comparison_pairs", + default=None, + help="Declared policy pair as policy_a,policy_b. Repeat to compare more pairs.", + ) parser.add_argument( "--git-sha", default="", @@ -3599,6 +3764,9 @@ def run_benchmark_cli(argv: list[str]) -> int: max_output_tokens=args.max_output_tokens, max_eval_models=args.max_eval_models, seed=args.seed, + resample_count=args.bootstrap_resample_count, + confidence_level=args.confidence_level, + comparison_pairs=_comparison_pairs_from_cli(args.comparison_pairs), max_workflow_depth=args.max_workflow_depth, git_sha=args.git_sha, workflow_run_id=args.workflow_run_id, diff --git a/docs/doctoring/nim-benchmark-evidence-grade.md b/docs/doctoring/nim-benchmark-evidence-grade.md index d8415e2f4..d885c6014 100644 --- a/docs/doctoring/nim-benchmark-evidence-grade.md +++ b/docs/doctoring/nim-benchmark-evidence-grade.md @@ -211,6 +211,90 @@ Current-head full/hosted verification, independent review and protected release remain separate gates. This corrects report provenance and cohort isolation; it does not establish probability sampling, model accuracy or faster decisions. +### Declared paired-bootstrap coverage (2026-09-07, proposed) + +The previous paired comparison used a hidden 2,000-resample 95% percentile +interval and a hard-coded policy subset (`conduct_bounded` versus `route_once`, +optional cheapest versus `route_once`, and hindsight pairs when a unique +winner existed). Those choices were not reconstructible as operator +declarations. Report version 4 requires `bootstrap_resample_count`, +`confidence_level`, and `comparison_pairs` in provenance. The method name is +`paired_bootstrap_percentile`. A coverage that cannot be represented with the +resample count fails closed. CLI and workflow flags carry the same +declarations; 2,000 and 0.95 in those files are run choices, not code +defaults. Hindsight identity remains a measurement field; comparing against it +requires an explicit pair. + +Efron (1979) grounds resampling observed task units. Efron and Tibshirani +(1993) ground the percentile interval and treat *B* as Monte Carlo precision. +This slice does not add a statistical dependency or change production +route/conduct defaults. Token and workflow-depth budgets are the successor +slice recorded below. + +```mermaid +sequenceDiagram + participant Operator as Run declaration + participant Compare as Paired comparison + participant Interval as Percentile interval + participant Report as Schema 4 report + Operator->>Compare: Policy pairs, resample count, coverage, seed + Compare->>Compare: Fail closed on missing or invalid declarations + Compare->>Interval: Shared locked-task differences + Interval->>Report: Mean difference and declared-coverage interval + Note over Operator,Report: Unobserved pairs are omitted; production gates still apply +``` + +### Declared workflow depth and token budgets (2026-09-07, proposed) + +The previous equal-budget cell used a hidden five-step workflow and a +264-token per-call output cap. Those numbers allocated evaluation compute +and shaped request planning without an operator declaration. Planning, +evaluation, CLI, and provenance now require positive integer declarations. +`None` is a fail-closed sentinel. The equal cell token budget is the product +of the declared output cap and the declared workflow depth. Five and 264 in +the workflow YAML are run choices, not code defaults. + +This slice does not change production route/conduct defaults. Held-out +bootstrap coverage is the successor slice recorded below. + +```mermaid +sequenceDiagram + participant Operator as Run declaration + participant Plan as Request plan + participant Cell as Equal-budget cell + participant Report as Schema 4 report + Operator->>Plan: Workflow depth and output-token cap + Plan->>Plan: Fail closed on missing or non-positive declarations + Plan->>Cell: Equal call envelope and token product + Cell->>Report: Configured budget and observed usage + Note over Operator,Report: Production route and conduct defaults stay locked +``` + +### Declared held-out bootstrap coverage (2026-09-07, proposed) + +The psychometric held-out harness used a hidden 2,000-sample 95% paired +interval. Those numbers chose Monte Carlo precision without an operator +declaration. `_paired_bootstrap_mean_ci`, `run_benchmark`, and the adaptive +calibration helper now require resample count, exclusive-unit-interval +coverage, and seed. The existing floor/ceil index mapping is kept so +synthetic fixtures stay comparable. The script entry writes 2,000, 0.95, and +seed 568 as this run's choices. The report records those fields. + +Efron (1979) grounds resampling observed units. This slice does not change +production route/conduct defaults. Other harness sample sizes remain later +work. + +```mermaid +sequenceDiagram + participant Operator as Run declaration + participant Interval as Held-out interval + participant Report as Held-out report + Operator->>Interval: Resample count, coverage, seed + Interval->>Interval: Fail closed on missing or non-representable declarations + Interval->>Report: Mean difference and declared-coverage interval + Note over Operator,Report: Production route and conduct defaults stay locked +``` + ### Failure-inclusive comparison repair (2026-09-05, proposed) The previous paired comparison selected only jointly successful cells even @@ -486,6 +570,9 @@ Chen, L., Zaharia, M., & Zou, J. (2023). FrugalGPT: How to use large language models while reducing cost and improving performance. *arXiv*. https://doi.org/10.48550/arXiv.2305.05176 +Efron, B., & Tibshirani, R. J. (1993). *An introduction to the bootstrap*. +Chapman & Hall. https://doi.org/10.1201/9780429246593 + Efron, B. (1979). Bootstrap methods: Another look at the jackknife. *The Annals of Statistics, 7*(1), 1–26. https://doi.org/10.1214/aos/1176344552 diff --git a/docs/nim_benchmark.md b/docs/nim_benchmark.md index 7053c824d..8742e4bad 100644 --- a/docs/nim_benchmark.md +++ b/docs/nim_benchmark.md @@ -19,15 +19,24 @@ The detailed engineering and evidence record is python -m contextual_orchestrator nim-benchmark --dry-run \ --pricing-scenario examples/nim_pricing_scenario.json \ --output-dir benchmark_artifacts \ + --bootstrap-resample-count 2000 \ + --confidence-level 0.95 \ + --comparison-pair conduct_bounded,route_once \ --max-workflow-depth 5 \ --max-output-tokens 264 # Live CI run: the workflow injects NVIDIA_NIM_API_KEY only into the live step. # The process bootstraps it into the credential registry and runtime access -# resolves the credential by name. +# resolves the credential by name. Resample count, coverage, comparison pairs, +# workflow depth, and output-token cap are required declarations; the values +# below are this run's choices, not hidden code defaults. python -m contextual_orchestrator nim-benchmark \ --max-total-requests 2000 \ + --max-workflow-depth 5 \ --max-output-tokens 264 \ + --bootstrap-resample-count 2000 \ + --confidence-level 0.95 \ + --comparison-pair conduct_bounded,route_once \ --git-sha "$GITHUB_SHA" \ --workflow-run-id "$GITHUB_RUN_ID" ``` @@ -210,18 +219,27 @@ failure record retains `task_score: null`; zero delivery reward is not an estimate of an unobserved answer's correctness or a psychometric response. Each comparison reports A-minus-B mean delivered-score and elapsed-time -differences with paired 95% bootstrap intervals, successful outcome counts on -the shared tasks, and unmatched task counts. Elapsed time ends at the recorded -terminal outcome, including a failure or timeout. A fast failure is therefore -visible alongside its zero delivery reward; lower elapsed time alone is not an -improvement in service. The intervals condition on the common task set and the -selected policies, including the explicitly labelled hindsight worker. -When direct workers tie for the highest quality, no unique hindsight worker -is selected and its comparisons are omitted. The observations and other policy -comparisons remain available; model names never break a quality tie. +differences with paired percentile bootstrap intervals, successful outcome +counts on the shared tasks, and unmatched task counts. Coverage, resample +count, and the compared policy pairs are declared in provenance; they are not +hidden 2,000-resample or 95% defaults and not a baked-in conduct/route subset. +Elapsed time ends at the recorded terminal outcome, including a failure or +timeout. A fast failure is therefore visible alongside its zero delivery +reward; lower elapsed time alone is not an improvement in service. The +intervals condition on the common task set and the declared policy pairs. +Hindsight worker identity remains a separate measurement field; comparing +against it requires an explicit pair declaration. + +### Comparison report version 4 + +Version 4 requires declared `bootstrap_resample_count`, `confidence_level`, +and `comparison_pairs` in report provenance. The percentile method name is +`paired_bootstrap_percentile`; coverage is a separate numeric field. A +declared coverage that cannot be represented with the resample count fails +closed. Older reports must be regenerated. Reports using version 1 compared only jointly successful tasks. Their values -must not be pooled with version 2 or 3, and the validator rejects old schemas. +must not be pooled with version 2, 3, or 4, and the validator rejects old schemas. Version 2 observations require their independently preserved evaluation plan before migration; do not infer a complete plan from surviving observations. No task-count or completion-fraction threshold authorizes production promotion. diff --git a/docs/papers/README.md b/docs/papers/README.md index 216b49b16..8d49c0ea6 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -222,6 +222,15 @@ verified observed-task evidence and the protected release process. unobserved psychometric response. Mean intervals do not establish p95 performance. Citation and summary only; redistribution was not established. +- Efron, B., & Tibshirani, R. J. (1993). *An introduction to the bootstrap*. + Chapman & Hall. https://doi.org/10.1201/9780429246593 + Grounds the percentile interval: ordered bootstrap replications, with + coverage α taken from the operator declaration rather than a hidden 95% + constant, and resample count B as a Monte Carlo precision declaration. + NIM report version 4 records both values in provenance and fails closed + when coverage cannot be represented with B. Citation and summary only; + redistribution was not established. + - **Holistic Evaluation of Language Models (HELM)** — Percy Liang, Rishi Bommasani, Tony Lee, et al. arXiv:2211.09110, 2022 (TMLR 2023). `helm-holistic-evaluation-2211.09110.pdf` diff --git a/docs/planning/adrs/0042-declared-paired-bootstrap-coverage.md b/docs/planning/adrs/0042-declared-paired-bootstrap-coverage.md new file mode 100644 index 000000000..30a22bc9a --- /dev/null +++ b/docs/planning/adrs/0042-declared-paired-bootstrap-coverage.md @@ -0,0 +1,83 @@ +--- +id: "0042" +title: "Declare paired-bootstrap coverage and comparison pairs" +status: proposed +proposed_date: "2026-09-07" +deciders: + - "repository maintainer" +affected_components: + - "contextual_orchestrator/nim_benchmark.py" +related: + - path: "docs/planning/adrs/0034-anti-heuristic-routing-evidence.md" + relation: extends +success_criteria: + - metric: "no hidden interval default" + target: "omitted resample_count or confidence_level fails closed" + source: "tests/test_nim_benchmark.py::test_paired_bootstrap_rejects_undeclared_or_invalid_coverage" + - metric: "no baked-in policy subset" + target: "omitted comparison_pairs fails closed; unobserved pairs are skipped, not invented" + source: "tests/test_nim_benchmark.py::test_paired_policy_comparisons_reject_undeclared_or_invalid_pairs" +--- + +# ADR 0042: Declare paired-bootstrap coverage and comparison pairs + +- Status: Proposed +- Date: 2026-09-07 +- Doctoring record: [`docs/doctoring/nim-benchmark-evidence-grade.md`](../../doctoring/nim-benchmark-evidence-grade.md) + +## Product requirement + +A buyer comparing `route` and `conduct` needs to know which policies were +compared and how the uncertainty interval was formed. A hidden 2,000-resample +95% interval, or a hard-coded conduct/route/cheapest/hindsight subset, cannot +be defended as measurement design. The operator must declare resample count, +percentile coverage, and policy pairs for each run. + +## Decision + +In the context of NIM paired policy evidence, facing hidden 2,000-resample +95% intervals and a baked-in comparison subset, we chose required declarations +and against restoring those constants or auto-comparing every observed pair, +to keep the estimand explicit, accepting that a run without flags fails closed +and that undeclared hindsight comparisons are omitted. + +`paired_bootstrap_mean_difference` takes keyword-only `resample_count`, +`confidence_level`, and `seed`. `None` is a fail-closed sentinel, not a +statistical default. Coverage must be a finite exclusive unit interval. The +percentile indices follow Efron and Tibshirani (1993); if the lower and upper +indices collapse, the declaration cannot be represented and the run fails. +The method name is `paired_bootstrap_percentile`; coverage is a numeric field. + +`paired_policy_comparisons` takes declared `comparison_pairs`. Empty, malformed, +duplicate, or identical-name pairs fail closed. Unobserved or disjoint pairs +are omitted rather than imputed. Hindsight identity remains a separate +measurement; comparing against it requires an explicit pair. + +Report schema 4.0.0 records the declarations in provenance. Production +route/conduct defaults stay locked. Token and workflow-depth budgets are the +successor slice in ADR 0043. + +## Alternatives considered + +- Keep 2,000 and 0.95 as code defaults. Rejected: they hide Monte Carlo + precision and coverage from the report consumer. +- Auto-compare every observed locked policy pair. Rejected for this slice: + exhaustive pairing is a different estimand and would change dry-run reports + without an operator declaration. It can be declared later as an explicit + pair list. +- Adopt RankWeave's released comparison API. Rejected: v0.18.0 does not accept + generic response times or a paired percentile of mean differences. + +## Consequences + +Positive: interval coverage and compared policies are reconstructible from the +report and the workflow flags. + +Negative: existing CLI, workflow, and library callers must pass the +declarations; schema 3 reports cannot be reused. + +## Remaining work + +Token and workflow-depth budgets move to ADR 0043. Held-out bootstrap +coverage moves to ADR 0044. This ADR is Proposed until independent review +and protected delivery. diff --git a/docs/planning/adrs/0043-declared-workflow-token-budgets.md b/docs/planning/adrs/0043-declared-workflow-token-budgets.md index 4ed077510..87488ebbc 100644 --- a/docs/planning/adrs/0043-declared-workflow-token-budgets.md +++ b/docs/planning/adrs/0043-declared-workflow-token-budgets.md @@ -10,6 +10,8 @@ affected_components: related: - path: "docs/planning/adrs/0041-generalize-models-dev-cost-classification.md" relation: related + - path: "docs/planning/adrs/0044-declared-heldout-bootstrap-coverage.md" + relation: followed_by success_criteria: - metric: "no hidden workflow envelope" target: "omitted maximum_calls or max_workflow_depth fails closed" @@ -51,8 +53,8 @@ Workflow YAML and tests may still write 5 and 264 as this run's choices. Report schema stays 4.0.0; those fields already exist in provenance and are now validated as declarations. Production route/conduct defaults stay locked. -The psychometric held-out harness's 2,000-sample 95% interval remains a later -slice. +The psychometric held-out harness's 2,000-sample 95% interval is the +successor slice in ADR 0044. ## Alternatives considered @@ -74,5 +76,5 @@ declarations; omitting them fails closed. ## Remaining work -The psychometric held-out harness still uses a 2,000-sample 95% interval. -This ADR is Proposed until independent review and protected delivery. +Held-out bootstrap coverage moves to ADR 0044. This ADR is Proposed until +independent review and protected delivery. diff --git a/docs/planning/adrs/0044-declared-heldout-bootstrap-coverage.md b/docs/planning/adrs/0044-declared-heldout-bootstrap-coverage.md new file mode 100644 index 000000000..008951015 --- /dev/null +++ b/docs/planning/adrs/0044-declared-heldout-bootstrap-coverage.md @@ -0,0 +1,70 @@ +--- +id: "0044" +title: "Declare held-out paired-bootstrap coverage" +status: proposed +proposed_date: "2026-09-07" +deciders: + - "repository maintainer" +affected_components: + - "scripts/benchmark_psychometric_heldout.py" +related: + - path: "docs/planning/adrs/0042-declared-paired-bootstrap-coverage.md" + relation: extends +success_criteria: + - metric: "no hidden held-out interval default" + target: "omitted resample_count, confidence_level, or seed fails closed" + source: "tests/test_psychometric_routing.py::test_paired_bootstrap_interval_requires_declared_coverage" +--- + +# ADR 0044: Declare held-out paired-bootstrap coverage + +- Status: Proposed +- Date: 2026-09-07 +- Doctoring record: [`docs/doctoring/nim-benchmark-evidence-grade.md`](../../doctoring/nim-benchmark-evidence-grade.md) + +## Product requirement + +Buyer-facing accuracy and decision-latency intervals on the psychometric +held-out harness must be reconstructible. A hidden 2,000-sample 95% interval +is Monte Carlo precision chosen by the repository, not a declared analysis. + +## Decision + +In the context of the held-out warm-start harness, facing +`BOOTSTRAP_SAMPLES = 2_000` and a baked-in 95% percentile, we chose required +declarations and against restoring those constants, to keep coverage explicit, +accepting that `run_benchmark()` without kwargs fails closed. + +`_paired_bootstrap_mean_ci` takes keyword-only `resample_count`, +`confidence_level`, and `seed`. Percentile indices keep the existing +floor/ceil mapping so historical synthetic fixtures stay comparable; a +coverage that cannot be represented with the resample count fails closed. +The script entry passes 2,000, 0.95, and seed 568 as this run's choices. +The report records `bootstrap_samples`, `bootstrap_confidence_level`, and +`bootstrap_seed`. + +Production route/conduct defaults stay locked. Nested `*_ci95` JSON key names +remain a later naming slice. Other harness sample sizes (assignment trials, +DIF, reliability) are unchanged. + +## Alternatives considered + +- Keep 2,000 and 0.95 as module constants. Rejected: they hide Monte Carlo + precision from the report consumer. +- Switch to the NIM integer-index formula from ADR 0042. Rejected for this + slice: it would move the 2,000-sample lower index and invalidate existing + synthetic fixtures without a new estimand. + +## Consequences + +Positive: held-out interval coverage is reconstructible from the report and +the script entry. + +Negative: library callers of `run_benchmark` and the calibration helper must +pass the declarations. + +## Remaining work + +Other repository-authored harness sample sizes stay open. Nested `*_ci95` +key names still embed 95. This ADR is Proposed until independent review and +protected delivery. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 71d71a2ef..ded9d72ae 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,42 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-07 declared held-out bootstrap coverage (proposed) + +Successor of the declared workflow-budget slice removes hidden +`BOOTSTRAP_SAMPLES = 2_000` and the baked-in 95% percentile from +`scripts/benchmark_psychometric_heldout.py`. Resample count, exclusive-unit-interval +coverage, and seed are required declarations. Missing, boolean, non-positive, +or non-representable declarations fail closed. The script entry and full +harness tests pass 2,000, 0.95, and seed 568 as this run's choices. ADR 0044 +is Proposed. + +Local contract tests on this working tree: 19 related declaration and +boundary tests passed. This is not buyer-held-out accuracy, p95 latency, or +protected merge evidence. Production route/conduct defaults stay locked. +Other harness sample sizes and nested `*_ci95` key names remain later work. +Parent [#1067](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1067) +still needs independent review. + +## 2026-09-07 declared paired-bootstrap coverage (proposed) + +Child successor of [#1074](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1074) +removes the hidden 2,000-resample 95% interval and the baked-in +`conduct_bounded` / `route_once` / cheapest / hindsight comparison subset from +`contextual_orchestrator/nim_benchmark.py`. Resample count, exclusive-unit-interval +coverage, seed, and policy pairs are required declarations. Missing, boolean, +non-positive, non-finite, empty, duplicate, or degenerate declarations fail +closed. The percentile method name no longer embeds 95. Report schema 4.0.0 +records the declarations in provenance. The workflow and CLI must pass them +explicitly; 2,000 and 0.95 in those files are run declarations, not code +defaults. + +Local three-file coverage on this working tree: NIM statements/branches 100%, +interrogate 100%, 175 related tests passed. This is not buyer-held-out +accuracy, p95 latency, or protected merge evidence. Production route/conduct +defaults stay locked. Token and workflow-depth budgets are the successor +slice. Parent [#1067](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1067) +still needs independent review. + ## 2026-09-07 declared workflow depth and token budgets (proposed) Removes hidden `MAX_WORKFLOW_DEPTH = 5` and `DEFAULT_MAX_OUTPUT_TOKENS = 264` diff --git a/scripts/benchmark_psychometric_heldout.py b/scripts/benchmark_psychometric_heldout.py index d33f98812..8dec2f8eb 100644 --- a/scripts/benchmark_psychometric_heldout.py +++ b/scripts/benchmark_psychometric_heldout.py @@ -28,8 +28,11 @@ MODEL_IDS = tuple(f"model_{index}" for index in range(4)) UNSEEN_MODEL_ID = "model_unseen" TRAIN_CONTEXTS = 24 -BOOTSTRAP_SAMPLES = 2_000 -BOOTSTRAP_SEED = 568 +# Script-entry run declarations. Functions take these as required arguments; +# they are not hidden statistical defaults. +DECLARED_BOOTSTRAP_RESAMPLE_COUNT = 2_000 +DECLARED_BOOTSTRAP_CONFIDENCE_LEVEL = 0.95 +DECLARED_BOOTSTRAP_SEED = 568 LATENCY_REPETITIONS = 200 ASSIGNMENT_TRIALS = 24_000 ASSIGNMENT_SEED = 260_905 @@ -85,19 +88,76 @@ def _vector(angle: float) -> list[float]: return [math.cos(angle), math.sin(angle)] +def _require_declared_positive_int(value: object, field_name: str) -> int: + """Reject missing, boolean, or non-positive integer declarations.""" + if type(value) is not int or value < 1: + raise ValueError(f"{field_name} must be a declared positive integer") + return value + + +def _require_declared_confidence_level(value: object) -> float: + """Reject missing or non-exclusive-unit-interval coverage declarations.""" + if type(value) is not float or not math.isfinite(value) or not 0.0 < value < 1.0: + raise ValueError( + "confidence_level must be a declared finite exclusive unit interval" + ) + return value + + +def _require_declared_seed(value: object) -> int: + """Reject missing or boolean bootstrap seeds.""" + if type(value) is not int: + raise ValueError("seed must be a declared integer") + return value + + +def _declared_bootstrap( + *, + resample_count: int | None = None, + confidence_level: float | None = None, + seed: int | None = None, +) -> dict[str, int | float]: + """Return validated bootstrap declarations. ``None`` is fail-closed.""" + return { + "resample_count": _require_declared_positive_int( + resample_count, "resample_count" + ), + "confidence_level": _require_declared_confidence_level(confidence_level), + "seed": _require_declared_seed(seed), + } + + def _paired_bootstrap_mean_ci( - candidate: list[float], baseline: list[float] + candidate: list[float], + baseline: list[float], + *, + resample_count: int | None = None, + confidence_level: float | None = None, + seed: int | None = None, ) -> list[float]: - """Return a deterministic paired 95% interval for candidate-minus-baseline.""" + """Return a deterministic paired percentile interval for candidate-minus-baseline. + + ``resample_count``, ``confidence_level``, and ``seed`` are required + declarations. ``None`` is a fail-closed sentinel, not a statistical default. + """ if not candidate or len(candidate) != len(baseline): raise ValueError("paired samples must be non-empty and equal length") + iterations = _require_declared_positive_int(resample_count, "resample_count") + coverage = _require_declared_confidence_level(confidence_level) + declared_seed = _require_declared_seed(seed) differences = [left - right for left, right in zip(candidate, baseline)] - generator = random.Random(BOOTSTRAP_SEED) + generator = random.Random(declared_seed) means = sorted( statistics.fmean(generator.choices(differences, k=len(differences))) - for _ in range(BOOTSTRAP_SAMPLES) + for _ in range(iterations) ) - return [means[math.floor(0.025 * BOOTSTRAP_SAMPLES)], means[math.ceil(0.975 * BOOTSTRAP_SAMPLES) - 1]] + lower_quantile = (1.0 - coverage) / 2.0 + upper_quantile = 1.0 - lower_quantile + lower_index = math.floor(lower_quantile * iterations) + upper_index = math.ceil(upper_quantile * iterations) - 1 + if not 0 <= lower_index < upper_index < iterations: + raise ValueError("declared coverage cannot be represented with the resample count") + return [means[lower_index], means[upper_index]] def _build_evidence(*, two_neighbor: bool) -> PsychometricRoutingEvidence: @@ -1212,8 +1272,18 @@ def _validate_parameter_uncertainty() -> dict[str, object]: } -def _validate_adaptive_candidate_calibration() -> dict[str, object]: +def _validate_adaptive_candidate_calibration( + *, + resample_count: int | None = None, + confidence_level: float | None = None, + seed: int | None = None, +) -> dict[str, object]: """Compare information-selected and random onboarding queries on known truth.""" + bootstrap = _declared_bootstrap( + resample_count=resample_count, + confidence_level=confidence_level, + seed=seed, + ) from fast_mlsirm import cat_next_item candidate_thetas = tuple( @@ -1327,7 +1397,7 @@ def evaluate( } paired_delta_ci95 = { metric: _paired_bootstrap_mean_ci( - adaptive_samples[metric], random_samples[metric] + adaptive_samples[metric], random_samples[metric], **bootstrap ) for metric in adaptive_samples } @@ -1420,9 +1490,10 @@ def summarize_stratum(lower: float, upper: float | None) -> dict[str, float]: sequential_queries, [float(ADAPTIVE_CALIBRATION_MAX_ITEMS)] * ADAPTIVE_CALIBRATION_CANDIDATES, + **bootstrap, ), "accuracy_delta_ci95": _paired_bootstrap_mean_ci( - sequential_correct, fixed_correct + sequential_correct, fixed_correct, **bootstrap ), "confidence_resolved_rate": statistics.fmean( row[4] for row in classification_rows @@ -1579,10 +1650,14 @@ def monte_carlo_se(metric: str) -> float: }, "heldout_paired_delta_ci95": { "coverage": _paired_bootstrap_mean_ci( - heldout_samples["resolved"], heldout_baseline_samples["resolved"] + heldout_samples["resolved"], + heldout_baseline_samples["resolved"], + **bootstrap, ), "all_candidate_queries": _paired_bootstrap_mean_ci( - heldout_samples["queries"], heldout_baseline_samples["queries"] + heldout_samples["queries"], + heldout_baseline_samples["queries"], + **bootstrap, ), }, "replication_audit": { @@ -1654,13 +1729,29 @@ def monte_carlo_se(metric: str) -> float: } -def run_benchmark() -> dict[str, object]: - """Return paired held-out accuracy uncertainty and decision latency.""" +def run_benchmark( + *, + resample_count: int | None = None, + confidence_level: float | None = None, + seed: int | None = None, +) -> dict[str, object]: + """Return paired held-out accuracy uncertainty and decision latency. + + Bootstrap resample count, percentile coverage, and seed are required + declarations. ``None`` is a fail-closed sentinel, not a statistical default. + """ + bootstrap = _declared_bootstrap( + resample_count=resample_count, + confidence_level=confidence_level, + seed=seed, + ) baseline_evidence = _build_evidence(two_neighbor=False) candidate_evidence = _build_evidence(two_neighbor=True) baseline, baseline_samples = _evaluate_quality(baseline_evidence) candidate, candidate_samples = _evaluate_quality(candidate_evidence) - adaptive_candidate_calibration = _validate_adaptive_candidate_calibration() + adaptive_candidate_calibration = _validate_adaptive_candidate_calibration( + **bootstrap + ) unseen_predictions = sum( bool( candidate_evidence.ranked_evidence( @@ -1726,7 +1817,7 @@ def run_benchmark() -> dict[str, object]: } delta_ci95 = { metric: _paired_bootstrap_mean_ci( - candidate_samples[metric], baseline_samples[metric] + candidate_samples[metric], baseline_samples[metric], **bootstrap ) for metric in candidate_samples } @@ -1974,7 +2065,9 @@ def run_benchmark() -> dict[str, object]: result: dict[str, object] = { **candidate, "baseline": baseline, - "bootstrap_samples": BOOTSTRAP_SAMPLES, + "bootstrap_samples": bootstrap["resample_count"], + "bootstrap_confidence_level": bootstrap["confidence_level"], + "bootstrap_seed": bootstrap["seed"], "contexts_held_out": TRAIN_CONTEXTS, "contexts_train": TRAIN_CONTEXTS, "delta": delta, @@ -2024,7 +2117,11 @@ def run_benchmark() -> dict[str, object]: def main() -> None: """Print the held-out benchmark report as stable JSON.""" - result = run_benchmark() + result = run_benchmark( + resample_count=DECLARED_BOOTSTRAP_RESAMPLE_COUNT, + confidence_level=DECLARED_BOOTSTRAP_CONFIDENCE_LEVEL, + seed=DECLARED_BOOTSTRAP_SEED, + ) print(json.dumps(result, sort_keys=True)) diff --git a/tests/test_nim_benchmark.py b/tests/test_nim_benchmark.py index 1eb028d9c..51da0b12c 100644 --- a/tests/test_nim_benchmark.py +++ b/tests/test_nim_benchmark.py @@ -64,9 +64,17 @@ def _declared_policy_kwargs(**overrides: object) -> dict: return payload +DECLARED_RESAMPLE_COUNT = 2000 +DECLARED_CONFIDENCE_LEVEL = 0.95 +DECLARED_COMPARISON_PAIRS = (("conduct_bounded", "route_once"),) + + def _declared_run_kwargs(**overrides: object) -> dict: - """Return explicit workflow-budget declarations for benchmark runs.""" + """Return explicit measurement and workflow-budget declarations for runs.""" payload: dict = { + "resample_count": DECLARED_RESAMPLE_COUNT, + "confidence_level": DECLARED_CONFIDENCE_LEVEL, + "comparison_pairs": DECLARED_COMPARISON_PAIRS, "max_output_tokens": DECLARED_MAX_OUTPUT_TOKENS, "max_workflow_depth": DECLARED_MAX_WORKFLOW_DEPTH, } @@ -74,13 +82,21 @@ def _declared_run_kwargs(**overrides: object) -> dict: return payload +CLI_MEASUREMENT_FLAGS = [ + "--bootstrap-resample-count", + "2000", + "--confidence-level", + "0.95", + "--comparison-pair", + "conduct_bounded,route_once", +] CLI_WORKFLOW_BUDGET_FLAGS = [ "--max-workflow-depth", "5", "--max-output-tokens", "264", ] -CLI_RUN_FLAGS = [*CLI_WORKFLOW_BUDGET_FLAGS] +CLI_RUN_FLAGS = [*CLI_MEASUREMENT_FLAGS, *CLI_WORKFLOW_BUDGET_FLAGS] PRICING_SCENARIO_PATH = str(REPO_ROOT / "examples" / "nim_pricing_scenario.json") FAKE_ENDPOINT = "https://nim.example.test/v1" @@ -1578,8 +1594,8 @@ def chat(self, *args, **kwargs): None, RememberedContractClient(), nb.RequestBudget(100), - **_declared_policy_kwargs() - ) + **_declared_policy_kwargs(), + ) def test_evaluate_policies_all_arms_with_pricing() -> None: @@ -1593,7 +1609,7 @@ def test_evaluate_policies_all_arms_with_pricing() -> None: nb._BudgetedModelClient(budget), budget, nb._deterministic_timer(), - **_declared_policy_kwargs() + **_declared_policy_kwargs(), ) cells = evaluation["evaluation_cells"] policies = {cell["policy_name"] for cell in cells} @@ -1642,7 +1658,7 @@ def take_usage(self): None, ReportedUsageClient(), nb.RequestBudget(100), - **_declared_policy_kwargs() + **_declared_policy_kwargs(), ) assert any( cell["token_usage_source"] == "reported" @@ -1703,8 +1719,12 @@ def test_evaluate_policies_skip_reasons_without_pricing() -> None: agents = _mock_agents("vendor/model-a") budget = nb.RequestBudget(200) evaluation = nb.evaluate_policies( - agents, _mini_manifest(), None, ModelClient(), budget, - **_declared_policy_kwargs() + agents, + _mini_manifest(), + None, + ModelClient(), + budget, + **_declared_policy_kwargs(), ) assert evaluation["cheapest_worker_skip_reason"] == "no_pricing_scenario_supplied" unpriced_scenario = { @@ -1718,7 +1738,7 @@ def test_evaluate_policies_skip_reasons_without_pricing() -> None: unpriced_scenario, ModelClient(), nb.RequestBudget(200), - **_declared_policy_kwargs() + **_declared_policy_kwargs(), ) assert evaluation["cheapest_worker_skip_reason"] == "no_worker_priced_by_scenario" @@ -1730,16 +1750,82 @@ def test_evaluate_policies_skip_reasons_without_pricing() -> None: def test_paired_bootstrap_requires_pairs_and_is_deterministic() -> None: with pytest.raises(nb.BenchmarkContractError): - nb.paired_bootstrap_mean_difference([]) + nb.paired_bootstrap_mean_difference( + [], + resample_count=2000, + confidence_level=0.95, + seed=11, + ) first = nb.paired_bootstrap_mean_difference( - [(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], seed=11 + [(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], + resample_count=2000, + confidence_level=0.95, + seed=11, ) second = nb.paired_bootstrap_mean_difference( - [(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], seed=11 + [(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], + resample_count=2000, + confidence_level=0.95, + seed=11, ) assert first == second assert first["ci_low"] <= first["mean_difference"] <= first["ci_high"] assert first["pair_count"] == 3 + assert first["iterations"] == 2000 + assert first["confidence_level"] == 0.95 + assert first["method"] == "paired_bootstrap_percentile" + + +def test_paired_bootstrap_rejects_undeclared_or_invalid_coverage() -> None: + """Resample count and coverage are operator declarations, not hidden defaults.""" + pairs = [(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)] + with pytest.raises(nb.BenchmarkContractError, match="resample_count"): + nb.paired_bootstrap_mean_difference(pairs, seed=11) + with pytest.raises(nb.BenchmarkContractError, match="confidence_level"): + nb.paired_bootstrap_mean_difference(pairs, resample_count=2000, seed=11) + with pytest.raises(nb.BenchmarkContractError, match="seed"): + nb.paired_bootstrap_mean_difference( + pairs, resample_count=2000, confidence_level=0.95 + ) + for invalid_count in (True, False, 0, -1, 1.5, "2000"): + with pytest.raises(nb.BenchmarkContractError, match="resample_count"): + nb.paired_bootstrap_mean_difference( + pairs, + resample_count=invalid_count, + confidence_level=0.95, + seed=11, + ) + for invalid_coverage in (True, 0.0, 1.0, 1.5, float("nan"), float("inf"), "0.95"): + with pytest.raises(nb.BenchmarkContractError, match="confidence_level"): + nb.paired_bootstrap_mean_difference( + pairs, + resample_count=2000, + confidence_level=invalid_coverage, + seed=11, + ) + with pytest.raises(nb.BenchmarkContractError, match="seed"): + nb.paired_bootstrap_mean_difference( + pairs, + resample_count=2000, + confidence_level=0.95, + seed=True, + ) + with pytest.raises(nb.BenchmarkContractError, match="cannot be represented"): + nb.paired_bootstrap_mean_difference( + pairs, + resample_count=2, + confidence_level=0.95, + seed=11, + ) + declared = nb.paired_bootstrap_mean_difference( + pairs, + resample_count=3, + confidence_level=0.5, + seed=11, + ) + assert declared["iterations"] == 3 + assert declared["confidence_level"] == 0.5 + assert declared["method"] == "paired_bootstrap_percentile" def test_pareto_frontier_excludes_dominated_rows() -> None: @@ -1753,6 +1839,20 @@ def test_pareto_frontier_excludes_dominated_rows() -> None: assert [row["name"] for row in frontier] == ["good_cheap", "bad_cheap"] +# Fixture helper for paired comparison unit tests. +def _declared_comparison_kwargs(**overrides: object) -> dict: + """Return explicit comparison declarations for unit fixtures.""" + payload: dict = { + "seed": 3, + "comparison_pairs": DECLARED_COMPARISON_PAIRS, + "resample_count": DECLARED_RESAMPLE_COUNT, + "confidence_level": DECLARED_CONFIDENCE_LEVEL, + } + payload.update(overrides) + return payload + + + def _synthetic_cell( policy: str, task_id: str, score, outcome: str = "success", cost=0.5 ) -> dict: @@ -1916,7 +2016,7 @@ def test_best_single_worker_hindsight_selection_fails_closed_on_ties() -> None: "conduct_bounded", ) ] - comparisons = nb.paired_policy_comparisons(cells, seed=3) + comparisons = nb.paired_policy_comparisons(cells, **_declared_comparison_kwargs()) assert len(comparisons) == 1 assert comparisons[0]["policy_a"] == "conduct_bounded" assert comparisons[0]["policy_b"] == "route_once" @@ -1927,7 +2027,7 @@ def test_paired_policy_comparisons_skip_missing_and_disjoint() -> None: _synthetic_cell("conduct_bounded", "task_one", 1.0), _synthetic_cell("route_once", "task_two", 0.0), ] - assert nb.paired_policy_comparisons(disjoint, seed=3) == [] + assert nb.paired_policy_comparisons(disjoint, **_declared_comparison_kwargs()) == [] cells = [ _synthetic_cell("conduct_bounded", "task_one", 1.0), _synthetic_cell("route_once", "task_one", 0.0), @@ -1935,10 +2035,59 @@ def test_paired_policy_comparisons_skip_missing_and_disjoint() -> None: # A task observed for only one policy cannot form a pair. _synthetic_cell("route_once", "task_three", None, outcome="failure"), ] - comparisons = nb.paired_policy_comparisons(cells, seed=3) + comparisons = nb.paired_policy_comparisons( + cells, + **_declared_comparison_kwargs( + comparison_pairs=( + ("conduct_bounded", "route_once"), + ("route_once", "direct_single_worker:vendor/model-a"), + ("cheapest_eligible_worker", "route_once"), + ) + ), + ) pairs = {(row["policy_a"], row["policy_b"]) for row in comparisons} assert ("conduct_bounded", "route_once") in pairs assert ("route_once", "direct_single_worker:vendor/model-a") in pairs + assert ("cheapest_eligible_worker", "route_once") not in pairs + + +def test_paired_policy_comparisons_reject_undeclared_or_invalid_pairs() -> None: + """Policy pairs are operator declarations, not a baked-in subset.""" + cells = [ + _synthetic_cell("conduct_bounded", "task_one", 1.0), + _synthetic_cell("route_once", "task_one", 0.0), + ] + with pytest.raises(nb.BenchmarkContractError, match="comparison_pairs"): + nb.paired_policy_comparisons(cells, seed=3, resample_count=2000, confidence_level=0.95) + with pytest.raises(nb.BenchmarkContractError, match="comparison_pairs"): + nb.paired_policy_comparisons( + cells, **_declared_comparison_kwargs(comparison_pairs=()) + ) + with pytest.raises(nb.BenchmarkContractError, match="two policy names"): + nb.paired_policy_comparisons( + cells, **_declared_comparison_kwargs(comparison_pairs=(("route_once",),)) + ) + with pytest.raises(nb.BenchmarkContractError, match="distinct nonempty"): + nb.paired_policy_comparisons( + cells, + **_declared_comparison_kwargs( + comparison_pairs=(("route_once", "route_once"),) + ), + ) + with pytest.raises(nb.BenchmarkContractError, match="duplicate comparison pair"): + nb.paired_policy_comparisons( + cells, + **_declared_comparison_kwargs( + comparison_pairs=( + ("conduct_bounded", "route_once"), + ("conduct_bounded", "route_once"), + ) + ), + ) + with pytest.raises(nb.BenchmarkContractError, match="resample_count"): + nb.paired_policy_comparisons( + cells, **_declared_comparison_kwargs(resample_count=None) + ) @pytest.mark.parametrize("failure_outcome", ["failure", "timeout"]) @@ -1955,7 +2104,7 @@ def test_paired_comparisons_retain_failed_delivery_and_elapsed_time( ] for cell, latency in zip(cells, [100.0, 150.0, 2000.0, 50.0, 5.0]): cell["end_to_end_latency_ms"] = latency - comparison = nb.paired_policy_comparisons(cells, seed=3)[0] + comparison = nb.paired_policy_comparisons(cells, **_declared_comparison_kwargs())[0] assert comparison["pair_count"] == 2 assert comparison["mean_difference"] == -0.5 assert (comparison["ci_low"], comparison["ci_high"]) == (-1.0, 0.0) @@ -1978,7 +2127,7 @@ def test_paired_comparisons_keep_all_failed_pairs_without_inventing_scores() -> ] cells[0]["end_to_end_latency_ms"] = 900.0 cells[1]["end_to_end_latency_ms"] = 700.0 - comparison = nb.paired_policy_comparisons(cells, seed=3)[0] + comparison = nb.paired_policy_comparisons(cells, **_declared_comparison_kwargs())[0] assert comparison["pair_count"] == 1 assert comparison["mean_difference"] == 0.0 assert (comparison["ci_low"], comparison["ci_high"]) == (0.0, 0.0) @@ -2005,11 +2154,13 @@ def test_paired_comparisons_exclude_exploratory_tasks_and_reject_duplicate_cells _synthetic_cell("route_once", "exploratory_task", 1.0), ] cells[2]["task_split"] = cells[3]["task_split"] = "exploratory" - comparison = nb.paired_policy_comparisons(cells, seed=3)[0] + comparison = nb.paired_policy_comparisons(cells, **_declared_comparison_kwargs())[0] assert comparison["pair_count"] == 1 assert comparison["mean_difference"] == 0.0 with pytest.raises(nb.BenchmarkContractError, match="duplicate policy/task"): - nb.paired_policy_comparisons([*cells, cells[0]], seed=3) + nb.paired_policy_comparisons( + [*cells, cells[0]], **_declared_comparison_kwargs() + ) @pytest.mark.parametrize( @@ -2037,7 +2188,7 @@ def test_paired_comparisons_reject_invalid_observations( ] cells[0][field_name] = invalid_value with pytest.raises(nb.BenchmarkContractError, match=field_name): - nb.paired_policy_comparisons(cells, seed=3) + nb.paired_policy_comparisons(cells, **_declared_comparison_kwargs()) def test_pareto_frontiers_exclude_unknown_cost_policies() -> None: @@ -2131,6 +2282,36 @@ def test_report_schema_validation_reports_missing_paths() -> None: ), "unknown cheapest worker skip reason", ), + ( + lambda report: report["provenance"]["benchmark_parameters"].__setitem__( + "bootstrap_resample_count", 0 + ), + "resample_count", + ), + ( + lambda report: report["provenance"]["benchmark_parameters"].__setitem__( + "confidence_level", 1.0 + ), + "confidence_level", + ), + ( + lambda report: report["provenance"]["benchmark_parameters"].__setitem__( + "comparison_pairs", [] + ), + "comparison_pairs", + ), + ( + lambda report: report["provenance"]["benchmark_parameters"].__setitem__( + "max_output_tokens", 0 + ), + "max_output_tokens", + ), + ( + lambda report: report["provenance"]["benchmark_parameters"].__setitem__( + "max_workflow_depth", True + ), + "max_workflow_depth", + ), ], ) def test_report_schema_rejects_invalid_evaluation_contract( @@ -2150,7 +2331,7 @@ def _dry_report(output_dir: str) -> dict: PRICING_SCENARIO_PATH, output_dir, max_total_requests=900, - **_declared_run_kwargs() + **_declared_run_kwargs(), ) @@ -2192,12 +2373,14 @@ def test_report_renders_failed_delivery_and_rejects_legacy_estimand(tmp_path: Pa ] cells[0]["end_to_end_latency_ms"] = 900.0 cells[1]["end_to_end_latency_ms"] = 700.0 - report["evaluation"]["paired_comparisons"] = nb.paired_policy_comparisons(cells, 3) + report["evaluation"]["paired_comparisons"] = nb.paired_policy_comparisons( + cells, **_declared_comparison_kwargs() + ) summary = nb.render_markdown_summary(report) assert "-1.0 [-1.0, -1.0]" in summary assert "200.0 [200.0, 200.0] ms" in summary assert "successful outcomes A/B 0/1 and 1/1" in summary - assert report["benchmark_schema_version"] == "3.0.0" + assert report["benchmark_schema_version"] == "4.0.0" report["benchmark_schema_version"] = "1.0.0" with pytest.raises(nb.BenchmarkContractError, match="unsupported benchmark schema"): nb.validate_report_schema(report) @@ -2439,13 +2622,35 @@ def transport(*_args) -> tuple[int, bytes]: TASK_MANIFEST_PATH, None, "unused", - max_output_tokens=0, - max_workflow_depth=DECLARED_MAX_WORKFLOW_DEPTH, transport=transport, + **_declared_run_kwargs(max_output_tokens=0), ) assert calls == 0 +def test_run_benchmark_requires_declared_workflow_budget() -> None: + """Output-token and workflow-depth budgets are run declarations.""" + parameters = inspect.signature(nb.run_benchmark).parameters + assert parameters["max_output_tokens"].default is None + assert parameters["max_workflow_depth"].default is None + with pytest.raises(nb.BenchmarkContractError, match="max_output_tokens"): + nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + "unused", + **_declared_run_kwargs(max_output_tokens=None, max_workflow_depth=None), + ) + with pytest.raises(nb.BenchmarkContractError, match="max_workflow_depth"): + nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + "unused", + **_declared_run_kwargs(max_workflow_depth=None), + ) + + def test_dry_run_pipeline_covers_every_modality_and_is_deterministic() -> None: with tempfile.TemporaryDirectory() as tmp: first = _dry_report(os.path.join(tmp, "one")) @@ -2501,6 +2706,16 @@ def test_dry_run_pipeline_covers_every_modality_and_is_deterministic() -> None: assert first["evaluation"]["best_single_worker_hindsight"] is None assert first["evaluation"]["pareto_frontiers"]["quality_vs_latency"] assert first["evaluation"]["paired_comparisons"] + assert first["provenance"]["benchmark_parameters"][ + "bootstrap_resample_count" + ] == 2000 + assert first["provenance"]["benchmark_parameters"]["confidence_level"] == 0.95 + assert first["provenance"]["benchmark_parameters"]["comparison_pairs"] == [ + ["conduct_bounded", "route_once"] + ] + assert first["evaluation"]["paired_comparisons"][0]["method"] == ( + "paired_bootstrap_percentile" + ) # Deterministic artifacts: identical reports across runs. with open(os.path.join(tmp, "one", "benchmark_report.json"), "rb") as handle: first_bytes = handle.read() @@ -2683,7 +2898,9 @@ def test_cli_dry_run_succeeds() -> None: "--output-dir", tmp, "--max-total-requests", - "900", *CLI_RUN_FLAGS] + "900", + *CLI_RUN_FLAGS, + ] ) assert exit_code == 0 printed = json.loads(stdout.getvalue()) @@ -2695,7 +2912,12 @@ def test_cli_fails_closed_on_missing_manifest() -> None: stdout = io.StringIO() with contextlib.redirect_stdout(stdout): exit_code = nb.run_benchmark_cli( - ["--dry-run", "--task-manifest", "does/not/exist.json", *CLI_RUN_FLAGS] + [ + "--dry-run", + "--task-manifest", + "does/not/exist.json", + *CLI_RUN_FLAGS, + ] ) assert exit_code == 1 assert json.loads(stdout.getvalue())["benchmark_failed_closed"] is True @@ -2713,7 +2935,9 @@ def test_cli_live_fails_closed_without_secret( "--git-sha", "d" * 40, "--workflow-run-id", - "run-1", *CLI_RUN_FLAGS] + "run-1", + *CLI_RUN_FLAGS, + ] ) assert exit_code == 1 assert json.loads(stdout.getvalue())["error_class"] == "NotConfigured" @@ -2735,27 +2959,22 @@ def fail(*args, **kwargs): assert "[REDACTED]" in stdout.getvalue() - -def test_run_benchmark_requires_declared_workflow_budget() -> None: - """Output-token and workflow-depth budgets are run declarations.""" - parameters = inspect.signature(nb.run_benchmark).parameters - assert parameters["max_output_tokens"].default is None - assert parameters["max_workflow_depth"].default is None - with pytest.raises(nb.BenchmarkContractError, match="max_output_tokens"): - nb.run_benchmark( - "dry_run", - TASK_MANIFEST_PATH, - None, - "unused", - ) - with pytest.raises(nb.BenchmarkContractError, match="max_workflow_depth"): - nb.run_benchmark( - "dry_run", - TASK_MANIFEST_PATH, - None, - "unused", - max_output_tokens=264, - ) +def test_cli_fails_closed_without_measurement_declaration() -> None: + """CLI cannot invent a 2,000-resample 95% interval or policy subset.""" + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli(["--dry-run", "--task-manifest", TASK_MANIFEST_PATH]) + assert exit_code == 1 + payload = json.loads(stdout.getvalue()) + assert payload["benchmark_failed_closed"] is True + assert payload["error_class"] == "BenchmarkContractError" + with pytest.raises(nb.BenchmarkContractError, match="two policy names"): + nb._comparison_pairs_from_cli(["route_once"]) + with pytest.raises(nb.BenchmarkContractError, match="two policy names"): + nb._comparison_pairs_from_cli([123]) + assert nb._comparison_pairs_from_cli(["conduct_bounded,route_once"]) == ( + ("conduct_bounded", "route_once"), + ) def test_cli_fails_closed_without_workflow_budget_declaration() -> None: @@ -2763,7 +2982,7 @@ def test_cli_fails_closed_without_workflow_budget_declaration() -> None: stdout = io.StringIO() with contextlib.redirect_stdout(stdout): exit_code = nb.run_benchmark_cli( - ["--dry-run", "--task-manifest", TASK_MANIFEST_PATH] + ["--dry-run", "--task-manifest", TASK_MANIFEST_PATH, *CLI_MEASUREMENT_FLAGS] ) assert exit_code == 1 payload = json.loads(stdout.getvalue()) @@ -2773,5 +2992,6 @@ def test_cli_fails_closed_without_workflow_budget_declaration() -> None: "max_workflow_depth" in payload["error_message"] ) + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/tests/test_nim_benchmark_release_acceptance.py b/tests/test_nim_benchmark_release_acceptance.py index 4af082e6a..8f3831b7f 100644 --- a/tests/test_nim_benchmark_release_acceptance.py +++ b/tests/test_nim_benchmark_release_acceptance.py @@ -24,21 +24,24 @@ from contextual_orchestrator.orchestrator import ModelClient +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = str(REPOSITORY_ROOT / "examples" / "nim_task_manifest.json") +EXAMPLE_PRICING_PATH = REPOSITORY_ROOT / "examples" / "nim_pricing_scenario.json" +FAKE_ENDPOINT = "https://nim.example.test/v1" + DECLARED_MAX_WORKFLOW_DEPTH = 5 DECLARED_MAX_OUTPUT_TOKENS = 264 DECLARED_POLICY_TOTAL_TOKEN_BUDGET = ( DECLARED_MAX_WORKFLOW_DEPTH * DECLARED_MAX_OUTPUT_TOKENS ) DECLARED_RUN_KWARGS = { + "resample_count": 2000, + "confidence_level": 0.95, + "comparison_pairs": (("conduct_bounded", "route_once"),), "max_output_tokens": DECLARED_MAX_OUTPUT_TOKENS, "max_workflow_depth": DECLARED_MAX_WORKFLOW_DEPTH, } -REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -TASK_MANIFEST_PATH = str(REPOSITORY_ROOT / "examples" / "nim_task_manifest.json") -EXAMPLE_PRICING_PATH = REPOSITORY_ROOT / "examples" / "nim_pricing_scenario.json" -FAKE_ENDPOINT = "https://nim.example.test/v1" - @pytest.fixture(autouse=True) def _isolated_credentials() -> None: diff --git a/tests/test_nim_benchmark_workflow_contract.py b/tests/test_nim_benchmark_workflow_contract.py index cca138532..a93cca497 100644 --- a/tests/test_nim_benchmark_workflow_contract.py +++ b/tests/test_nim_benchmark_workflow_contract.py @@ -88,6 +88,9 @@ def test_scheduled_live_budget_covers_the_reviewed_current_catalog_scale() -> No assert 'echo "max_requests=2000" >> "$GITHUB_OUTPUT"' in workflow assert 'echo "max_requests=300" >> "$GITHUB_OUTPUT"' not in workflow + assert "--bootstrap-resample-count 2000" in workflow + assert "--confidence-level 0.95" in workflow + assert "--comparison-pair conduct_bounded,route_once" in workflow assert "--max-workflow-depth 5" in workflow assert "--max-output-tokens 264" in workflow diff --git a/tests/test_psychometric_benchmark_boundaries.py b/tests/test_psychometric_benchmark_boundaries.py index 22f3c74da..a7f2bbc92 100644 --- a/tests/test_psychometric_benchmark_boundaries.py +++ b/tests/test_psychometric_benchmark_boundaries.py @@ -11,6 +11,12 @@ from scripts import benchmark_psychometric_heldout as heldout from scripts import benchmark_psychometric_routing as routing +SMALL_HELDOUT_BOOTSTRAP = { + "resample_count": 20, + "confidence_level": 0.95, + "seed": 568, +} + def test_sequential_drift_seeded_reference_is_unchanged(): """Default complete detections preserve the established threshold and delays.""" @@ -83,7 +89,6 @@ def test_selective_coverage_uses_actual_odd_sized_strata( ): """An oracle resolving every row must cover each unequal stratum exactly once.""" monkeypatch.setattr(heldout, "ADAPTIVE_CALIBRATION_CANDIDATES", candidate_count) - monkeypatch.setattr(heldout, "BOOTSTRAP_SAMPLES", 20) monkeypatch.setattr(heldout, "SELECTIVE_CLASSIFICATION_REPLICATIONS", 2) monkeypatch.setattr(heldout, "SELECTIVE_CLASSIFICATION_MAX_ERROR_UPPER", 1.0) candidate_index = -1 @@ -107,9 +112,9 @@ def oracle(_bundle, responses, **_kwargs): monkeypatch.setattr(heldout.fast_mlsirm, "cat_next_item", oracle) if unresolved_scope: with pytest.raises(ValueError, match="no confidence-resolved candidates"): - heldout._validate_adaptive_candidate_calibration() + heldout._validate_adaptive_candidate_calibration(**SMALL_HELDOUT_BOOTSTRAP) return - report = heldout._validate_adaptive_candidate_calibration() + report = heldout._validate_adaptive_candidate_calibration(**SMALL_HELDOUT_BOOTSTRAP) screen = report["classification_stopping"]["risk_coverage_screen"] for point in (screen["heldout"], screen["heldout_baseline"]): for metric in ( @@ -127,13 +132,12 @@ def test_calibration_rejects_empty_generated_strata(monkeypatch, candidate_count """A missing denominator fails explicitly before the native calibration call.""" monkeypatch.setattr(heldout, "ADAPTIVE_CALIBRATION_CANDIDATES", candidate_count) with pytest.raises(ValueError, match="non-empty near-cut and directional strata"): - heldout._validate_adaptive_candidate_calibration() + heldout._validate_adaptive_candidate_calibration(**SMALL_HELDOUT_BOOTSTRAP) def test_calibration_rejects_undefined_resolution_summary(monkeypatch): """No resolved decisions cannot be reported as zero error or divide by zero.""" monkeypatch.setattr(heldout, "ADAPTIVE_CALIBRATION_CANDIDATES", 9) - monkeypatch.setattr(heldout, "BOOTSTRAP_SAMPLES", 20) monkeypatch.setattr( heldout.fast_mlsirm, "cat_next_item", @@ -144,7 +148,7 @@ def test_calibration_rejects_undefined_resolution_summary(monkeypatch): }, ) with pytest.raises(ValueError, match="no confidence-resolved candidates"): - heldout._validate_adaptive_candidate_calibration() + heldout._validate_adaptive_candidate_calibration(**SMALL_HELDOUT_BOOTSTRAP) @pytest.mark.parametrize("sample_count", [1, 21, 101]) diff --git a/tests/test_psychometric_routing.py b/tests/test_psychometric_routing.py index 493e5044e..8d22a09fb 100644 --- a/tests/test_psychometric_routing.py +++ b/tests/test_psychometric_routing.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import math import numpy as np from dataclasses import replace @@ -58,10 +59,17 @@ def test_expected_brier_includes_bernoulli_outcome_variance() -> None: assert _expected_brier(1.0, 0.5) == 0.5 +DECLARED_HELDOUT_BOOTSTRAP = { + "resample_count": heldout_benchmark.DECLARED_BOOTSTRAP_RESAMPLE_COUNT, + "confidence_level": heldout_benchmark.DECLARED_BOOTSTRAP_CONFIDENCE_LEVEL, + "seed": heldout_benchmark.DECLARED_BOOTSTRAP_SEED, +} + + def test_paired_bootstrap_interval_uses_within_context_differences() -> None: """Keep a constant within-context delta constant in every bootstrap replicate.""" assert _paired_bootstrap_mean_ci( - [0.1, 0.2, 0.3], [0.2, 0.3, 0.4] + [0.1, 0.2, 0.3], [0.2, 0.3, 0.4], **DECLARED_HELDOUT_BOOTSTRAP ) == pytest.approx([-0.1, -0.1]) @@ -71,6 +79,46 @@ def test_paired_bootstrap_interval_rejects_unpaired_samples() -> None: _paired_bootstrap_mean_ci([0.1], []) +def test_paired_bootstrap_interval_requires_declared_coverage() -> None: + """Held-out intervals cannot invent a 2,000-sample 95% default.""" + pairs = ([0.1, 0.2, 0.3], [0.2, 0.3, 0.4]) + parameters = inspect.signature(_paired_bootstrap_mean_ci).parameters + assert parameters["resample_count"].default is None + assert parameters["confidence_level"].default is None + assert parameters["seed"].default is None + with pytest.raises(ValueError, match="resample_count"): + _paired_bootstrap_mean_ci(*pairs) + with pytest.raises(ValueError, match="confidence_level"): + _paired_bootstrap_mean_ci(*pairs, resample_count=20, seed=568) + with pytest.raises(ValueError, match="seed"): + _paired_bootstrap_mean_ci(*pairs, resample_count=20, confidence_level=0.95) + with pytest.raises(ValueError, match="resample_count"): + _paired_bootstrap_mean_ci(*pairs, resample_count=True, confidence_level=0.95, seed=1) + with pytest.raises(ValueError, match="confidence_level"): + _paired_bootstrap_mean_ci(*pairs, resample_count=20, confidence_level=1.0, seed=1) + with pytest.raises(ValueError, match="cannot be represented"): + _paired_bootstrap_mean_ci(*pairs, resample_count=1, confidence_level=0.95, seed=1) + assert _paired_bootstrap_mean_ci( + *pairs, resample_count=20, confidence_level=0.95, seed=568 + ) == pytest.approx([-0.1, -0.1]) + + +def test_heldout_run_benchmark_requires_declared_bootstrap() -> None: + """The harness entry cannot restore hidden bootstrap constants.""" + parameters = inspect.signature(heldout_benchmark.run_benchmark).parameters + assert parameters["resample_count"].default is None + assert parameters["confidence_level"].default is None + assert parameters["seed"].default is None + calibration = inspect.signature( + heldout_benchmark._validate_adaptive_candidate_calibration + ).parameters + assert calibration["resample_count"].default is None + with pytest.raises(ValueError, match="resample_count"): + heldout_benchmark.run_benchmark() + with pytest.raises(ValueError, match="resample_count"): + heldout_benchmark._validate_adaptive_candidate_calibration() + + def test_heldout_report_pairs_every_delta_with_its_interval(monkeypatch) -> None: """Pin full-size synthetic diagnostics and retain unexecuted production gates. @@ -88,7 +136,7 @@ def test_heldout_report_pairs_every_delta_with_its_interval(monkeypatch) -> None ), ) - report = heldout_benchmark.run_benchmark() + report = heldout_benchmark.run_benchmark(**DECLARED_HELDOUT_BOOTSTRAP) assert ( report["latency_repetitions_per_context"]