diff --git a/ai/tests/test_tiny_ai_cross_device_parity_gate.py b/ai/tests/test_tiny_ai_cross_device_parity_gate.py index 77fc43ae3f..468f331dfd 100644 --- a/ai/tests/test_tiny_ai_cross_device_parity_gate.py +++ b/ai/tests/test_tiny_ai_cross_device_parity_gate.py @@ -10,6 +10,9 @@ 4. Target provider silent fallback / unbound detection 5. End-to-end CLI execution with real repository models and fixtures 6. Output artifact schema (JSON and Markdown) +7. The CUDA use_tf32=0 provider-option pin on every session +8. The transnet_v2 provider-gap probe (named ScatterND cause accepted, other + failures and non-CUDA targets rejected) """ from __future__ import annotations @@ -17,6 +20,7 @@ import json import sys from pathlib import Path +from typing import Any from unittest.mock import MagicMock, patch import numpy as np @@ -24,22 +28,55 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from scripts.ci.tiny_ai_cross_device_parity_gate import ( + CUDA_TF32_PROVIDER_OPTIONS, DEFAULT_FP16_TOLERANCE, DEFAULT_FP32_TOLERANCE, + TRANSNET_V2_SCATTERND_GAP, + CaseResult, + ParityGateReport, + create_session, evaluate_case, + evaluate_provider_gap, load_fp16_inputs, load_fp32_inputs, main, render_json, render_markdown, run_parity_gate, + session_provider_options, ) -EXPECTED_CASES = 2 +EXPECTED_CASES = 3 REPO_ROOT = Path(__file__).resolve().parents[2] FP32_MODEL = REPO_ROOT / "model" / "tiny" / "vmaf_tiny_v2.onnx" FP16_MODEL = REPO_ROOT / "model" / "tiny" / "smoke_fp16_v0.onnx" +TRANSNET_MODEL = REPO_ROOT / "model" / "tiny" / "transnet_v2.onnx" FEATURES_PARQUET = REPO_ROOT / "ai" / "testdata" / "bisect" / "features.parquet" +# The measured ORT 1.31.1 CUDA failure, wrapped in the node context ORT prints. +SCATTERND_ERROR = ( + "[ONNXRuntimeError] : 1 : FAIL : Non-zero status code returned while running " + "ScatterND node. Name:'fork_segmentsum_scatter' Status Message: element_type 6 " + "(int32) not implemented for ScatterND operator." +) + + +def mock_session( + providers: list[str], + input_meta: dict[str, Any] | None = None, +) -> MagicMock: + """A mocked ORT session reporting the given bound providers. + + The metadata contract the gate reads is get_providers() and get_inputs(); + input_meta overrides the mocked input's kwargs (name, shape, type) when a + test needs a specific input signature. A session built without input_meta + leaves get_inputs at its MagicMock default, for the paths that never reach + it. + """ + session = MagicMock() + session.get_providers.return_value = list(providers) + if input_meta is not None: + session.get_inputs.return_value = [MagicMock(**input_meta)] + return session def test_missing_target_provider_fails_closed() -> None: @@ -88,13 +125,10 @@ def test_missing_reference_provider_fails_closed() -> None: def test_target_provider_unbound_detection(tmp_path: Path) -> None: """If target session falls back and does not bind target provider, fail closed.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"}) - mock_target_sess = MagicMock() # Simulate ORT fallback: requested CUDA, but session bound CPU only - mock_target_sess.get_providers.return_value = ["CPUExecutionProvider"] + mock_target_sess = mock_session(["CPUExecutionProvider"]) with patch( "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", @@ -116,13 +150,8 @@ def test_target_provider_unbound_detection(tmp_path: Path) -> None: def test_numerical_mismatch_fp32_fails_closed() -> None: """Error exceeding FP32 tolerance (1e-4) must fail closed.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")] - - mock_target_sess = MagicMock() - mock_target_sess.get_providers.return_value = ["TargetProvider"] - mock_target_sess.get_inputs.return_value = [MagicMock(name="features")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"}) + mock_target_sess = mock_session(["TargetProvider"], {"name": "features"}) ref_output = np.array([50.0, 60.0, 70.0], dtype=np.float32) # Delta of 5e-4 exceeds tolerance of 1e-4 @@ -155,13 +184,8 @@ def test_numerical_mismatch_fp32_fails_closed() -> None: def test_numerical_mismatch_fp16_fails_closed() -> None: """Error exceeding FP16 tolerance (1e-2) must fail closed.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="x")] - - mock_target_sess = MagicMock() - mock_target_sess.get_providers.return_value = ["TargetProvider"] - mock_target_sess.get_inputs.return_value = [MagicMock(name="x")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "x"}) + mock_target_sess = mock_session(["TargetProvider"], {"name": "x"}) ref_output = np.array([[[[0.5, 0.5], [0.5, 0.5]]]], dtype=np.float16) # Delta of 0.05 exceeds tolerance of 0.01 @@ -193,13 +217,8 @@ def test_numerical_mismatch_fp16_fails_closed() -> None: def test_numerical_match_within_tolerance_passes() -> None: """Error within tolerance must report status OK and ok=True.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")] - - mock_target_sess = MagicMock() - mock_target_sess.get_providers.return_value = ["TargetProvider"] - mock_target_sess.get_inputs.return_value = [MagicMock(name="features")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"}) + mock_target_sess = mock_session(["TargetProvider"], {"name": "features"}) ref_output = np.array([50.0, 60.0, 70.0], dtype=np.float32) # Delta of 5e-5 is within tolerance of 1e-4 @@ -229,6 +248,164 @@ def test_numerical_match_within_tolerance_passes() -> None: assert case.max_abs_error <= DEFAULT_FP32_TOLERANCE +def test_session_provider_options_pin_strict_fp32_on_cuda_only() -> None: + """The CUDA EP carries use_tf32=0; every other provider carries no options.""" + assert CUDA_TF32_PROVIDER_OPTIONS == {"use_tf32": 0} + assert session_provider_options("CUDAExecutionProvider") == {"use_tf32": 0} + assert session_provider_options("CPUExecutionProvider") == {} + assert session_provider_options("OpenVINOExecutionProvider") == {} + + +def test_create_session_wires_the_tf32_pin_into_every_provider_list() -> None: + """The session builder passes the pin aligned with the CUDA provider entry.""" + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.ort.InferenceSession", + return_value=MagicMock(), + ) as factory: + create_session(FP32_MODEL, "CUDAExecutionProvider") + cuda_kwargs = factory.call_args.kwargs + create_session(FP32_MODEL, "CPUExecutionProvider") + cpu_kwargs = factory.call_args.kwargs + assert cuda_kwargs["providers"] == ["CUDAExecutionProvider"] + assert cuda_kwargs["provider_options"] == [{"use_tf32": 0}] + assert cpu_kwargs["providers"] == ["CPUExecutionProvider"] + assert cpu_kwargs["provider_options"] == [{}] + + +def test_transnet_v2_named_scatternd_failure_is_the_expected_gap() -> None: + """The measured ScatterND int32 failure passes as PROVIDER_GAP on CUDA.""" + assert TRANSNET_V2_SCATTERND_GAP in SCATTERND_ERROR + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", + side_effect=RuntimeError(SCATTERND_ERROR), + ): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert case.ok + assert case.status == "PROVIDER_GAP" + assert "ScatterND" in case.note + + +def test_transnet_v2_other_failure_fails_closed() -> None: + """Any other CUDA failure of the probe still fails the gate.""" + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", + side_effect=RuntimeError("CUDA failure 2: out of memory"), + ): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_named_cause_is_not_accepted_off_cuda() -> None: + """The named-cause acceptance is bound to the CUDA EP; the CPU leg is unaffected.""" + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", + side_effect=RuntimeError(SCATTERND_ERROR), + ): + case = evaluate_provider_gap(FP32_MODEL, "CPUExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_successful_execute_reports_the_gap_closed() -> None: + """A gap-listed model that executes on the target is fine: reported, never a failure.""" + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"}, + ) + bound.run.return_value = [np.zeros((1, 100), dtype=np.float32)] + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert case.ok + assert case.status == "OK" + assert "gap closed" in case.note + + +def test_transnet_v2_run_time_scatternd_failure_is_the_expected_gap() -> None: + """The gap is a Run-time kernel failure: a bound session that fails Run matches.""" + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"}, + ) + bound.run.side_effect = RuntimeError(SCATTERND_ERROR) + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert case.ok + assert case.status == "PROVIDER_GAP" + assert "ScatterND" in case.note + + +def test_transnet_v2_run_time_other_failure_fails_closed() -> None: + """A bound session whose Run fails for any other reason still fails the gate.""" + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"}, + ) + bound.run.side_effect = RuntimeError("Non-zero status code returned while running Conv node") + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_unfeedable_input_type_fails_closed() -> None: + """A probe input the execute probe cannot feed is an error, not a gap.""" + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "x", "shape": [1, 4], "type": "tensor(int64)"}, + ) + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_silent_fallback_fails_closed() -> None: + """The probe session falling back to the CPU fails closed, as elsewhere.""" + fallback = mock_session(["CPUExecutionProvider"]) + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=fallback): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "PROVIDER_UNBOUND" + + +def test_transnet_v2_missing_model_fails(tmp_path: Path) -> None: + """A missing transnet_v2 file is an error, not an expected gap.""" + case = evaluate_provider_gap(tmp_path / "absent.onnx", "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_markdown_surfaces_tf32_pin_and_expected_gap() -> None: + """The printed summary names the TF32 mode and the per-model provider gaps.""" + report = ParityGateReport( + reference_provider="CPUExecutionProvider", + target_provider="CUDAExecutionProvider", + target_available=True, + ok=True, + status="PASS", + cases=[ + CaseResult( + name="transnet_v2 provider gap", + model_path=str(TRANSNET_MODEL), + precision="execute", + tolerance=0.0, + max_abs_error=0.0, + mean_abs_error=0.0, + ok=True, + status="PROVIDER_GAP", + note="expected gap", + ) + ], + tf32_pinned=True, + ) + md_text = render_markdown(report) + assert "use_tf32=0 pinned on every CUDAExecutionProvider session" in md_text + assert "Expected provider gaps**: transnet_v2 provider gap" in md_text + payload = render_json(report) + assert payload["tf32_pinned"] is True + + def test_missing_model_file_fails(tmp_path: Path) -> None: """Missing model path must return ERROR.""" non_existent = tmp_path / "does_not_exist.onnx" @@ -252,6 +429,7 @@ def test_real_models_cpu_self_parity() -> None: fp32_model=FP32_MODEL, fp16_model=FP16_MODEL, features_path=FEATURES_PARQUET, + transnet_model=TRANSNET_MODEL, ref_provider="CPUExecutionProvider", target_provider="CPUExecutionProvider", fp32_tol=DEFAULT_FP32_TOLERANCE, @@ -264,6 +442,11 @@ def test_real_models_cpu_self_parity() -> None: assert c.ok assert c.status == "OK" assert c.max_abs_error == 0.0 + # The transnet_v2 leg is a bind probe: on the CPU EP it simply binds, and the + # named ScatterND gap logic never applies to a non-CUDA target. + probe = report.cases[-1] + assert probe.name == "transnet_v2 provider gap" + assert "CPUExecutionProvider" in probe.note def test_load_fp32_inputs_real_parquet() -> None: @@ -305,6 +488,7 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None: fp32_model=FP32_MODEL, fp16_model=FP16_MODEL, features_path=FEATURES_PARQUET, + transnet_model=TRANSNET_MODEL, ref_provider="CPUExecutionProvider", target_provider="CPUExecutionProvider", ) @@ -312,6 +496,7 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None: assert json_dict["schema_version"] == 1 assert json_dict["ok"] is True assert json_dict["status"] == "PASS" + assert json_dict["tf32_pinned"] is False assert len(json_dict["cases"]) == EXPECTED_CASES md_text = render_markdown(report) @@ -321,6 +506,7 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None: ) assert "FP32 vmaf_tiny_v2" in md_text assert "FP16 smoke_fp16_v0" in md_text + assert "not pinned (no CUDA session in this run)" in md_text assert "**PASS**" in md_text diff --git a/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md b/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md new file mode 100644 index 0000000000..81ec38f959 --- /dev/null +++ b/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md @@ -0,0 +1,9 @@ +- **The tiny-AI parity gate pins strict fp32 on the CUDA EP and names the `transnet_v2` provider gap** (#2947). + `scripts/ci/tiny_ai_cross_device_parity_gate.py` creates every `CUDAExecutionProvider` session with the + provider option `use_tf32=0`: onnxruntime-gpu 1.31.1 leaves TF32 enabled by default, which pushed 13/24 fp32 + tiny models past the documented 1e-4 bound (worst 3.9e-02; with the pin, worst 3.8e-05), and the + session-config entry `ep.cuda.use_tf32` is silently ignored by that release. `transnet_v2` runs as an + execute probe — the CUDA leg binds but its `ScatterND` node fails at Run time on 1.31.1 + (`element_type 6 (int32) not implemented for ScatterND operator`) — and that one named failure is + recorded as an expected provider gap instead of failing the gate, while any other `transnet_v2` + failure still fails. The JSON and Markdown summaries name the TF32 mode and the per-model provider gaps. diff --git a/docs/ai/inference.md b/docs/ai/inference.md index 959a9cc794..10de518e8e 100644 --- a/docs/ai/inference.md +++ b/docs/ai/inference.md @@ -512,6 +512,33 @@ device on the same pair, and compare. `/cross-backend-diff` and `scripts/ci/cross_backend_vif_diff.py` cover the feature backends, not the tiny-AI execution providers. +### The CUDA EP on onnxruntime-gpu 1.31.1: TF32 and transnet_v2 + +The two bounds above are strict fp32 bounds, and the gate pins the CUDA EP to +strict fp32 to meet them. Measured across the 26-model tiny registry on two +local GPUs ([#2947](https://github.com/VMAFx/vmafx/issues/2947), comment +6696661349): + +- onnxruntime-gpu 1.31.1 leaves TF32 enabled by default in the CUDA EP. TF32 + GEMM/conv breaks the documented 1e-4 fp32 bound for 13 of the 24 fp32 + models, drifting 6e-04 to 3.9e-02 (worst case about 1e-2, stable across both + GPUs) at CI-like batch sizes. The parity gate therefore creates every CUDA + session with the provider option `use_tf32=0`, which puts all 24 models back + inside the bound (worst 3.8e-05). The session-config entry + `ep.cuda.use_tf32` is silently ignored by 1.31.1 — only the provider option + works. Opting back into TF32 means re-measuring against the band above, not + loosening the 1e-4 bound. +- `transnet_v2` cannot execute on the CUDA EP in 1.31.1: the leg binds, but its + ScatterND node `fork_segmentsum_scatter` raises + `element_type 6 (int32) not implemented for ScatterND operator`, + deterministically on both GPUs. The CPU EP is unaffected. The gate runs the + model's target-provider leg as a zeros-input execute probe and records that + exact failure — whether it surfaces while creating the session or at Run + time — as an expected provider gap (`PROVIDER_GAP`); any other transnet_v2 + failure still fails the gate, and a gap-listed model that executes is + reported as the gap closing. +- fp16 (`smoke_fp16_v0`) and int8 models are exact and unaffected by TF32. + ## History - **2026-05-02, ADR-0241.** `vmaf_tiny_v3` (`mlp_medium`, 6 to 32 to 16 to 1, diff --git a/scripts/ci/tiny_ai_cross_device_parity_gate.py b/scripts/ci/tiny_ai_cross_device_parity_gate.py index a0ead289ea..5773b1d701 100755 --- a/scripts/ci/tiny_ai_cross_device_parity_gate.py +++ b/scripts/ci/tiny_ai_cross_device_parity_gate.py @@ -8,6 +8,15 @@ tolerances: - FP32: absolute error <= 1e-4 on model/tiny/vmaf_tiny_v2.onnx - FP16: absolute error <= 1e-2 on model/tiny/smoke_fp16_v0.onnx +- transnet_v2: a zeros-input execute probe against the target provider, not a + numerical case (its CUDA leg cannot execute on ORT 1.31.1, see + TRANSNET_V2_SCATTERND_GAP) + +Every CUDAExecutionProvider session is created with the provider option +use_tf32=0: ORT 1.31.1 leaves TF32 enabled by default in the CUDA EP and TF32 +GEMM/conv breaks the documented 1e-4 fp32 bound, while the session-config entry +ep.cuda.use_tf32 is silently ignored by that release (issue #2947, comment +6696661349). Fail-closed contract: - If a requested target provider is absent from onnxruntime.get_available_providers(), @@ -15,6 +24,8 @@ - If onnxruntime silently falls back to CPU for the target session, the gate detects the provider mismatch and fails closed. - If numerical delta exceeds tolerance, the gate fails closed. +- The transnet_v2 probe fails closed on every failure except the one named CUDA + ScatterND gap recorded in TRANSNET_V2_SCATTERND_GAP. - Generates JSON summary (--json-out) and Markdown report (--md-out). """ @@ -48,6 +59,25 @@ DEFAULT_FP16_TOLERANCE = 1e-2 DEFAULT_REF_PROVIDER = "CPUExecutionProvider" DEFAULT_TARGET_PROVIDER = "CUDAExecutionProvider" +# ORT 1.31.1 ships the CUDA EP with TF32 enabled by default; TF32 GEMM/conv +# pushed 13/24 fp32 tiny models past the documented 1e-4 bound (drift +# 6e-04..3.9e-02 at CI-like batch sizes) while use_tf32=0 passed all 24 (worst +# 3.8e-05). The session-config entry ep.cuda.use_tf32 is silently ignored by +# 1.31.1, so the pin must ride the provider option list (issue #2947, comment +# 6696661349). +CUDA_TF32_PROVIDER_OPTIONS: dict[str, int] = {"use_tf32": 0} +# transnet_v2 binds the CUDA EP in ORT 1.31.1 but cannot execute there: the +# ScatterND node fork_segmentsum_scatter fails at Run time with this status +# message (element type 6 is ONNX int32), deterministically on both local +# measurement GPUs; the CPU EP is unaffected. The gate runs the model's +# target-provider leg as a zeros-input execute probe, records that one named +# failure as the model's expected provider gap, and fails on every other +# transnet_v2 failure (issue #2947, comment 6696661349). +TRANSNET_V2_MODEL_ID = "transnet_v2" +# The measured ORT 1.31.1 CUDA message, byte for byte as the node raises it +# (testdata sweep, both GPUs): dropping the "(int32) " segment made the named +# gap unmatchable against the error it names. +TRANSNET_V2_SCATTERND_GAP = "element_type 6 (int32) not implemented for ScatterND operator" @dataclass @@ -72,6 +102,7 @@ class ParityGateReport: status: str cases: list[CaseResult] = field(default_factory=list) note: str = "" + tf32_pinned: bool = False def load_fp32_inputs(features_path: Path, max_rows: int = 256) -> np.ndarray: @@ -100,12 +131,28 @@ def load_fp16_inputs() -> np.ndarray: return np.array([[[[0.125, 0.25], [0.5, 0.875]]]], dtype=np.float16) +def session_provider_options(provider: str) -> dict[str, int]: + """Provider options for one session of a pinned-provider run. + + The CUDA EP pins strict fp32 (see CUDA_TF32_PROVIDER_OPTIONS); every other + provider runs on its defaults. + """ + if provider == DEFAULT_TARGET_PROVIDER: + return dict(CUDA_TF32_PROVIDER_OPTIONS) + return {} + + def create_session(model_path: Path, provider: str) -> ort.InferenceSession: """Create an ONNX Runtime inference session pinned to a single provider.""" opts = ort.SessionOptions() opts.inter_op_num_threads = 1 opts.intra_op_num_threads = 1 - return ort.InferenceSession(str(model_path), sess_options=opts, providers=[provider]) + return ort.InferenceSession( + str(model_path), + sess_options=opts, + providers=[provider], + provider_options=[session_provider_options(provider)], + ) def run_session_inference( @@ -183,6 +230,68 @@ def fail(status: str, note: str) -> CaseResult: ) +def evaluate_provider_gap(model_path: Path, target_provider: str) -> CaseResult: + """transnet_v2's target-provider leg: a zeros-input execute probe. + + The model binds the CUDA EP in ORT 1.31.1 but cannot execute there (see + TRANSNET_V2_SCATTERND_GAP), so this leg runs one zeros input on the target + provider and judges only that outcome: the one named CUDA failure is the + expected gap and passes as PROVIDER_GAP, a successful run is reported as + the gap closing, and anything else fails the gate. The zeros input is never + compared numerically; this leg measures executability, not parity. + """ + name = f"{TRANSNET_V2_MODEL_ID} provider gap" + + def case(status: str, ok: bool, note: str) -> CaseResult: + return CaseResult( + name=name, + model_path=str(model_path), + precision="execute", + tolerance=0.0, + max_abs_error=0.0, + mean_abs_error=0.0, + ok=ok, + status=status, + note=note, + ) + + if not model_path.is_file(): + return case("ERROR", False, f"Model file not found: {model_path}") + try: + sess = create_session(model_path, target_provider) + bound = sess.get_providers() + if target_provider not in bound: + return case( + "PROVIDER_UNBOUND", + False, + f"Target provider '{target_provider}' failed to bind; bound providers: {bound}", + ) + in_meta = sess.get_inputs()[0] + shape = [d if isinstance(d, int) and d > 0 else 1 for d in in_meta.shape] + dtype = {"tensor(float)": np.float32}.get(in_meta.type) + if dtype is None: + return case("ERROR", False, f"Probe cannot feed input type {in_meta.type}") + sess.run(None, {in_meta.name: np.zeros(shape, dtype=dtype)}) + except Exception as exc: + exc_text = str(exc) + named_gap = ( + target_provider == DEFAULT_TARGET_PROVIDER and TRANSNET_V2_SCATTERND_GAP in exc_text + ) + if named_gap: + return case( + "PROVIDER_GAP", + True, + f"expected gap: {TRANSNET_V2_SCATTERND_GAP} (ORT 1.31.1 CUDA EP)", + ) + return case("ERROR", False, f"Exception during probe inference: {exc}") + return case( + "OK", + True, + f"gap closed: {TRANSNET_V2_MODEL_ID} executes on {target_provider}; " + "numerical parity not compared by this probe", + ) + + def _stop_report( ref: str, target: str, available: bool, status: str, note: str, ok: bool = False ) -> ParityGateReport: @@ -201,6 +310,8 @@ def run_parity_gate( fp32_model: Path, fp16_model: Path, features_path: Path, + *, + transnet_model: Path = Path("model/tiny/transnet_v2.onnx"), ref_provider: str = DEFAULT_REF_PROVIDER, target_provider: str = DEFAULT_TARGET_PROVIDER, fp32_tol: float = DEFAULT_FP32_TOLERANCE, @@ -233,7 +344,11 @@ def run_parity_gate( except (FileNotFoundError, ValueError, RuntimeError) as exc: return _stop_report(ref_provider, target_provider, True, "BAD_FIXTURE", str(exc)) cases = _run_cases( - (fp32_model, fp16_model), fp32_inputs, (fp32_tol, fp16_tol), ref_provider, target_provider + (fp32_model, fp16_model, transnet_model), + fp32_inputs, + (fp32_tol, fp16_tol), + ref_provider, + target_provider, ) all_ok = all(c.ok for c in cases) return ParityGateReport( @@ -243,17 +358,18 @@ def run_parity_gate( ok=all_ok, status="PASS" if all_ok else "FAIL", cases=cases, + tf32_pinned=DEFAULT_TARGET_PROVIDER in (ref_provider, target_provider), ) def _run_cases( - models: tuple[Path, Path], + models: tuple[Path, Path, Path], fp32_inputs: np.ndarray, tolerances: tuple[float, float], ref_provider: str, target_provider: str, ) -> list[CaseResult]: - """The FP32 and FP16 cases, in that order.""" + """The FP32 and FP16 numerical cases, then the transnet_v2 execute probe.""" return [ evaluate_case( "FP32 vmaf_tiny_v2", @@ -273,21 +389,31 @@ def _run_cases( ref_provider, target_provider, ), + evaluate_provider_gap(models[2], target_provider), ] def render_markdown(report: ParityGateReport) -> str: """Render human-readable Markdown summary table.""" + tf32_mode = ( + "use_tf32=0 pinned on every CUDAExecutionProvider session" + if report.tf32_pinned + else "not pinned (no CUDA session in this run)" + ) lines = [ "# Tiny AI Cross-Device Parity Gate Report (T-TINY-AI-CROSS-DEVICE-PARITY-UNGATED-2026-09-25)", "", f"- **Reference Provider**: `{report.reference_provider}`", f"- **Target Provider**: `{report.target_provider}`", f"- **Target Available**: `{report.target_available}`", + f"- **TF32**: {tf32_mode}", f"- **Verdict**: **{report.status}**", ] if report.note: lines.append(f"- **Note**: {report.note}") + gaps = [c.name for c in report.cases if c.status == "PROVIDER_GAP"] + if gaps: + lines.append(f"- **Expected provider gaps**: {', '.join(gaps)}") lines.extend( [ "", @@ -319,6 +445,11 @@ def render_json(report: ParityGateReport) -> dict[str, Any]: "ai/testdata/bisect/features.parquet", "canonical features parquet fixture", ), + ( + "--transnet-model", + "model/tiny/transnet_v2.onnx", + "transnet_v2 ONNX model for the provider-gap probe", + ), ) _TOLERANCE_OPTIONS = ( ("--fp32-tolerance", DEFAULT_FP32_TOLERANCE, "FP32"), @@ -363,6 +494,7 @@ def main(argv: list[str] | None = None) -> int: fp32_model=args.fp32_model, fp16_model=args.fp16_model, features_path=args.fp32_features, + transnet_model=args.transnet_model, ref_provider=args.reference_provider, target_provider=args.target_provider, fp32_tol=args.fp32_tolerance,