fix(ai): pin strict fp32 on the CUDA EP and record the transnet_v2 provider gap

ai/tests/test_tiny_ai_cross_device_parity_gate.py

@@ -10,36 +10,73 @@
 4. Target provider silent fallback / unbound detection
 5. End-to-end CLI execution with real repository models and fixtures
 6. Output artifact schema (JSON and Markdown)
+7. The CUDA use_tf32=0 provider-option pin on every session
+8. The transnet_v2 provider-gap probe (named ScatterND cause accepted, other
+   failures and non-CUDA targets rejected)
 """
 
 from __future__ import annotations
 
 import json
 import sys
 from pathlib import Path
+from typing import Any
 from unittest.mock import MagicMock, patch
 
 import numpy as np
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
 
 from scripts.ci.tiny_ai_cross_device_parity_gate import (
+    CUDA_TF32_PROVIDER_OPTIONS,
     DEFAULT_FP16_TOLERANCE,
     DEFAULT_FP32_TOLERANCE,
+    TRANSNET_V2_SCATTERND_GAP,
+    CaseResult,
+    ParityGateReport,
+    create_session,
     evaluate_case,
+    evaluate_provider_gap,
     load_fp16_inputs,
     load_fp32_inputs,
     main,
     render_json,
     render_markdown,
     run_parity_gate,
+    session_provider_options,
 )
 
-EXPECTED_CASES = 2
+EXPECTED_CASES = 3
 REPO_ROOT = Path(__file__).resolve().parents[2]
 FP32_MODEL = REPO_ROOT / "model" / "tiny" / "vmaf_tiny_v2.onnx"
 FP16_MODEL = REPO_ROOT / "model" / "tiny" / "smoke_fp16_v0.onnx"
+TRANSNET_MODEL = REPO_ROOT / "model" / "tiny" / "transnet_v2.onnx"
 FEATURES_PARQUET = REPO_ROOT / "ai" / "testdata" / "bisect" / "features.parquet"
+# The measured ORT 1.31.1 CUDA failure, wrapped in the node context ORT prints.
+SCATTERND_ERROR = (
+    "[ONNXRuntimeError] : 1 : FAIL : Non-zero status code returned while running "
+    "ScatterND node. Name:'fork_segmentsum_scatter' Status Message: element_type 6 "
+    "(int32) not implemented for ScatterND operator."
+)
+
+
+def mock_session(
+    providers: list[str],
+    input_meta: dict[str, Any] | None = None,
+) -> MagicMock:
+    """A mocked ORT session reporting the given bound providers.
+
+    The metadata contract the gate reads is get_providers() and get_inputs();
+    input_meta overrides the mocked input's kwargs (name, shape, type) when a
+    test needs a specific input signature. A session built without input_meta
+    leaves get_inputs at its MagicMock default, for the paths that never reach
+    it.
+    """
+    session = MagicMock()
+    session.get_providers.return_value = list(providers)
+    if input_meta is not None:
+        session.get_inputs.return_value = [MagicMock(**input_meta)]
+    return session
 
 
 def test_missing_target_provider_fails_closed() -> None:
@@ -88,13 +125,10 @@ def test_missing_reference_provider_fails_closed() -> None:
 
 def test_target_provider_unbound_detection(tmp_path: Path) -> None:
     """If target session falls back and does not bind target provider, fail closed."""
-    mock_ref_sess = MagicMock()
-    mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"]
-    mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")]
+    mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"})
 
-    mock_target_sess = MagicMock()
     # Simulate ORT fallback: requested CUDA, but session bound CPU only
-    mock_target_sess.get_providers.return_value = ["CPUExecutionProvider"]
+    mock_target_sess = mock_session(["CPUExecutionProvider"])
 
     with patch(
         "scripts.ci.tiny_ai_cross_device_parity_gate.create_session",
@@ -116,13 +150,8 @@ def test_target_provider_unbound_detection(tmp_path: Path) -> None:
 
 def test_numerical_mismatch_fp32_fails_closed() -> None:
     """Error exceeding FP32 tolerance (1e-4) must fail closed."""
-    mock_ref_sess = MagicMock()
-    mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"]
-    mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")]
-
-    mock_target_sess = MagicMock()
-    mock_target_sess.get_providers.return_value = ["TargetProvider"]
-    mock_target_sess.get_inputs.return_value = [MagicMock(name="features")]
+    mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"})
+    mock_target_sess = mock_session(["TargetProvider"], {"name": "features"})
 
     ref_output = np.array([50.0, 60.0, 70.0], dtype=np.float32)
     # Delta of 5e-4 exceeds tolerance of 1e-4
@@ -155,13 +184,8 @@ def test_numerical_mismatch_fp32_fails_closed() -> None:
 
 def test_numerical_mismatch_fp16_fails_closed() -> None:
     """Error exceeding FP16 tolerance (1e-2) must fail closed."""
-    mock_ref_sess = MagicMock()
-    mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"]
-    mock_ref_sess.get_inputs.return_value = [MagicMock(name="x")]
-
-    mock_target_sess = MagicMock()
-    mock_target_sess.get_providers.return_value = ["TargetProvider"]
-    mock_target_sess.get_inputs.return_value = [MagicMock(name="x")]
+    mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "x"})
+    mock_target_sess = mock_session(["TargetProvider"], {"name": "x"})
 
     ref_output = np.array([[[[0.5, 0.5], [0.5, 0.5]]]], dtype=np.float16)
     # Delta of 0.05 exceeds tolerance of 0.01
@@ -193,13 +217,8 @@ def test_numerical_mismatch_fp16_fails_closed() -> None:
 
 def test_numerical_match_within_tolerance_passes() -> None:
     """Error within tolerance must report status OK and ok=True."""
-    mock_ref_sess = MagicMock()
-    mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"]
-    mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")]
-
-    mock_target_sess = MagicMock()
-    mock_target_sess.get_providers.return_value = ["TargetProvider"]
-    mock_target_sess.get_inputs.return_value = [MagicMock(name="features")]
+    mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"})
+    mock_target_sess = mock_session(["TargetProvider"], {"name": "features"})
 
     ref_output = np.array([50.0, 60.0, 70.0], dtype=np.float32)
     # Delta of 5e-5 is within tolerance of 1e-4
@@ -229,6 +248,164 @@ def test_numerical_match_within_tolerance_passes() -> None:
         assert case.max_abs_error <= DEFAULT_FP32_TOLERANCE
 
 
+def test_session_provider_options_pin_strict_fp32_on_cuda_only() -> None:
+    """The CUDA EP carries use_tf32=0; every other provider carries no options."""
+    assert CUDA_TF32_PROVIDER_OPTIONS == {"use_tf32": 0}
+    assert session_provider_options("CUDAExecutionProvider") == {"use_tf32": 0}
+    assert session_provider_options("CPUExecutionProvider") == {}
+    assert session_provider_options("OpenVINOExecutionProvider") == {}
+
+
+def test_create_session_wires_the_tf32_pin_into_every_provider_list() -> None:
+    """The session builder passes the pin aligned with the CUDA provider entry."""
+    with patch(
+        "scripts.ci.tiny_ai_cross_device_parity_gate.ort.InferenceSession",
+        return_value=MagicMock(),
+    ) as factory:
+        create_session(FP32_MODEL, "CUDAExecutionProvider")
+        cuda_kwargs = factory.call_args.kwargs
+        create_session(FP32_MODEL, "CPUExecutionProvider")
+        cpu_kwargs = factory.call_args.kwargs
+    assert cuda_kwargs["providers"] == ["CUDAExecutionProvider"]
+    assert cuda_kwargs["provider_options"] == [{"use_tf32": 0}]
+    assert cpu_kwargs["providers"] == ["CPUExecutionProvider"]
+    assert cpu_kwargs["provider_options"] == [{}]
+
+
+def test_transnet_v2_named_scatternd_failure_is_the_expected_gap() -> None:
+    """The measured ScatterND int32 failure passes as PROVIDER_GAP on CUDA."""
+    assert TRANSNET_V2_SCATTERND_GAP in SCATTERND_ERROR
+    with patch(
+        "scripts.ci.tiny_ai_cross_device_parity_gate.create_session",
+        side_effect=RuntimeError(SCATTERND_ERROR),
+    ):
+        case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider")
+    assert case.ok
+    assert case.status == "PROVIDER_GAP"
+    assert "ScatterND" in case.note
+
+
+def test_transnet_v2_other_failure_fails_closed() -> None:
+    """Any other CUDA failure of the probe still fails the gate."""
+    with patch(
+        "scripts.ci.tiny_ai_cross_device_parity_gate.create_session",
+        side_effect=RuntimeError("CUDA failure 2: out of memory"),
+    ):
+        case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider")
+    assert not case.ok
+    assert case.status == "ERROR"
+
+
+def test_transnet_v2_named_cause_is_not_accepted_off_cuda() -> None:
+    """The named-cause acceptance is bound to the CUDA EP; the CPU leg is unaffected."""
+    with patch(
+        "scripts.ci.tiny_ai_cross_device_parity_gate.create_session",
+        side_effect=RuntimeError(SCATTERND_ERROR),
+    ):
+        case = evaluate_provider_gap(FP32_MODEL, "CPUExecutionProvider")
+    assert not case.ok
+    assert case.status == "ERROR"
+
+
+def test_transnet_v2_successful_execute_reports_the_gap_closed() -> None:
+    """A gap-listed model that executes on the target is fine: reported, never a failure."""
+    bound = mock_session(
+        ["CUDAExecutionProvider"],
+        {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"},
+    )
+    bound.run.return_value = [np.zeros((1, 100), dtype=np.float32)]
+    with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound):
+        case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider")
+    assert case.ok
+    assert case.status == "OK"
+    assert "gap closed" in case.note
+
+
+def test_transnet_v2_run_time_scatternd_failure_is_the_expected_gap() -> None:
+    """The gap is a Run-time kernel failure: a bound session that fails Run matches."""
+    bound = mock_session(
+        ["CUDAExecutionProvider"],
+        {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"},
+    )
+    bound.run.side_effect = RuntimeError(SCATTERND_ERROR)
+    with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound):
+        case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider")
+    assert case.ok
+    assert case.status == "PROVIDER_GAP"
+    assert "ScatterND" in case.note
+
+
+def test_transnet_v2_run_time_other_failure_fails_closed() -> None:
+    """A bound session whose Run fails for any other reason still fails the gate."""
+    bound = mock_session(
+        ["CUDAExecutionProvider"],
+        {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"},
+    )
+    bound.run.side_effect = RuntimeError("Non-zero status code returned while running Conv node")
+    with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound):
+        case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider")
+    assert not case.ok
+    assert case.status == "ERROR"
+
+
+def test_transnet_v2_unfeedable_input_type_fails_closed() -> None:
+    """A probe input the execute probe cannot feed is an error, not a gap."""
+    bound = mock_session(
+        ["CUDAExecutionProvider"],
+        {"name": "x", "shape": [1, 4], "type": "tensor(int64)"},
+    )
+    with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound):
+        case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider")
+    assert not case.ok
+    assert case.status == "ERROR"
+
+
+def test_transnet_v2_silent_fallback_fails_closed() -> None:
+    """The probe session falling back to the CPU fails closed, as elsewhere."""
+    fallback = mock_session(["CPUExecutionProvider"])
+    with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=fallback):
+        case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider")
+    assert not case.ok
+    assert case.status == "PROVIDER_UNBOUND"
+
+
+def test_transnet_v2_missing_model_fails(tmp_path: Path) -> None:
+    """A missing transnet_v2 file is an error, not an expected gap."""
+    case = evaluate_provider_gap(tmp_path / "absent.onnx", "CUDAExecutionProvider")
+    assert not case.ok
+    assert case.status == "ERROR"
+
+
+def test_markdown_surfaces_tf32_pin_and_expected_gap() -> None:
+    """The printed summary names the TF32 mode and the per-model provider gaps."""
+    report = ParityGateReport(
+        reference_provider="CPUExecutionProvider",
+        target_provider="CUDAExecutionProvider",
+        target_available=True,
+        ok=True,
+        status="PASS",
+        cases=[
+            CaseResult(
+                name="transnet_v2 provider gap",
+                model_path=str(TRANSNET_MODEL),
+                precision="execute",
+                tolerance=0.0,
+                max_abs_error=0.0,
+                mean_abs_error=0.0,
+                ok=True,
+                status="PROVIDER_GAP",
+                note="expected gap",
+            )
+        ],
+        tf32_pinned=True,
+    )
+    md_text = render_markdown(report)
+    assert "use_tf32=0 pinned on every CUDAExecutionProvider session" in md_text
+    assert "Expected provider gaps**: transnet_v2 provider gap" in md_text
+    payload = render_json(report)
+    assert payload["tf32_pinned"] is True
+
+
 def test_missing_model_file_fails(tmp_path: Path) -> None:
     """Missing model path must return ERROR."""
     non_existent = tmp_path / "does_not_exist.onnx"
@@ -252,6 +429,7 @@ def test_real_models_cpu_self_parity() -> None:
         fp32_model=FP32_MODEL,
         fp16_model=FP16_MODEL,
         features_path=FEATURES_PARQUET,
+        transnet_model=TRANSNET_MODEL,
         ref_provider="CPUExecutionProvider",
         target_provider="CPUExecutionProvider",
         fp32_tol=DEFAULT_FP32_TOLERANCE,
@@ -264,6 +442,11 @@ def test_real_models_cpu_self_parity() -> None:
         assert c.ok
         assert c.status == "OK"
         assert c.max_abs_error == 0.0
+    # The transnet_v2 leg is a bind probe: on the CPU EP it simply binds, and the
+    # named ScatterND gap logic never applies to a non-CUDA target.
+    probe = report.cases[-1]
+    assert probe.name == "transnet_v2 provider gap"
+    assert "CPUExecutionProvider" in probe.note
 
 
 def test_load_fp32_inputs_real_parquet() -> None:
@@ -305,13 +488,15 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None:
         fp32_model=FP32_MODEL,
         fp16_model=FP16_MODEL,
         features_path=FEATURES_PARQUET,
+        transnet_model=TRANSNET_MODEL,
         ref_provider="CPUExecutionProvider",
         target_provider="CPUExecutionProvider",
     )
     json_dict = render_json(report)
     assert json_dict["schema_version"] == 1
     assert json_dict["ok"] is True
     assert json_dict["status"] == "PASS"
+    assert json_dict["tf32_pinned"] is False
     assert len(json_dict["cases"]) == EXPECTED_CASES
 
     md_text = render_markdown(report)
@@ -321,6 +506,7 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None:
     )
     assert "FP32 vmaf_tiny_v2" in md_text
     assert "FP16 smoke_fp16_v0" in md_text
+    assert "not pinned (no CUDA session in this run)" in md_text
     assert "**PASS**" in md_text
 
 

changelog.d/fixed/tiny-ai-cuda-tf32-pin.md

@@ -0,0 +1,9 @@
+- **The tiny-AI parity gate pins strict fp32 on the CUDA EP and names the `transnet_v2` provider gap** (#2947).
+  `scripts/ci/tiny_ai_cross_device_parity_gate.py` creates every `CUDAExecutionProvider` session with the
+  provider option `use_tf32=0`: onnxruntime-gpu 1.31.1 leaves TF32 enabled by default, which pushed 13/24 fp32
+  tiny models past the documented 1e-4 bound (worst 3.9e-02; with the pin, worst 3.8e-05), and the
+  session-config entry `ep.cuda.use_tf32` is silently ignored by that release. `transnet_v2` runs as an
+  execute probe — the CUDA leg binds but its `ScatterND` node fails at Run time on 1.31.1
+  (`element_type 6 (int32) not implemented for ScatterND operator`) — and that one named failure is
+  recorded as an expected provider gap instead of failing the gate, while any other `transnet_v2`
+  failure still fails. The JSON and Markdown summaries name the TF32 mode and the per-model provider gaps.

docs/ai/inference.md

@@ -512,6 +512,33 @@ device on the same pair, and compare. `/cross-backend-diff` and
 `scripts/ci/cross_backend_vif_diff.py` cover the feature backends, not the
 tiny-AI execution providers.
 
+### The CUDA EP on onnxruntime-gpu 1.31.1: TF32 and transnet_v2
+
+The two bounds above are strict fp32 bounds, and the gate pins the CUDA EP to
+strict fp32 to meet them. Measured across the 26-model tiny registry on two
+local GPUs ([#2947](https://github.com/VMAFx/vmafx/issues/2947), comment
+6696661349):
+
+- onnxruntime-gpu 1.31.1 leaves TF32 enabled by default in the CUDA EP. TF32
+  GEMM/conv breaks the documented 1e-4 fp32 bound for 13 of the 24 fp32
+  models, drifting 6e-04 to 3.9e-02 (worst case about 1e-2, stable across both
+  GPUs) at CI-like batch sizes. The parity gate therefore creates every CUDA
+  session with the provider option `use_tf32=0`, which puts all 24 models back
+  inside the bound (worst 3.8e-05). The session-config entry
+  `ep.cuda.use_tf32` is silently ignored by 1.31.1 — only the provider option
+  works. Opting back into TF32 means re-measuring against the band above, not
+  loosening the 1e-4 bound.
+- `transnet_v2` cannot execute on the CUDA EP in 1.31.1: the leg binds, but its
+  ScatterND node `fork_segmentsum_scatter` raises
+  `element_type 6 (int32) not implemented for ScatterND operator`,
+  deterministically on both GPUs. The CPU EP is unaffected. The gate runs the
+  model's target-provider leg as a zeros-input execute probe and records that
+  exact failure — whether it surfaces while creating the session or at Run
+  time — as an expected provider gap (`PROVIDER_GAP`); any other transnet_v2
+  failure still fails the gate, and a gap-listed model that executes is
+  reported as the gap closing.
+- fp16 (`smoke_fp16_v0`) and int8 models are exact and unaffected by TF32.
+
 ## History
 
 - **2026-05-02, ADR-0241.** `vmaf_tiny_v3` (`mlp_medium`, 6 to 32 to 16 to 1,