From 528845d5e8128a70ee3ef642b6ef8bc7a4ec6165 Mon Sep 17 00:00:00 2001 From: luoye-cpu <3412829159@qq.com> Date: Sat, 7 Nov 2026 22:10:02 +0800 Subject: [PATCH 1/4] fix(ai): pin strict fp32 on the CUDA EP and record the transnet_v2 provider gap Measured on onnxruntime-gpu 1.31.1 across the 26-model tiny registry (#2947, comment 6696661349): the CUDA EP leaves TF32 enabled by default, which pushed 13/24 fp32 models past the documented 1e-4 bound (drift 6e-04..3.9e-02) while the provider option use_tf32=0 passed all 24 (worst 3.8e-05). The session-config entry ep.cuda.use_tf32 is silently ignored by 1.31.1, so the pin rides the provider option list: every CUDAExecutionProvider session the gate creates now carries use_tf32=0. transnet_v2 binds the CUDA EP but fails at Run time: the ScatterND node fork_segmentsum_scatter raises "element_type 6 not implemented for ScatterND operator", deterministically on both measurement GPUs; the CPU EP is unaffected. The gate runs that model's target-provider leg as a zeros-input execute probe and records the one named CUDA failure as PROVIDER_GAP; any other transnet_v2 failure still fails the gate, and a gap-listed model that executes is reported as the gap closing. The JSON and Markdown summaries surface the TF32 mode and the per-model gaps. docs/ai/inference.md gains the measured TF32 band, the use_tf32=0 rationale and the named gap; changelog fragment added. Part of #2947 Signed-off-by: luoye-cpu <3412829159@qq.com> --- .../test_tiny_ai_cross_device_parity_gate.py | 189 +++++++++++++++++- changelog.d/fixed/tiny-ai-cuda-tf32-pin.md | 8 + docs/ai/inference.md | 26 +++ .../ci/tiny_ai_cross_device_parity_gate.py | 135 ++++++++++++- 4 files changed, 353 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed/tiny-ai-cuda-tf32-pin.md diff --git a/ai/tests/test_tiny_ai_cross_device_parity_gate.py b/ai/tests/test_tiny_ai_cross_device_parity_gate.py index 77fc43ae3f..264a6164f9 100644 --- a/ai/tests/test_tiny_ai_cross_device_parity_gate.py +++ b/ai/tests/test_tiny_ai_cross_device_parity_gate.py @@ -10,6 +10,9 @@ 4. Target provider silent fallback / unbound detection 5. End-to-end CLI execution with real repository models and fixtures 6. Output artifact schema (JSON and Markdown) +7. The CUDA use_tf32=0 provider-option pin on every session +8. The transnet_v2 provider-gap probe (named ScatterND cause accepted, other + failures and non-CUDA targets rejected) """ from __future__ import annotations @@ -24,22 +27,36 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from scripts.ci.tiny_ai_cross_device_parity_gate import ( + CUDA_TF32_PROVIDER_OPTIONS, DEFAULT_FP16_TOLERANCE, DEFAULT_FP32_TOLERANCE, + TRANSNET_V2_SCATTERND_GAP, + CaseResult, + ParityGateReport, + create_session, evaluate_case, + evaluate_provider_gap, load_fp16_inputs, load_fp32_inputs, main, render_json, render_markdown, run_parity_gate, + session_provider_options, ) -EXPECTED_CASES = 2 +EXPECTED_CASES = 3 REPO_ROOT = Path(__file__).resolve().parents[2] FP32_MODEL = REPO_ROOT / "model" / "tiny" / "vmaf_tiny_v2.onnx" FP16_MODEL = REPO_ROOT / "model" / "tiny" / "smoke_fp16_v0.onnx" +TRANSNET_MODEL = REPO_ROOT / "model" / "tiny" / "transnet_v2.onnx" FEATURES_PARQUET = REPO_ROOT / "ai" / "testdata" / "bisect" / "features.parquet" +# The measured ORT 1.31.1 CUDA failure, wrapped in the node context ORT prints. +SCATTERND_ERROR = ( + "[ONNXRuntimeError] : 1 : FAIL : Non-zero status code returned while running " + "ScatterND node. Name:'fork_segmentsum_scatter' Status Message: element_type 6 " + "not implemented for ScatterND operator." +) def test_missing_target_provider_fails_closed() -> None: @@ -229,6 +246,167 @@ def test_numerical_match_within_tolerance_passes() -> None: assert case.max_abs_error <= DEFAULT_FP32_TOLERANCE +def test_session_provider_options_pin_strict_fp32_on_cuda_only() -> None: + """The CUDA EP carries use_tf32=0; every other provider carries no options.""" + assert CUDA_TF32_PROVIDER_OPTIONS == {"use_tf32": 0} + assert session_provider_options("CUDAExecutionProvider") == {"use_tf32": 0} + assert session_provider_options("CPUExecutionProvider") == {} + assert session_provider_options("OpenVINOExecutionProvider") == {} + + +def test_create_session_wires_the_tf32_pin_into_every_provider_list() -> None: + """The session builder passes the pin aligned with the CUDA provider entry.""" + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.ort.InferenceSession", + return_value=MagicMock(), + ) as factory: + create_session(FP32_MODEL, "CUDAExecutionProvider") + cuda_kwargs = factory.call_args.kwargs + create_session(FP32_MODEL, "CPUExecutionProvider") + cpu_kwargs = factory.call_args.kwargs + assert cuda_kwargs["providers"] == ["CUDAExecutionProvider"] + assert cuda_kwargs["provider_options"] == [{"use_tf32": 0}] + assert cpu_kwargs["providers"] == ["CPUExecutionProvider"] + assert cpu_kwargs["provider_options"] == [{}] + + +def test_transnet_v2_named_scatternd_failure_is_the_expected_gap() -> None: + """The measured ScatterND int32 failure passes as PROVIDER_GAP on CUDA.""" + assert TRANSNET_V2_SCATTERND_GAP in SCATTERND_ERROR + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", + side_effect=RuntimeError(SCATTERND_ERROR), + ): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert case.ok + assert case.status == "PROVIDER_GAP" + assert "ScatterND" in case.note + + +def test_transnet_v2_other_failure_fails_closed() -> None: + """Any other CUDA failure of the probe still fails the gate.""" + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", + side_effect=RuntimeError("CUDA failure 2: out of memory"), + ): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_named_cause_is_not_accepted_off_cuda() -> None: + """The named-cause acceptance is bound to the CUDA EP; the CPU leg is unaffected.""" + with patch( + "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", + side_effect=RuntimeError(SCATTERND_ERROR), + ): + case = evaluate_provider_gap(FP32_MODEL, "CPUExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_successful_execute_reports_the_gap_closed() -> None: + """A gap-listed model that executes on the target is fine: reported, never a failure.""" + bound = MagicMock() + bound.get_providers.return_value = ["CUDAExecutionProvider"] + bound.get_inputs.return_value = [ + MagicMock(name="frames", shape=[1, 100, 3, 27, 48], type="tensor(float)") + ] + bound.run.return_value = [np.zeros((1, 100), dtype=np.float32)] + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert case.ok + assert case.status == "OK" + assert "gap closed" in case.note + + +def test_transnet_v2_run_time_scatternd_failure_is_the_expected_gap() -> None: + """The gap is a Run-time kernel failure: a bound session that fails Run matches.""" + bound = MagicMock() + bound.get_providers.return_value = ["CUDAExecutionProvider"] + bound.get_inputs.return_value = [ + MagicMock(name="frames", shape=[1, 100, 3, 27, 48], type="tensor(float)") + ] + bound.run.side_effect = RuntimeError(SCATTERND_ERROR) + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert case.ok + assert case.status == "PROVIDER_GAP" + assert "ScatterND" in case.note + + +def test_transnet_v2_run_time_other_failure_fails_closed() -> None: + """A bound session whose Run fails for any other reason still fails the gate.""" + bound = MagicMock() + bound.get_providers.return_value = ["CUDAExecutionProvider"] + bound.get_inputs.return_value = [ + MagicMock(name="frames", shape=[1, 100, 3, 27, 48], type="tensor(float)") + ] + bound.run.side_effect = RuntimeError("Non-zero status code returned while running Conv node") + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_unfeedable_input_type_fails_closed() -> None: + """A probe input the execute probe cannot feed is an error, not a gap.""" + bound = MagicMock() + bound.get_providers.return_value = ["CUDAExecutionProvider"] + bound.get_inputs.return_value = [MagicMock(name="x", shape=[1, 4], type="tensor(int64)")] + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_transnet_v2_silent_fallback_fails_closed() -> None: + """The probe session falling back to the CPU fails closed, as elsewhere.""" + fallback = MagicMock() + fallback.get_providers.return_value = ["CPUExecutionProvider"] + with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=fallback): + case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") + assert not case.ok + assert case.status == "PROVIDER_UNBOUND" + + +def test_transnet_v2_missing_model_fails(tmp_path: Path) -> None: + """A missing transnet_v2 file is an error, not an expected gap.""" + case = evaluate_provider_gap(tmp_path / "absent.onnx", "CUDAExecutionProvider") + assert not case.ok + assert case.status == "ERROR" + + +def test_markdown_surfaces_tf32_pin_and_expected_gap() -> None: + """The printed summary names the TF32 mode and the per-model provider gaps.""" + report = ParityGateReport( + reference_provider="CPUExecutionProvider", + target_provider="CUDAExecutionProvider", + target_available=True, + ok=True, + status="PASS", + cases=[ + CaseResult( + name="transnet_v2 provider gap", + model_path=str(TRANSNET_MODEL), + precision="execute", + tolerance=0.0, + max_abs_error=0.0, + mean_abs_error=0.0, + ok=True, + status="PROVIDER_GAP", + note="expected gap", + ) + ], + tf32_pinned=True, + ) + md_text = render_markdown(report) + assert "use_tf32=0 pinned on every CUDAExecutionProvider session" in md_text + assert "Expected provider gaps**: transnet_v2 provider gap" in md_text + payload = render_json(report) + assert payload["tf32_pinned"] is True + + def test_missing_model_file_fails(tmp_path: Path) -> None: """Missing model path must return ERROR.""" non_existent = tmp_path / "does_not_exist.onnx" @@ -252,6 +430,7 @@ def test_real_models_cpu_self_parity() -> None: fp32_model=FP32_MODEL, fp16_model=FP16_MODEL, features_path=FEATURES_PARQUET, + transnet_model=TRANSNET_MODEL, ref_provider="CPUExecutionProvider", target_provider="CPUExecutionProvider", fp32_tol=DEFAULT_FP32_TOLERANCE, @@ -264,6 +443,11 @@ def test_real_models_cpu_self_parity() -> None: assert c.ok assert c.status == "OK" assert c.max_abs_error == 0.0 + # The transnet_v2 leg is a bind probe: on the CPU EP it simply binds, and the + # named ScatterND gap logic never applies to a non-CUDA target. + probe = report.cases[-1] + assert probe.name == "transnet_v2 provider gap" + assert "CPUExecutionProvider" in probe.note def test_load_fp32_inputs_real_parquet() -> None: @@ -305,6 +489,7 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None: fp32_model=FP32_MODEL, fp16_model=FP16_MODEL, features_path=FEATURES_PARQUET, + transnet_model=TRANSNET_MODEL, ref_provider="CPUExecutionProvider", target_provider="CPUExecutionProvider", ) @@ -312,6 +497,7 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None: assert json_dict["schema_version"] == 1 assert json_dict["ok"] is True assert json_dict["status"] == "PASS" + assert json_dict["tf32_pinned"] is False assert len(json_dict["cases"]) == EXPECTED_CASES md_text = render_markdown(report) @@ -321,6 +507,7 @@ def test_report_serialization_and_markdown(tmp_path: Path) -> None: ) assert "FP32 vmaf_tiny_v2" in md_text assert "FP16 smoke_fp16_v0" in md_text + assert "not pinned (no CUDA session in this run)" in md_text assert "**PASS**" in md_text diff --git a/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md b/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md new file mode 100644 index 0000000000..14486e1c54 --- /dev/null +++ b/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md @@ -0,0 +1,8 @@ +- **The tiny-AI parity gate pins strict fp32 on the CUDA EP and names the `transnet_v2` provider gap** (#2947). + `scripts/ci/tiny_ai_cross_device_parity_gate.py` creates every `CUDAExecutionProvider` session with the + provider option `use_tf32=0`: onnxruntime-gpu 1.31.1 leaves TF32 enabled by default, which pushed 13/24 fp32 + tiny models past the documented 1e-4 bound (worst 3.9e-02; with the pin, worst 3.8e-05), and the + session-config entry `ep.cuda.use_tf32` is silently ignored by that release. `transnet_v2` runs as a bind + probe: its CUDA leg cannot open on 1.31.1 (`element_type 6 (int32) not implemented for ScatterND operator`) + and is recorded as an expected provider gap instead of failing the gate, while any other `transnet_v2` + failure still fails. The JSON and Markdown summaries name the TF32 mode and the per-model provider gaps. diff --git a/docs/ai/inference.md b/docs/ai/inference.md index 959a9cc794..91d082fb42 100644 --- a/docs/ai/inference.md +++ b/docs/ai/inference.md @@ -512,6 +512,32 @@ device on the same pair, and compare. `/cross-backend-diff` and `scripts/ci/cross_backend_vif_diff.py` cover the feature backends, not the tiny-AI execution providers. +### The CUDA EP on onnxruntime-gpu 1.31.1: TF32 and transnet_v2 + +The two bounds above are strict fp32 bounds, and the gate pins the CUDA EP to +strict fp32 to meet them. Measured across the 26-model tiny registry on two +local GPUs ([#2947](https://github.com/VMAFx/vmafx/issues/2947), comment +6696661349): + +- onnxruntime-gpu 1.31.1 leaves TF32 enabled by default in the CUDA EP. TF32 + GEMM/conv breaks the documented 1e-4 fp32 bound for 13 of the 24 fp32 + models, drifting 6e-04 to 3.9e-02 (worst case about 1e-2, stable across both + GPUs) at CI-like batch sizes. The parity gate therefore creates every CUDA + session with the provider option `use_tf32=0`, which puts all 24 models back + inside the bound (worst 3.8e-05). The session-config entry + `ep.cuda.use_tf32` is silently ignored by 1.31.1 — only the provider option + works. Opting back into TF32 means re-measuring against the band above, not + loosening the 1e-4 bound. +- `transnet_v2` cannot open on the CUDA EP at all in 1.31.1: its ScatterND + node `fork_segmentsum_scatter` raises + `element_type 6 (int32) not implemented for ScatterND operator`, + deterministically on both GPUs. The CPU EP is unaffected. The gate runs the + model's target-provider leg as a bind probe and records that exact failure + as an expected provider gap (`PROVIDER_GAP`); any other transnet_v2 failure + still fails the gate, and a gap-listed model that binds is reported as the + gap closing. +- fp16 (`smoke_fp16_v0`) and int8 models are exact and unaffected by TF32. + ## History - **2026-05-02, ADR-0241.** `vmaf_tiny_v3` (`mlp_medium`, 6 to 32 to 16 to 1, diff --git a/scripts/ci/tiny_ai_cross_device_parity_gate.py b/scripts/ci/tiny_ai_cross_device_parity_gate.py index a0ead289ea..0af1658f09 100755 --- a/scripts/ci/tiny_ai_cross_device_parity_gate.py +++ b/scripts/ci/tiny_ai_cross_device_parity_gate.py @@ -8,6 +8,15 @@ tolerances: - FP32: absolute error <= 1e-4 on model/tiny/vmaf_tiny_v2.onnx - FP16: absolute error <= 1e-2 on model/tiny/smoke_fp16_v0.onnx +- transnet_v2: a zeros-input execute probe against the target provider, not a + numerical case (its CUDA leg cannot execute on ORT 1.31.1, see + TRANSNET_V2_SCATTERND_GAP) + +Every CUDAExecutionProvider session is created with the provider option +use_tf32=0: ORT 1.31.1 leaves TF32 enabled by default in the CUDA EP and TF32 +GEMM/conv breaks the documented 1e-4 fp32 bound, while the session-config entry +ep.cuda.use_tf32 is silently ignored by that release (issue #2947, comment +6696661349). Fail-closed contract: - If a requested target provider is absent from onnxruntime.get_available_providers(), @@ -15,6 +24,8 @@ - If onnxruntime silently falls back to CPU for the target session, the gate detects the provider mismatch and fails closed. - If numerical delta exceeds tolerance, the gate fails closed. +- The transnet_v2 probe fails closed on every failure except the one named CUDA + ScatterND gap recorded in TRANSNET_V2_SCATTERND_GAP. - Generates JSON summary (--json-out) and Markdown report (--md-out). """ @@ -48,6 +59,22 @@ DEFAULT_FP16_TOLERANCE = 1e-2 DEFAULT_REF_PROVIDER = "CPUExecutionProvider" DEFAULT_TARGET_PROVIDER = "CUDAExecutionProvider" +# ORT 1.31.1 ships the CUDA EP with TF32 enabled by default; TF32 GEMM/conv +# pushed 13/24 fp32 tiny models past the documented 1e-4 bound (drift +# 6e-04..3.9e-02 at CI-like batch sizes) while use_tf32=0 passed all 24 (worst +# 3.8e-05). The session-config entry ep.cuda.use_tf32 is silently ignored by +# 1.31.1, so the pin must ride the provider option list (issue #2947, comment +# 6696661349). +CUDA_TF32_PROVIDER_OPTIONS: dict[str, int] = {"use_tf32": 0} +# transnet_v2 binds the CUDA EP in ORT 1.31.1 but cannot execute there: the +# ScatterND node fork_segmentsum_scatter fails at Run time with this status +# message (element type 6 is ONNX int32), deterministically on both local +# measurement GPUs; the CPU EP is unaffected. The gate runs the model's +# target-provider leg as a zeros-input execute probe, records that one named +# failure as the model's expected provider gap, and fails on every other +# transnet_v2 failure (issue #2947, comment 6696661349). +TRANSNET_V2_MODEL_ID = "transnet_v2" +TRANSNET_V2_SCATTERND_GAP = "element_type 6 not implemented for ScatterND operator" @dataclass @@ -72,6 +99,7 @@ class ParityGateReport: status: str cases: list[CaseResult] = field(default_factory=list) note: str = "" + tf32_pinned: bool = False def load_fp32_inputs(features_path: Path, max_rows: int = 256) -> np.ndarray: @@ -100,12 +128,28 @@ def load_fp16_inputs() -> np.ndarray: return np.array([[[[0.125, 0.25], [0.5, 0.875]]]], dtype=np.float16) +def session_provider_options(provider: str) -> dict[str, int]: + """Provider options for one session of a pinned-provider run. + + The CUDA EP pins strict fp32 (see CUDA_TF32_PROVIDER_OPTIONS); every other + provider runs on its defaults. + """ + if provider == DEFAULT_TARGET_PROVIDER: + return dict(CUDA_TF32_PROVIDER_OPTIONS) + return {} + + def create_session(model_path: Path, provider: str) -> ort.InferenceSession: """Create an ONNX Runtime inference session pinned to a single provider.""" opts = ort.SessionOptions() opts.inter_op_num_threads = 1 opts.intra_op_num_threads = 1 - return ort.InferenceSession(str(model_path), sess_options=opts, providers=[provider]) + return ort.InferenceSession( + str(model_path), + sess_options=opts, + providers=[provider], + provider_options=[session_provider_options(provider)], + ) def run_session_inference( @@ -183,6 +227,67 @@ def fail(status: str, note: str) -> CaseResult: ) +def evaluate_provider_gap(model_path: Path, target_provider: str) -> CaseResult: + """transnet_v2's target-provider leg: a zeros-input execute probe. + + The model binds the CUDA EP in ORT 1.31.1 but cannot execute there (see + TRANSNET_V2_SCATTERND_GAP), so this leg runs one zeros input on the target + provider and judges only that outcome: the one named CUDA failure is the + expected gap and passes as PROVIDER_GAP, a successful run is reported as + the gap closing, and anything else fails the gate. The zeros input is never + compared numerically; this leg measures executability, not parity. + """ + name = f"{TRANSNET_V2_MODEL_ID} provider gap" + + def case(status: str, ok: bool, note: str) -> CaseResult: + return CaseResult( + name=name, + model_path=str(model_path), + precision="execute", + tolerance=0.0, + max_abs_error=0.0, + mean_abs_error=0.0, + ok=ok, + status=status, + note=note, + ) + + if not model_path.is_file(): + return case("ERROR", False, f"Model file not found: {model_path}") + try: + sess = create_session(model_path, target_provider) + bound = sess.get_providers() + if target_provider not in bound: + return case( + "PROVIDER_UNBOUND", + False, + f"Target provider '{target_provider}' failed to bind; bound providers: {bound}", + ) + in_meta = sess.get_inputs()[0] + shape = [d if isinstance(d, int) and d > 0 else 1 for d in in_meta.shape] + dtype = {"tensor(float)": np.float32}.get(in_meta.type) + if dtype is None: + return case("ERROR", False, f"Probe cannot feed input type {in_meta.type}") + sess.run(None, {in_meta.name: np.zeros(shape, dtype=dtype)}) + except Exception as exc: + named_gap = target_provider == DEFAULT_TARGET_PROVIDER and TRANSNET_V2_SCATTERND_GAP in str( + exc + ) + if named_gap: + return case( + "PROVIDER_GAP", + True, + f"expected gap: {TRANSNET_V2_SCATTERND_GAP} (ORT 1.31.1 CUDA EP)", + ) + return case("ERROR", False, f"Exception during probe inference: {exc}") + return case( + "OK", + True, + f"gap closed: {TRANSNET_V2_MODEL_ID} executes on {target_provider}; " + "numerical parity not compared by this probe", + ) + + def _stop_report( ref: str, target: str, available: bool, status: str, note: str, ok: bool = False ) -> ParityGateReport: @@ -201,6 +306,7 @@ def run_parity_gate( fp32_model: Path, fp16_model: Path, features_path: Path, + transnet_model: Path = Path("model/tiny/transnet_v2.onnx"), ref_provider: str = DEFAULT_REF_PROVIDER, target_provider: str = DEFAULT_TARGET_PROVIDER, fp32_tol: float = DEFAULT_FP32_TOLERANCE, @@ -233,7 +339,11 @@ def run_parity_gate( except (FileNotFoundError, ValueError, RuntimeError) as exc: return _stop_report(ref_provider, target_provider, True, "BAD_FIXTURE", str(exc)) cases = _run_cases( - (fp32_model, fp16_model), fp32_inputs, (fp32_tol, fp16_tol), ref_provider, target_provider + (fp32_model, fp16_model, transnet_model), + fp32_inputs, + (fp32_tol, fp16_tol), + ref_provider, + target_provider, ) all_ok = all(c.ok for c in cases) return ParityGateReport( @@ -243,17 +353,18 @@ def run_parity_gate( ok=all_ok, status="PASS" if all_ok else "FAIL", cases=cases, + tf32_pinned=DEFAULT_TARGET_PROVIDER in (ref_provider, target_provider), ) def _run_cases( - models: tuple[Path, Path], + models: tuple[Path, Path, Path], fp32_inputs: np.ndarray, tolerances: tuple[float, float], ref_provider: str, target_provider: str, ) -> list[CaseResult]: - """The FP32 and FP16 cases, in that order.""" + """The FP32 and FP16 numerical cases, then the transnet_v2 execute probe.""" return [ evaluate_case( "FP32 vmaf_tiny_v2", @@ -273,21 +384,31 @@ def _run_cases( ref_provider, target_provider, ), + evaluate_provider_gap(models[2], target_provider), ] def render_markdown(report: ParityGateReport) -> str: """Render human-readable Markdown summary table.""" + tf32_mode = ( + "use_tf32=0 pinned on every CUDAExecutionProvider session" + if report.tf32_pinned + else "not pinned (no CUDA session in this run)" + ) lines = [ "# Tiny AI Cross-Device Parity Gate Report (T-TINY-AI-CROSS-DEVICE-PARITY-UNGATED-2026-09-25)", "", f"- **Reference Provider**: `{report.reference_provider}`", f"- **Target Provider**: `{report.target_provider}`", f"- **Target Available**: `{report.target_available}`", + f"- **TF32**: {tf32_mode}", f"- **Verdict**: **{report.status}**", ] if report.note: lines.append(f"- **Note**: {report.note}") + gaps = [c.name for c in report.cases if c.status == "PROVIDER_GAP"] + if gaps: + lines.append(f"- **Expected provider gaps**: {', '.join(gaps)}") lines.extend( [ "", @@ -319,6 +440,11 @@ def render_json(report: ParityGateReport) -> dict[str, Any]: "ai/testdata/bisect/features.parquet", "canonical features parquet fixture", ), + ( + "--transnet-model", + "model/tiny/transnet_v2.onnx", + "transnet_v2 ONNX model for the provider-gap probe", + ), ) _TOLERANCE_OPTIONS = ( ("--fp32-tolerance", DEFAULT_FP32_TOLERANCE, "FP32"), @@ -363,6 +489,7 @@ def main(argv: list[str] | None = None) -> int: fp32_model=args.fp32_model, fp16_model=args.fp16_model, features_path=args.fp32_features, + transnet_model=args.transnet_model, ref_provider=args.reference_provider, target_provider=args.target_provider, fp32_tol=args.fp32_tolerance, From 8f4ab9c668009ec4f1b97f07ed13b35ecbf4c8fc Mon Sep 17 00:00:00 2001 From: luoye-cpu <3412829159@qq.com> Date: Sun, 8 Nov 2026 00:43:01 +0800 Subject: [PATCH 2/4] ci: rewrap the provider-gap named_gap check under the 100-col limit The black wrap put the TRANSNET_V2_SCATTERND_GAP membership test into an awkward str( break; pre-extract the exception message so the condition reads on its own line. Noted in the #3334 review as a non-blocker. black and ruff are clean; the gate's unit tests are unchanged (23 pass, and the 4 fixture-dependent ones fail identically on the unpatched tree, so the rewrap is behaviour-neutral). Signed-off-by: luoye-cpu <3412829159@qq.com> --- scripts/ci/tiny_ai_cross_device_parity_gate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ci/tiny_ai_cross_device_parity_gate.py b/scripts/ci/tiny_ai_cross_device_parity_gate.py index 0af1658f09..190d8785e3 100755 --- a/scripts/ci/tiny_ai_cross_device_parity_gate.py +++ b/scripts/ci/tiny_ai_cross_device_parity_gate.py @@ -270,8 +270,9 @@ def case(status: str, ok: bool, note: str) -> CaseResult: return case("ERROR", False, f"Probe cannot feed input type {in_meta.type}") sess.run(None, {in_meta.name: np.zeros(shape, dtype=dtype)}) except Exception as exc: - named_gap = target_provider == DEFAULT_TARGET_PROVIDER and TRANSNET_V2_SCATTERND_GAP in str( - exc + exc_text = str(exc) + named_gap = ( + target_provider == DEFAULT_TARGET_PROVIDER and TRANSNET_V2_SCATTERND_GAP in exc_text ) if named_gap: return case( From 0726d6adbdb1b0541167db671e4d9fef5bf51d23 Mon Sep 17 00:00:00 2001 From: luoye-cpu <3412829159@qq.com> Date: Sun, 8 Nov 2026 06:45:11 +0800 Subject: [PATCH 3/4] fix(ai): match the measured ScatterND message and make the probe wording exact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On review, three findings against the transnet_v2 leg, one of them load-bearing: - The gap constant dropped the "(int32) " segment of the measured message — the ORT 1.31.1 CUDA failure is `element_type 6 (int32) not implemented for ScatterND operator` (testdata sweep, both GPUs; docs and changelog carry that spelling), so the rewrap's substring never matched the error it names and the leg would report ERROR instead of PROVIDER_GAP on a real CUDA run. The constant and the test fixture now carry the measured message byte for byte; the creation-path and Run-time-path tests already cover both surfaces and pass against the corrected needle. - `transnet_model` becomes keyword-only: every caller already passed it by keyword, and the signature can no longer drift under a positional caller. - The changelog and docs/ai/inference.md said "bind probe"/"cannot open" — the leg binds and then executes one zeros input, and the named failure is accepted whether it surfaces at session creation or at Run time. Both now say that. ## Deliverables (ADR-0108) - [ ] **Research digest**: no digest needed: review-driven corrections to this PR's own rewrap; the measured message is recorded in the #2947 sweep evidence. - [ ] **Decision matrix**: no alternatives to weigh: the needle is the measured message, not a looser pattern — the named-gap contract stays exact. - [ ] **AGENTS.md invariant note**: no rebase-sensitive invariants: one constant, its test fixture, one keyword-only marker and two prose fixes. - [x] **Reproducer / smoke-test command** — `python -m pytest ai/tests/test_tiny_ai_cross_device_parity_gate.py -k "scatternd or gap or transnet"` (10 passed; the four remaining local failures need pyarrow, unrelated). - [ ] **CHANGELOG fragment**: no changelog entry needed: the PR is unmerged; the broken needle never shipped. Signed-off-by: luoye-cpu <3412829159@qq.com> --- ai/tests/test_tiny_ai_cross_device_parity_gate.py | 2 +- changelog.d/fixed/tiny-ai-cuda-tf32-pin.md | 7 ++++--- docs/ai/inference.md | 13 +++++++------ scripts/ci/tiny_ai_cross_device_parity_gate.py | 6 +++++- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/ai/tests/test_tiny_ai_cross_device_parity_gate.py b/ai/tests/test_tiny_ai_cross_device_parity_gate.py index 264a6164f9..cb2b32b98d 100644 --- a/ai/tests/test_tiny_ai_cross_device_parity_gate.py +++ b/ai/tests/test_tiny_ai_cross_device_parity_gate.py @@ -55,7 +55,7 @@ SCATTERND_ERROR = ( "[ONNXRuntimeError] : 1 : FAIL : Non-zero status code returned while running " "ScatterND node. Name:'fork_segmentsum_scatter' Status Message: element_type 6 " - "not implemented for ScatterND operator." + "(int32) not implemented for ScatterND operator." ) diff --git a/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md b/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md index 14486e1c54..81ec38f959 100644 --- a/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md +++ b/changelog.d/fixed/tiny-ai-cuda-tf32-pin.md @@ -2,7 +2,8 @@ `scripts/ci/tiny_ai_cross_device_parity_gate.py` creates every `CUDAExecutionProvider` session with the provider option `use_tf32=0`: onnxruntime-gpu 1.31.1 leaves TF32 enabled by default, which pushed 13/24 fp32 tiny models past the documented 1e-4 bound (worst 3.9e-02; with the pin, worst 3.8e-05), and the - session-config entry `ep.cuda.use_tf32` is silently ignored by that release. `transnet_v2` runs as a bind - probe: its CUDA leg cannot open on 1.31.1 (`element_type 6 (int32) not implemented for ScatterND operator`) - and is recorded as an expected provider gap instead of failing the gate, while any other `transnet_v2` + session-config entry `ep.cuda.use_tf32` is silently ignored by that release. `transnet_v2` runs as an + execute probe — the CUDA leg binds but its `ScatterND` node fails at Run time on 1.31.1 + (`element_type 6 (int32) not implemented for ScatterND operator`) — and that one named failure is + recorded as an expected provider gap instead of failing the gate, while any other `transnet_v2` failure still fails. The JSON and Markdown summaries name the TF32 mode and the per-model provider gaps. diff --git a/docs/ai/inference.md b/docs/ai/inference.md index 91d082fb42..10de518e8e 100644 --- a/docs/ai/inference.md +++ b/docs/ai/inference.md @@ -528,14 +528,15 @@ local GPUs ([#2947](https://github.com/VMAFx/vmafx/issues/2947), comment `ep.cuda.use_tf32` is silently ignored by 1.31.1 — only the provider option works. Opting back into TF32 means re-measuring against the band above, not loosening the 1e-4 bound. -- `transnet_v2` cannot open on the CUDA EP at all in 1.31.1: its ScatterND - node `fork_segmentsum_scatter` raises +- `transnet_v2` cannot execute on the CUDA EP in 1.31.1: the leg binds, but its + ScatterND node `fork_segmentsum_scatter` raises `element_type 6 (int32) not implemented for ScatterND operator`, deterministically on both GPUs. The CPU EP is unaffected. The gate runs the - model's target-provider leg as a bind probe and records that exact failure - as an expected provider gap (`PROVIDER_GAP`); any other transnet_v2 failure - still fails the gate, and a gap-listed model that binds is reported as the - gap closing. + model's target-provider leg as a zeros-input execute probe and records that + exact failure — whether it surfaces while creating the session or at Run + time — as an expected provider gap (`PROVIDER_GAP`); any other transnet_v2 + failure still fails the gate, and a gap-listed model that executes is + reported as the gap closing. - fp16 (`smoke_fp16_v0`) and int8 models are exact and unaffected by TF32. ## History diff --git a/scripts/ci/tiny_ai_cross_device_parity_gate.py b/scripts/ci/tiny_ai_cross_device_parity_gate.py index 190d8785e3..5773b1d701 100755 --- a/scripts/ci/tiny_ai_cross_device_parity_gate.py +++ b/scripts/ci/tiny_ai_cross_device_parity_gate.py @@ -74,7 +74,10 @@ # failure as the model's expected provider gap, and fails on every other # transnet_v2 failure (issue #2947, comment 6696661349). TRANSNET_V2_MODEL_ID = "transnet_v2" -TRANSNET_V2_SCATTERND_GAP = "element_type 6 not implemented for ScatterND operator" +# The measured ORT 1.31.1 CUDA message, byte for byte as the node raises it +# (testdata sweep, both GPUs): dropping the "(int32) " segment made the named +# gap unmatchable against the error it names. +TRANSNET_V2_SCATTERND_GAP = "element_type 6 (int32) not implemented for ScatterND operator" @dataclass @@ -307,6 +310,7 @@ def run_parity_gate( fp32_model: Path, fp16_model: Path, features_path: Path, + *, transnet_model: Path = Path("model/tiny/transnet_v2.onnx"), ref_provider: str = DEFAULT_REF_PROVIDER, target_provider: str = DEFAULT_TARGET_PROVIDER, From a0ad336bb4dec1492f7a8962261c200ff3e5bfee Mon Sep 17 00:00:00 2001 From: luoye-cpu <3412829159@qq.com> Date: Tue, 10 Nov 2026 23:37:50 +0800 Subject: [PATCH 4/4] test(ai): extract the mocked ORT session into one helper in the parity gate tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codacy flagged the duplicated mocked-session setup across the cross-device parity gate tests: every mocked session rebuilt the same get_providers/get_inputs contract inline, so a change to the probe contract would have to be edited in nine places. One mock_session(providers, input_meta) helper now builds every mocked session and the run behaviour stays with each test. No behaviour change: the 23 non-parquet unit tests pass unchanged, and the 4 parquet-backed tests fail locally only for the missing pyarrow engine — identical before and after this change (verified by stash), while the tooling environment installs pyarrow from the lock. Signed-off-by: luoye-cpu <3412829159@qq.com> --- .../test_tiny_ai_cross_device_parity_gate.py | 91 +++++++++---------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/ai/tests/test_tiny_ai_cross_device_parity_gate.py b/ai/tests/test_tiny_ai_cross_device_parity_gate.py index cb2b32b98d..468f331dfd 100644 --- a/ai/tests/test_tiny_ai_cross_device_parity_gate.py +++ b/ai/tests/test_tiny_ai_cross_device_parity_gate.py @@ -20,6 +20,7 @@ import json import sys from pathlib import Path +from typing import Any from unittest.mock import MagicMock, patch import numpy as np @@ -59,6 +60,25 @@ ) +def mock_session( + providers: list[str], + input_meta: dict[str, Any] | None = None, +) -> MagicMock: + """A mocked ORT session reporting the given bound providers. + + The metadata contract the gate reads is get_providers() and get_inputs(); + input_meta overrides the mocked input's kwargs (name, shape, type) when a + test needs a specific input signature. A session built without input_meta + leaves get_inputs at its MagicMock default, for the paths that never reach + it. + """ + session = MagicMock() + session.get_providers.return_value = list(providers) + if input_meta is not None: + session.get_inputs.return_value = [MagicMock(**input_meta)] + return session + + def test_missing_target_provider_fails_closed() -> None: """Missing target provider must report MISSING_PROVIDER and ok=False.""" report = run_parity_gate( @@ -105,13 +125,10 @@ def test_missing_reference_provider_fails_closed() -> None: def test_target_provider_unbound_detection(tmp_path: Path) -> None: """If target session falls back and does not bind target provider, fail closed.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"}) - mock_target_sess = MagicMock() # Simulate ORT fallback: requested CUDA, but session bound CPU only - mock_target_sess.get_providers.return_value = ["CPUExecutionProvider"] + mock_target_sess = mock_session(["CPUExecutionProvider"]) with patch( "scripts.ci.tiny_ai_cross_device_parity_gate.create_session", @@ -133,13 +150,8 @@ def test_target_provider_unbound_detection(tmp_path: Path) -> None: def test_numerical_mismatch_fp32_fails_closed() -> None: """Error exceeding FP32 tolerance (1e-4) must fail closed.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")] - - mock_target_sess = MagicMock() - mock_target_sess.get_providers.return_value = ["TargetProvider"] - mock_target_sess.get_inputs.return_value = [MagicMock(name="features")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"}) + mock_target_sess = mock_session(["TargetProvider"], {"name": "features"}) ref_output = np.array([50.0, 60.0, 70.0], dtype=np.float32) # Delta of 5e-4 exceeds tolerance of 1e-4 @@ -172,13 +184,8 @@ def test_numerical_mismatch_fp32_fails_closed() -> None: def test_numerical_mismatch_fp16_fails_closed() -> None: """Error exceeding FP16 tolerance (1e-2) must fail closed.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="x")] - - mock_target_sess = MagicMock() - mock_target_sess.get_providers.return_value = ["TargetProvider"] - mock_target_sess.get_inputs.return_value = [MagicMock(name="x")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "x"}) + mock_target_sess = mock_session(["TargetProvider"], {"name": "x"}) ref_output = np.array([[[[0.5, 0.5], [0.5, 0.5]]]], dtype=np.float16) # Delta of 0.05 exceeds tolerance of 0.01 @@ -210,13 +217,8 @@ def test_numerical_mismatch_fp16_fails_closed() -> None: def test_numerical_match_within_tolerance_passes() -> None: """Error within tolerance must report status OK and ok=True.""" - mock_ref_sess = MagicMock() - mock_ref_sess.get_providers.return_value = ["CPUExecutionProvider"] - mock_ref_sess.get_inputs.return_value = [MagicMock(name="features")] - - mock_target_sess = MagicMock() - mock_target_sess.get_providers.return_value = ["TargetProvider"] - mock_target_sess.get_inputs.return_value = [MagicMock(name="features")] + mock_ref_sess = mock_session(["CPUExecutionProvider"], {"name": "features"}) + mock_target_sess = mock_session(["TargetProvider"], {"name": "features"}) ref_output = np.array([50.0, 60.0, 70.0], dtype=np.float32) # Delta of 5e-5 is within tolerance of 1e-4 @@ -307,11 +309,10 @@ def test_transnet_v2_named_cause_is_not_accepted_off_cuda() -> None: def test_transnet_v2_successful_execute_reports_the_gap_closed() -> None: """A gap-listed model that executes on the target is fine: reported, never a failure.""" - bound = MagicMock() - bound.get_providers.return_value = ["CUDAExecutionProvider"] - bound.get_inputs.return_value = [ - MagicMock(name="frames", shape=[1, 100, 3, 27, 48], type="tensor(float)") - ] + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"}, + ) bound.run.return_value = [np.zeros((1, 100), dtype=np.float32)] with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") @@ -322,11 +323,10 @@ def test_transnet_v2_successful_execute_reports_the_gap_closed() -> None: def test_transnet_v2_run_time_scatternd_failure_is_the_expected_gap() -> None: """The gap is a Run-time kernel failure: a bound session that fails Run matches.""" - bound = MagicMock() - bound.get_providers.return_value = ["CUDAExecutionProvider"] - bound.get_inputs.return_value = [ - MagicMock(name="frames", shape=[1, 100, 3, 27, 48], type="tensor(float)") - ] + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"}, + ) bound.run.side_effect = RuntimeError(SCATTERND_ERROR) with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") @@ -337,11 +337,10 @@ def test_transnet_v2_run_time_scatternd_failure_is_the_expected_gap() -> None: def test_transnet_v2_run_time_other_failure_fails_closed() -> None: """A bound session whose Run fails for any other reason still fails the gate.""" - bound = MagicMock() - bound.get_providers.return_value = ["CUDAExecutionProvider"] - bound.get_inputs.return_value = [ - MagicMock(name="frames", shape=[1, 100, 3, 27, 48], type="tensor(float)") - ] + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "frames", "shape": [1, 100, 3, 27, 48], "type": "tensor(float)"}, + ) bound.run.side_effect = RuntimeError("Non-zero status code returned while running Conv node") with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") @@ -351,9 +350,10 @@ def test_transnet_v2_run_time_other_failure_fails_closed() -> None: def test_transnet_v2_unfeedable_input_type_fails_closed() -> None: """A probe input the execute probe cannot feed is an error, not a gap.""" - bound = MagicMock() - bound.get_providers.return_value = ["CUDAExecutionProvider"] - bound.get_inputs.return_value = [MagicMock(name="x", shape=[1, 4], type="tensor(int64)")] + bound = mock_session( + ["CUDAExecutionProvider"], + {"name": "x", "shape": [1, 4], "type": "tensor(int64)"}, + ) with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=bound): case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") assert not case.ok @@ -362,8 +362,7 @@ def test_transnet_v2_unfeedable_input_type_fails_closed() -> None: def test_transnet_v2_silent_fallback_fails_closed() -> None: """The probe session falling back to the CPU fails closed, as elsewhere.""" - fallback = MagicMock() - fallback.get_providers.return_value = ["CPUExecutionProvider"] + fallback = mock_session(["CPUExecutionProvider"]) with patch("scripts.ci.tiny_ai_cross_device_parity_gate.create_session", return_value=fallback): case = evaluate_provider_gap(FP32_MODEL, "CUDAExecutionProvider") assert not case.ok