From 9db6ef2273503eefb33b9d7645b44d187bc1db95 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Mon, 20 Jul 2026 01:08:55 +0800 Subject: [PATCH 1/2] Add XLM-RoBERTa NER CPU recipes --- .../cpu/token-classification_fp16_config.json | 76 +++++++++++++++++++ .../cpu/token-classification_fp32_config.json | 54 +++++++++++++ src/winml/modelkit/commands/analyze.py | 9 ++- src/winml/modelkit/quant/fp16.py | 64 +++++++++++++--- src/winml/modelkit/quant/passes/fp16.py | 5 +- src/winml/modelkit/session/session.py | 12 ++- src/winml/modelkit/sysinfo/device.py | 13 +++- tests/unit/optim/test_fp16.py | 18 +++++ tests/unit/session/test_winml_session.py | 12 +++ tests/unit/sysinfo/test_device.py | 8 ++ tests/unit/test_quant_passes.py | 18 ++++- 11 files changed, 262 insertions(+), 27 deletions(-) create mode 100644 examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp16_config.json create mode 100644 examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp32_config.json diff --git a/examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp16_config.json b/examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp16_config.json new file mode 100644 index 000000000..b85f3fb06 --- /dev/null +++ b/examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp16_config.json @@ -0,0 +1,76 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "input_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 250002 + ] + }, + { + "name": "attention_mask", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 2 + ] + } + ], + "output_tensors": [ + { + "name": "logits" + } + ] + }, + "optim": { + "clamp_constant_values": true + }, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "task": "token-classification", + "model_id": "Davlan/xlm-roberta-large-ner-hrl", + "model_type": "xlm-roberta", + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "token-classification", + "model_class": "XLMRobertaForTokenClassification", + "model_type": "xlm-roberta" + } +} diff --git a/examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp32_config.json b/examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp32_config.json new file mode 100644 index 000000000..02b0410ad --- /dev/null +++ b/examples/recipes/Davlan_xlm-roberta-large-ner-hrl/cpu/cpu/token-classification_fp32_config.json @@ -0,0 +1,54 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "input_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 250002 + ] + }, + { + "name": "attention_mask", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 2 + ] + } + ], + "output_tensors": [ + { + "name": "logits" + } + ] + }, + "optim": { + "clamp_constant_values": true + }, + "quant": null, + "compile": null, + "loader": { + "task": "token-classification", + "model_class": "XLMRobertaForTokenClassification", + "model_type": "xlm-roberta" + } +} diff --git a/src/winml/modelkit/commands/analyze.py b/src/winml/modelkit/commands/analyze.py index 0b997a5fa..99bcc86e5 100644 --- a/src/winml/modelkit/commands/analyze.py +++ b/src/winml/modelkit/commands/analyze.py @@ -979,10 +979,11 @@ def analyze( ) execution_pairs = _sort_ep_device_pairs(execution_pairs) - # Local pairs are still needed to gate --run-unknown-op probing - # (_resolve_run_unknown_op). Single-target `auto` selection is already - # local by construction, so no extra intersection/warning is required. - local_pairs = set(_get_local_ep_device_pairs()) + # Local pairs are only needed to gate runtime unknown-op probing. + # Static rule analysis (the default) must not load optional provider + # plugins merely to print an informational list: on Windows that can + # leave native plugin/catalog state alive until interpreter teardown. + local_pairs = set(_get_local_ep_device_pairs()) if run_unknown_op else set() if not execution_pairs: raise click.UsageError("No EP/device combination matched the current selection.") diff --git a/src/winml/modelkit/quant/fp16.py b/src/winml/modelkit/quant/fp16.py index b3a6fe3a3..23f8902ac 100644 --- a/src/winml/modelkit/quant/fp16.py +++ b/src/winml/modelkit/quant/fp16.py @@ -11,7 +11,9 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: @@ -21,7 +23,7 @@ def convert_to_fp16( - model: ModelProto, + model: ModelProto | str | Path, *, keep_io_types: bool = True, op_block_list: list[str] | None = None, @@ -34,7 +36,10 @@ def convert_to_fp16( Note: ORT's converter mutates the model in-place and returns the same object. Args: - model: Input ONNX ModelProto (will be mutated in-place by ORT). + model: Input ONNX ModelProto or path. Paths use ORT's file-based shape + inference, which supports models above protobuf's 2 GB in-memory + serialization limit and resolves external data relative to the + model file. keep_io_types: If True, preserve FP32 model inputs/outputs by inserting Cast nodes at boundaries. Recommended for CPU-safe inference. op_block_list: Op types to keep in FP32 (e.g., ["LayerNorm", "Softmax"]). @@ -49,14 +54,22 @@ def convert_to_fp16( # Skip if model is already FP16 (check floating-point initializer dtypes) fp32_types = {TensorProto.FLOAT, TensorProto.DOUBLE, TensorProto.BFLOAT16} - initializers = model.graph.initializer + model_path = Path(model) if isinstance(model, str | Path) else None + if model_path is not None: + import onnx + + inspection_model = onnx.load(str(model_path), load_external_data=False) + else: + inspection_model = cast("ModelProto", model) + + initializers = inspection_model.graph.initializer if initializers: float_inits = [t for t in initializers if t.data_type in fp32_types | {TensorProto.FLOAT16}] if float_inits and all(t.data_type == TensorProto.FLOAT16 for t in float_inits): logger.info("Model is already FP16 — skipping conversion.") - return model + return inspection_model - original_nodes = len(model.graph.node) + original_nodes = len(inspection_model.graph.node) logger.info("Converting model to FP16...") if keep_io_types: @@ -64,11 +77,40 @@ def convert_to_fp16( if op_block_list: logger.info(" Keeping ops in FP32: %s", op_block_list) - converted: ModelProto = convert_float_to_float16( - model, - keep_io_types=keep_io_types, - op_block_list=op_block_list, - ) + if model_path is not None: + from onnx.shape_inference import infer_shapes_path + + # ORT's converter uses NamedTemporaryFile while it is still open, + # which cannot be reopened by ONNX on Windows. Own the temporary path + # here, close it before inference, and retain the file-based API that + # supports protobufs above 2 GB. + with tempfile.NamedTemporaryFile( + dir=model_path.parent, suffix=".shape_inferred.onnx", delete=False + ) as temporary: + inferred_path = Path(temporary.name) + try: + infer_shapes_path(str(model_path), str(inferred_path)) + inferred_model = onnx.load(str(inferred_path)) + finally: + inferred_path.unlink(missing_ok=True) + converted = cast( + "ModelProto", + convert_float_to_float16( + inferred_model, + keep_io_types=keep_io_types, + disable_shape_infer=True, + op_block_list=op_block_list, + ), + ) + else: + converted = cast( + "ModelProto", + convert_float_to_float16( + model, + keep_io_types=keep_io_types, + op_block_list=op_block_list, + ), + ) # ORT's converter appends Cast nodes at the end of the node list (for # keep_io_types), which breaks topological ordering. Re-sort the graph diff --git a/src/winml/modelkit/quant/passes/fp16.py b/src/winml/modelkit/quant/passes/fp16.py index 6669fda02..030a3479e 100644 --- a/src/winml/modelkit/quant/passes/fp16.py +++ b/src/winml/modelkit/quant/passes/fp16.py @@ -44,7 +44,7 @@ def run( use_external_data: bool = True, ) -> QuantizeResult: """Convert *model_path* to FP16 and write the result to *output_path*.""" - from ...onnx import load_onnx, save_onnx + from ...onnx import save_onnx from ..config import QuantizeResult from ..fp16 import convert_to_fp16 @@ -59,9 +59,8 @@ def run( warnings: list[str] = [] logger.info("Running FP16-only conversion (no quantization)...") - model = load_onnx(model_path, validate=False) model = convert_to_fp16( - model, + model_path, keep_io_types=self._config.fp16_keep_io_types, op_block_list=self._config.fp16_op_block_list, ) diff --git a/src/winml/modelkit/session/session.py b/src/winml/modelkit/session/session.py index a627ad809..799dbb1dc 100644 --- a/src/winml/modelkit/session/session.py +++ b/src/winml/modelkit/session/session.py @@ -165,7 +165,13 @@ def __init__( (``add_provider_for_devices`` mutates the options and cannot be replayed). Defaults to ``ort.SessionOptions``. """ - WinMLSession._init_winml_eps_once() + # The built-in CPU EP needs no optional WindowsML plugin discovery. + # Avoid loading the WindowsML catalog for an explicitly requested CPU + # session: besides unnecessary startup work, mixing catalog/plugin + # lifetime with a CPU-only ORT session can crash native teardown on + # Windows after otherwise successful inference. + if ep not in ("cpu", "CPUExecutionProvider"): + WinMLSession._init_winml_eps_once() self._onnx_path = Path(onnx_path) if not self._onnx_path.exists(): @@ -355,9 +361,7 @@ def run( # Run inference (with optional perf tracking) output_names = [o.name for o in session.get_outputs()] if self._perf_stats: - outputs = self._perf_stats.record( - lambda: session.run(output_names, ort_inputs) - ) + outputs = self._perf_stats.record(lambda: session.run(output_names, ort_inputs)) else: outputs = session.run(output_names, ort_inputs) diff --git a/src/winml/modelkit/sysinfo/device.py b/src/winml/modelkit/sysinfo/device.py index b4dacd71f..13819b646 100644 --- a/src/winml/modelkit/sysinfo/device.py +++ b/src/winml/modelkit/sysinfo/device.py @@ -213,10 +213,21 @@ def resolve_device( if device != "auto" and device not in _VALID_DEVICES: raise ValueError(f"Unknown device '{device}'. Expected 'auto', 'npu', 'gpu', or 'cpu'.") + # CPUExecutionProvider is built into ORT and does not require WindowsML's + # optional provider catalog. Keep explicit --ep cpu resolution on this + # direct path so CPU-only commands do not load unrelated plugin handles + # whose native teardown may outlive an ORT session on Windows. + ep_full = normalize_ep_name(ep) + if device == "cpu" and ep_full == "CPUExecutionProvider": + import onnxruntime as ort + + if "CPUExecutionProvider" not in ort.get_available_providers(): + raise ValueError("Requested EP 'cpu' is not available on this system.") + return "cpu", ["cpu"] + device_ep_map = dict(_get_device_ep_map_from_ort()) if ep is not None: - ep_full = normalize_ep_name(ep) if ep_full not in EP_SUPPORTED_DEVICES: raise ValueError(f"Unknown EP '{ep}'. Expected one of: {sorted(EP_SUPPORTED_DEVICES)}") # Static policy gate (see issue #860): reject EP/device combos the EP diff --git a/tests/unit/optim/test_fp16.py b/tests/unit/optim/test_fp16.py index 2e63ab193..527c1a7b8 100644 --- a/tests/unit/optim/test_fp16.py +++ b/tests/unit/optim/test_fp16.py @@ -16,6 +16,7 @@ from __future__ import annotations import numpy as np +import onnx from onnx import ModelProto, TensorProto, helper, numpy_helper from winml.modelkit.quant.fp16 import convert_to_fp16 @@ -63,6 +64,23 @@ def test_converts_weights_to_fp16(self) -> None: has_fp16 = any(init.data_type == TensorProto.FLOAT16 for init in result.graph.initializer) assert has_fp16, "Expected at least one FP16 initializer after conversion" + def test_path_conversion_uses_external_data_safe_shape_inference(self, tmp_path) -> None: + """Path input converts external-data models without in-memory shape serialization.""" + model_path = tmp_path / "model.onnx" + model = _build_simple_fp32_model() + onnx.save_model( + model, + str(model_path), + save_as_external_data=True, + all_tensors_to_one_file=True, + location="model.onnx.data", + size_threshold=0, + ) + + result = convert_to_fp16(model_path) + + assert any(init.data_type == TensorProto.FLOAT16 for init in result.graph.initializer) + def test_default_keeps_io_types(self) -> None: """Default keep_io_types=True preserves FP32 model I/O.""" model = _build_simple_fp32_model() diff --git a/tests/unit/session/test_winml_session.py b/tests/unit/session/test_winml_session.py index 0d42073a2..c707388f6 100644 --- a/tests/unit/session/test_winml_session.py +++ b/tests/unit/session/test_winml_session.py @@ -18,6 +18,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from unittest.mock import MagicMock import numpy as np @@ -105,6 +106,17 @@ def test_explicit_ep_cpu_binds_cpu_execution_provider(self, simple_matmul_onnx: assert session.ep_name == "CPUExecutionProvider" assert session._session.get_providers()[0] == "CPUExecutionProvider" + def test_explicit_cpu_does_not_initialize_optional_winml_eps( + self, simple_matmul_onnx: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """CPU-only sessions must not acquire unrelated WindowsML plugin handles.""" + init_eps = MagicMock() + monkeypatch.setattr(WinMLSession, "_init_winml_eps_once", init_eps) + + WinMLSession(onnx_path=simple_matmul_onnx, device="cpu", ep="cpu") + + init_eps.assert_not_called() + class TestWinMLSessionCompilation: """Test WinMLSession compilation (EPContext creation).""" diff --git a/tests/unit/sysinfo/test_device.py b/tests/unit/sysinfo/test_device.py index 01e3ec374..a3486ded4 100644 --- a/tests/unit/sysinfo/test_device.py +++ b/tests/unit/sysinfo/test_device.py @@ -359,6 +359,14 @@ def test_ep_qnn_filters_devices_to_npu_and_gpu(self) -> None: assert device == "npu" assert available == ["npu", "gpu"] + def test_explicit_cpu_ep_bypasses_optional_provider_discovery(self) -> None: + """Built-in CPU resolution must not initialize the WindowsML plugin catalog.""" + with patch("winml.modelkit.sysinfo.device._get_device_ep_map_from_ort") as device_map: + device, available = resolve_device("cpu", ep="cpu") + + assert (device, available) == ("cpu", ["cpu"]) + device_map.assert_not_called() + def test_ep_qnn_auto_picks_gpu_when_no_npu(self) -> None: """ep='qnn' on a GPU-only system auto-selects gpu.""" with _patch_device_ep_map( diff --git a/tests/unit/test_quant_passes.py b/tests/unit/test_quant_passes.py index d78cbe410..05a0d6726 100644 --- a/tests/unit/test_quant_passes.py +++ b/tests/unit/test_quant_passes.py @@ -315,15 +315,19 @@ def test_reads_fp16_fields_from_config( output_path = tmp_path / "out.onnx" calls: list[dict] = [] - fake_model = SimpleNamespace() def fake_convert(model, *, keep_io_types, op_block_list): - calls.append({"keep_io_types": keep_io_types, "op_block_list": op_block_list}) + calls.append( + { + "model": model, + "keep_io_types": keep_io_types, + "op_block_list": op_block_list, + } + ) return model # Patch the source modules that are lazily imported inside run() fake_onnx_mod = ModuleType("winml.modelkit.onnx") - fake_onnx_mod.load_onnx = lambda *a, **k: fake_model # type: ignore[attr-defined] fake_onnx_mod.save_onnx = lambda *a, **k: None # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "winml.modelkit.onnx", fake_onnx_mod) @@ -334,7 +338,13 @@ def fake_convert(model, *, keep_io_types, op_block_list): result = FP16Pass(config).run(model_path, output_path) assert result.success - assert calls == [{"keep_io_types": False, "op_block_list": ["Gather"]}] + assert calls == [ + { + "model": model_path, + "keep_io_types": False, + "op_block_list": ["Gather"], + } + ] # --------------------------------------------------------------------------- From cb7f8305754d4148ae507493e4e40efb893559d9 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Mon, 20 Jul 2026 02:05:51 +0800 Subject: [PATCH 2/2] Fix FP16 external data relocation --- src/winml/modelkit/quant/fp16.py | 33 +++++---- tests/unit/optim/test_fp16.py | 117 ++++++++++++++++++++++--------- 2 files changed, 100 insertions(+), 50 deletions(-) diff --git a/src/winml/modelkit/quant/fp16.py b/src/winml/modelkit/quant/fp16.py index 23f8902ac..3aab274bd 100644 --- a/src/winml/modelkit/quant/fp16.py +++ b/src/winml/modelkit/quant/fp16.py @@ -17,17 +17,17 @@ if TYPE_CHECKING: - from onnx import ModelProto + import onnx logger = logging.getLogger(__name__) def convert_to_fp16( - model: ModelProto | str | Path, + model: onnx.ModelProto | str | Path, *, keep_io_types: bool = True, op_block_list: list[str] | None = None, -) -> ModelProto: +) -> onnx.ModelProto: """Convert an ONNX model from FP32 to FP16 precision. Uses onnxruntime.transformers.float16.convert_float_to_float16 internally. @@ -49,24 +49,29 @@ def convert_to_fp16( Returns: The converted model (same object as input due to ORT in-place mutation). """ - from onnx import TensorProto + import onnx from onnxruntime.transformers.float16 import convert_float_to_float16 # Skip if model is already FP16 (check floating-point initializer dtypes) - fp32_types = {TensorProto.FLOAT, TensorProto.DOUBLE, TensorProto.BFLOAT16} + fp32_types = {onnx.TensorProto.FLOAT, onnx.TensorProto.DOUBLE, onnx.TensorProto.BFLOAT16} model_path = Path(model) if isinstance(model, str | Path) else None if model_path is not None: - import onnx - inspection_model = onnx.load(str(model_path), load_external_data=False) else: - inspection_model = cast("ModelProto", model) + inspection_model = cast("onnx.ModelProto", model) initializers = inspection_model.graph.initializer if initializers: - float_inits = [t for t in initializers if t.data_type in fp32_types | {TensorProto.FLOAT16}] - if float_inits and all(t.data_type == TensorProto.FLOAT16 for t in float_inits): + float_inits = [ + t for t in initializers if t.data_type in fp32_types | {onnx.TensorProto.FLOAT16} + ] + if float_inits and all(t.data_type == onnx.TensorProto.FLOAT16 for t in float_inits): logger.info("Model is already FP16 — skipping conversion.") + if model_path is not None: + # A graph-only load retains external-data locations relative to + # the source model. Materialize those tensors before returning + # so callers can safely persist the result in another directory. + onnx.load_external_data_for_model(inspection_model, str(model_path.parent)) return inspection_model original_nodes = len(inspection_model.graph.node) @@ -78,8 +83,6 @@ def convert_to_fp16( logger.info(" Keeping ops in FP32: %s", op_block_list) if model_path is not None: - from onnx.shape_inference import infer_shapes_path - # ORT's converter uses NamedTemporaryFile while it is still open, # which cannot be reopened by ONNX on Windows. Own the temporary path # here, close it before inference, and retain the file-based API that @@ -89,12 +92,12 @@ def convert_to_fp16( ) as temporary: inferred_path = Path(temporary.name) try: - infer_shapes_path(str(model_path), str(inferred_path)) + onnx.shape_inference.infer_shapes_path(str(model_path), str(inferred_path)) inferred_model = onnx.load(str(inferred_path)) finally: inferred_path.unlink(missing_ok=True) converted = cast( - "ModelProto", + "onnx.ModelProto", convert_float_to_float16( inferred_model, keep_io_types=keep_io_types, @@ -104,7 +107,7 @@ def convert_to_fp16( ) else: converted = cast( - "ModelProto", + "onnx.ModelProto", convert_float_to_float16( model, keep_io_types=keep_io_types, diff --git a/tests/unit/optim/test_fp16.py b/tests/unit/optim/test_fp16.py index 527c1a7b8..e60f59379 100644 --- a/tests/unit/optim/test_fp16.py +++ b/tests/unit/optim/test_fp16.py @@ -15,10 +15,12 @@ from __future__ import annotations +import shutil + import numpy as np import onnx -from onnx import ModelProto, TensorProto, helper, numpy_helper +import winml.modelkit.onnx from winml.modelkit.quant.fp16 import convert_to_fp16 @@ -27,25 +29,29 @@ # ============================================================================= -def _build_simple_fp32_model() -> ModelProto: +def _build_simple_fp32_model() -> onnx.ModelProto: """Build a simple FP32 model: out = x + weight.""" - x = helper.make_tensor_value_info("x", TensorProto.FLOAT, [1, 4]) - out = helper.make_tensor_value_info("out", TensorProto.FLOAT, [1, 4]) - weight = numpy_helper.from_array(np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32), "weight") - add = helper.make_node("Add", ["x", "weight"], ["out"], name="add") - graph = helper.make_graph([add], "simple", [x], [out], [weight]) - return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + x = onnx.helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT, [1, 4]) + out = onnx.helper.make_tensor_value_info("out", onnx.TensorProto.FLOAT, [1, 4]) + weight = onnx.numpy_helper.from_array( + np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32), "weight" + ) + add = onnx.helper.make_node("Add", ["x", "weight"], ["out"], name="add") + graph = onnx.helper.make_graph([add], "simple", [x], [out], [weight]) + return onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 17)]) -def _build_multi_op_fp32_model() -> ModelProto: +def _build_multi_op_fp32_model() -> onnx.ModelProto: """Build a model with multiple ops: out = Relu(x + weight).""" - x = helper.make_tensor_value_info("x", TensorProto.FLOAT, [1, 4]) - out = helper.make_tensor_value_info("out", TensorProto.FLOAT, [1, 4]) - weight = numpy_helper.from_array(np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32), "weight") - add = helper.make_node("Add", ["x", "weight"], ["add_out"], name="add") - relu = helper.make_node("Relu", ["add_out"], ["out"], name="relu") - graph = helper.make_graph([add, relu], "multi_op", [x], [out], [weight]) - return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + x = onnx.helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT, [1, 4]) + out = onnx.helper.make_tensor_value_info("out", onnx.TensorProto.FLOAT, [1, 4]) + weight = onnx.numpy_helper.from_array( + np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32), "weight" + ) + add = onnx.helper.make_node("Add", ["x", "weight"], ["add_out"], name="add") + relu = onnx.helper.make_node("Relu", ["add_out"], ["out"], name="relu") + graph = onnx.helper.make_graph([add, relu], "multi_op", [x], [out], [weight]) + return onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 17)]) # ============================================================================= @@ -61,7 +67,9 @@ def test_converts_weights_to_fp16(self) -> None: model = _build_simple_fp32_model() result = convert_to_fp16(model) - has_fp16 = any(init.data_type == TensorProto.FLOAT16 for init in result.graph.initializer) + has_fp16 = any( + init.data_type == onnx.TensorProto.FLOAT16 for init in result.graph.initializer + ) assert has_fp16, "Expected at least one FP16 initializer after conversion" def test_path_conversion_uses_external_data_safe_shape_inference(self, tmp_path) -> None: @@ -79,7 +87,46 @@ def test_path_conversion_uses_external_data_safe_shape_inference(self, tmp_path) result = convert_to_fp16(model_path) - assert any(init.data_type == TensorProto.FLOAT16 for init in result.graph.initializer) + assert any(init.data_type == onnx.TensorProto.FLOAT16 for init in result.graph.initializer) + + def test_already_fp16_external_data_path_is_relocatable(self, tmp_path) -> None: + """Already-FP16 path input can be saved independently of its source sidecar.""" + source_dir = tmp_path / "source" + source_dir.mkdir() + source_path = source_dir / "source.onnx" + expected_weight = np.arange(1024, dtype=np.float16).reshape(1, 1024) + x = onnx.helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT16, [1, 1024]) + out = onnx.helper.make_tensor_value_info("out", onnx.TensorProto.FLOAT16, [1, 1024]) + weight = onnx.numpy_helper.from_array(expected_weight, "weight") + graph = onnx.helper.make_graph( + [onnx.helper.make_node("Add", ["x", "weight"], ["out"])], + "external_fp16", + [x], + [out], + [weight], + ) + model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 17)]) + onnx.save_model( + model, + str(source_path), + save_as_external_data=True, + all_tensors_to_one_file=True, + location="source.onnx.data", + size_threshold=0, + ) + + converted = convert_to_fp16(source_path) + destination_dir = tmp_path / "destination" + destination_path = destination_dir / "model.onnx" + winml.modelkit.onnx.save_onnx(converted, destination_path, threshold_size=0) + + assert (destination_dir / "model.onnx.data").is_file() + shutil.rmtree(source_dir) + onnx.checker.check_model(str(destination_path)) + relocated = onnx.load(str(destination_path)) + np.testing.assert_array_equal( + onnx.numpy_helper.to_array(relocated.graph.initializer[0]), expected_weight + ) def test_default_keeps_io_types(self) -> None: """Default keep_io_types=True preserves FP32 model I/O.""" @@ -87,9 +134,9 @@ def test_default_keeps_io_types(self) -> None: result = convert_to_fp16(model, keep_io_types=True) for inp in result.graph.input: - assert inp.type.tensor_type.elem_type == TensorProto.FLOAT + assert inp.type.tensor_type.elem_type == onnx.TensorProto.FLOAT for outp in result.graph.output: - assert outp.type.tensor_type.elem_type == TensorProto.FLOAT + assert outp.type.tensor_type.elem_type == onnx.TensorProto.FLOAT def test_keep_io_types_false_converts_io(self) -> None: """With keep_io_types=False, model I/O becomes FP16.""" @@ -97,9 +144,9 @@ def test_keep_io_types_false_converts_io(self) -> None: result = convert_to_fp16(model, keep_io_types=False) for inp in result.graph.input: - assert inp.type.tensor_type.elem_type == TensorProto.FLOAT16 + assert inp.type.tensor_type.elem_type == onnx.TensorProto.FLOAT16 for outp in result.graph.output: - assert outp.type.tensor_type.elem_type == TensorProto.FLOAT16 + assert outp.type.tensor_type.elem_type == onnx.TensorProto.FLOAT16 def test_preserves_model_structure(self) -> None: """FP16 conversion preserves graph structure (node count diff ≤ 2).""" @@ -131,13 +178,13 @@ def test_none_op_block_list_uses_ort_defaults(self) -> None: def test_skips_already_fp16_model(self) -> None: """If all floating-point initializers are already FP16, conversion is skipped.""" # Build a model with FP16 initializers directly - x = helper.make_tensor_value_info("x", TensorProto.FLOAT16, [1, 4]) - out = helper.make_tensor_value_info("out", TensorProto.FLOAT16, [1, 4]) + x = onnx.helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT16, [1, 4]) + out = onnx.helper.make_tensor_value_info("out", onnx.TensorProto.FLOAT16, [1, 4]) weight_data = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float16) - weight = numpy_helper.from_array(weight_data, "weight") - add = helper.make_node("Add", ["x", "weight"], ["out"], name="add") - graph = helper.make_graph([add], "fp16_model", [x], [out], [weight]) - model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + weight = onnx.numpy_helper.from_array(weight_data, "weight") + add = onnx.helper.make_node("Add", ["x", "weight"], ["out"], name="add") + graph = onnx.helper.make_graph([add], "fp16_model", [x], [out], [weight]) + model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 17)]) original_nodes = len(model.graph.node) result = convert_to_fp16(model) @@ -148,15 +195,15 @@ def test_skips_already_fp16_model(self) -> None: def test_skips_fp16_model_with_int_initializers(self) -> None: """FP16 model with non-float initializers (e.g. INT64 shapes) should still skip.""" - x = helper.make_tensor_value_info("x", TensorProto.FLOAT16, [1, 4]) - out = helper.make_tensor_value_info("out", TensorProto.FLOAT16, [1, 4]) + x = onnx.helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT16, [1, 4]) + out = onnx.helper.make_tensor_value_info("out", onnx.TensorProto.FLOAT16, [1, 4]) weight_data = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float16) - weight = numpy_helper.from_array(weight_data, "weight") + weight = onnx.numpy_helper.from_array(weight_data, "weight") # INT64 initializer (e.g., shape tensor) — should be ignored by skip logic - shape_tensor = numpy_helper.from_array(np.array([1, 4], dtype=np.int64), "shape") - add = helper.make_node("Add", ["x", "weight"], ["out"], name="add") - graph = helper.make_graph([add], "fp16_mixed", [x], [out], [weight, shape_tensor]) - model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + shape_tensor = onnx.numpy_helper.from_array(np.array([1, 4], dtype=np.int64), "shape") + add = onnx.helper.make_node("Add", ["x", "weight"], ["out"], name="add") + graph = onnx.helper.make_graph([add], "fp16_mixed", [x], [out], [weight, shape_tensor]) + model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 17)]) original_nodes = len(model.graph.node) result = convert_to_fp16(model)