feat(inference): support direct ONNX inpainting models

examples/recipes/opencv_inpainting_lama/cpu/cpu/inpainting_fp16_config.json

@@ -0,0 +1,42 @@
+{
+  "skip_optimize": true,
+  "export": null,
+  "optim": {},
+  "quant": {
+    "mode": "fp16",
+    "samples": 10,
+    "calibration_method": "minmax",
+    "weight_type": "uint8",
+    "activation_type": "uint8",
+    "per_channel": false,
+    "symmetric": false,
+    "weight_symmetric": null,
+    "activation_symmetric": null,
+    "save_calibration": false,
+    "distribution": "uniform",
+    "seed": null,
+    "calibration_load_path": null,
+    "calibration_save_path": null,
+    "op_types_to_quantize": null,
+    "nodes_to_exclude": null,
+    "fp16_keep_io_types": true,
+    "fp16_op_block_list": null
+  },
+  "compile": null,
+  "loader": {
+    "task": "inpainting"
+  },
+  "runtime": {
+    "pipeline": "inpainting",
+    "options": {
+      "image_input_name": "image",
+      "mask_input_name": "mask",
+      "output_name": "output",
+      "image_color_order": "bgr",
+      "image_value_range": [0, 1],
+      "mask_semantics": "nonzero-is-hole",
+      "output_color_order": "bgr",
+      "output_value_range": [0, 255]
+    }
+  }
+}

examples/recipes/opencv_inpainting_lama/cpu/cpu/inpainting_fp32_config.json

@@ -0,0 +1,23 @@
+{
+  "skip_optimize": true,
+  "export": null,
+  "optim": {},
+  "quant": null,
+  "compile": null,
+  "loader": {
+    "task": "inpainting"
+  },
+  "runtime": {
+    "pipeline": "inpainting",
+    "options": {
+      "image_input_name": "image",
+      "mask_input_name": "mask",
+      "output_name": "output",
+      "image_color_order": "bgr",
+      "image_value_range": [0, 1],
+      "mask_semantics": "nonzero-is-hole",
+      "output_color_order": "bgr",
+      "output_value_range": [0, 255]
+    }
+  }
+}

src/winml/modelkit/build/common.py

@@ -31,10 +31,12 @@
 def ensure_pre_quantized_stamped(
     config: WinMLBuildConfig, onnx_path: Path, *, force: bool = False
 ) -> None:
-    """Stamp ``config.skip_optimize`` (and clear ``config.quant``) once.
+    """Stamp ``config.skip_optimize`` and suppress incompatible quantization.
 
-    Sets ``config.skip_optimize = True`` and clears ``config.quant`` if the
-    input ONNX is already quantized.
+    Sets ``config.skip_optimize = True`` when the input ONNX is already
+    quantized. QDQ/QOperator quantization is suppressed, but an explicit FP16
+    conversion is preserved because it converts the graph's remaining float
+    tensors rather than re-quantizing integer weights.
 
     This is the **single defensive detection point** for the library entry
     points (``build_onnx_model``, ``build_hf_model``). When
@@ -50,20 +52,25 @@ def ensure_pre_quantized_stamped(
             ``is_quantized_onnx`` (used to honor the legacy
             ``skip_optimize=True`` kwarg from direct callers).
     """
+
+    def _clear_incompatible_quant() -> None:
+        if config.quant is not None and config.quant.mode != "fp16":
+            config.quant = None
+
     if config.skip_optimize:
-        config.quant = None
+        _clear_incompatible_quant()
         return
     if force:
         config.skip_optimize = True
-        config.quant = None
+        _clear_incompatible_quant()
         return
 
     if is_quantized_onnx(onnx_path):
         config.skip_optimize = True
-        config.quant = None
+        _clear_incompatible_quant()
         logger.info(
             "Pre-quantized model detected (QDQ or QOperator nodes present). "
-            "Skipping optimize + quantize stages."
+            "Skipping graph optimization and incompatible quantization."
         )
 
 

src/winml/modelkit/build/hf.py

@@ -254,14 +254,15 @@ def _name(base: str) -> str:
     is_pre_quantized = config.skip_optimize
 
     if is_pre_quantized:
-        logger.info("Skipping optimize + quantize stages (config.skip_optimize=True)")
+        logger.info("Skipping optimize stage (config.skip_optimize=True)")
         stages_skipped.append("optimize")
         # Skip the ORT-based graph optimization (no kernel for QOperator
         # ops like ConvInteger on the host EP). The autoconf re-optim/
         # analyze loop is disabled too -- ``run_optimize_analyze_loop``
         # forces ``max_optim_iterations=0`` when ``skip_optimize=True``,
         # so ``_run_analyze_loop`` is not invoked. The model still flows
-        # through later stages (quantize-skip + compile) for validation.
+        # through later stages. In particular, an explicit FP16 conversion
+        # still converts the graph's remaining float tensors before compile.
         current_path, _, analyze_iterations, analyze_unsupported_nodes, analyze_details = (
             run_optimize_analyze_loop(
                 model_path=current_path,
@@ -304,15 +305,14 @@ def _name(base: str) -> str:
     # =========================================================================
     # [4] QUANTIZE (optional — config.quant=None means skip)
     # =========================================================================
-    # No defensive ``is_quantized_onnx`` re-check here: when the model is
-    # pre-quantized, ``ensure_pre_quantized_stamped`` has already set
-    # ``config.quant = None`` at stage [3], so this branch naturally
-    # falls through to the ``quant is None`` skip path.
+    # No defensive ``is_quantized_onnx`` re-check here. The stage-[3]
+    # stamping suppresses incompatible integer quantization while preserving
+    # an explicit FP16 conversion of the graph's remaining float tensors.
     quant_result = None
-    if is_pre_quantized:
+    if is_pre_quantized and (config.quant is None or config.quant.mode != "fp16"):
         if "quantize" not in stages_skipped:
             stages_skipped.append("quantize")
-        logger.info("Quantize skipped (pre-quantized model)")
+        logger.info("Quantize skipped (incompatible with pre-quantized model)")
     elif config.quant is not None:
         logger.info("Quantizing model...")
         t0 = time.monotonic()
@@ -399,6 +399,7 @@ def _name(base: str) -> str:
         source="hf",
         model_id=model_label,
         task=task,
+        runtime=config.runtime.to_dict() if config.runtime is not None else None,
         cache_key=cache_key,
         config_hash=cache_key.rsplit("_", 1)[-1] if cache_key and "_" in cache_key else None,
         timestamp=datetime.datetime.now(datetime.timezone.utc).isoformat(),

src/winml/modelkit/build/onnx.py

@@ -165,14 +165,15 @@ def _name(base: str) -> str:
     is_pre_quantized = config.skip_optimize
 
     if is_pre_quantized:
-        logger.info("Skipping optimize + quantize stages (config.skip_optimize=True)")
+        logger.info("Skipping optimize stage (config.skip_optimize=True)")
         stages_skipped.append("optimize")
         # Skip the ORT-based graph optimization (no kernel for QOperator
         # ops like ConvInteger on the host EP). The autoconf re-optim/
         # analyze loop is disabled too -- ``run_optimize_analyze_loop``
         # forces ``max_optim_iterations=0`` when ``skip_optimize=True``,
         # so ``_run_analyze_loop`` is not invoked. The model still flows
-        # through later stages (quantize-skip + compile) for validation.
+        # through later stages. In particular, an explicit FP16 conversion
+        # still converts the graph's remaining float tensors before compile.
         current_path, _, analyze_iters, analyze_unsupported, analyze_details = (
             run_optimize_analyze_loop(
                 model_path=current_path,
@@ -210,16 +211,15 @@ def _name(base: str) -> str:
     # =========================================================================
     # [2] QUANTIZE (optional — config.quant=None means skip)
     # =========================================================================
-    # No defensive ``is_quantized_onnx`` re-check here: when the model is
-    # pre-quantized, ``ensure_pre_quantized_stamped`` has already set
-    # ``config.quant = None`` at stage [1], so this branch naturally
-    # falls through to the ``quant is None`` skip path.
+    # No defensive ``is_quantized_onnx`` re-check here. The stage-[1]
+    # stamping suppresses incompatible integer quantization while preserving
+    # an explicit FP16 conversion of the graph's remaining float tensors.
     quant_result = None
-    if is_pre_quantized:
+    if is_pre_quantized and (config.quant is None or config.quant.mode != "fp16"):
         # Already handled above -- skip quantize for pre-quantized models
         if "quantize" not in stages_skipped:
             stages_skipped.append("quantize")
-        logger.info("Quantize skipped (pre-quantized model)")
+        logger.info("Quantize skipped (incompatible with pre-quantized model)")
     elif config.quant is not None:
         logger.info("Quantizing model...")
         t0 = time.monotonic()
@@ -307,6 +307,8 @@ def _name(base: str) -> str:
         timestamp=datetime.datetime.now(datetime.timezone.utc).isoformat(),
         elapsed_seconds=round(elapsed, 3),
         final_artifact=final_path.name,
+        task=config.loader.task,
+        runtime=config.runtime.to_dict() if config.runtime is not None else None,
         stages=manifest_stages,
         analyze_iterations=analyze_iters,
         analyze_unsupported_node_count=analyze_unsupported,

src/winml/modelkit/commands/build.py

@@ -38,7 +38,7 @@
     print_stages_header,
 )
 from ..utils.logging import configure_logging
-from ..utils.model_input import ModelInputKind, classify_model_input
+from ..utils.model_input import ModelInputKind, classify_model_input, resolve_model_input
 
 
 if TYPE_CHECKING:
@@ -935,12 +935,7 @@ def build(
         # Hub-hosted ONNX (e.g. ``onnx-community/sam3-tracker-ONNX/onnx/...``)
         # is downloaded once and treated as a local .onnx file thereafter.
         model_input = None
-        if model:
-            model = cli_utils.normalize_model_arg(model)
-            if model:
-                model_input = classify_model_input(model)
-                if model_input.kind is ModelInputKind.INVALID:
-                    raise click.UsageError(model_input.error or f"Invalid model input: {model}")
+        source_model_input = None
 
         # Load or auto-generate config
         if config_file is not None:
@@ -949,6 +944,20 @@ def build(
                 no_quant=not quant,
                 no_compile=no_compile,
             )
+            if model:
+                direct_onnx = (
+                    not isinstance(config_or_configs, list) and config_or_configs.export is None
+                )
+                source_model_input = resolve_model_input(
+                    model,
+                    discover_repo_onnx=direct_onnx,
+                )
+                if source_model_input.kind is ModelInputKind.INVALID:
+                    raise click.UsageError(
+                        source_model_input.error or f"Invalid model input: {model}"
+                    )
+                model = source_model_input.local_path or model
+                model_input = classify_model_input(model)
             if export_overrides:
                 from ..config import merge_export_overrides
 
@@ -976,6 +985,12 @@ def build(
         else:
             if not model:
                 raise click.UsageError("-m/--model is required when -c is not provided.")
+            source_model_input = resolve_model_input(model, discover_repo_onnx=True)
+            if source_model_input.kind is ModelInputKind.INVALID:
+                invalid_message = source_model_input.error or f"Invalid model input: {model}"
+                raise click.UsageError(invalid_message)
+            model = source_model_input.local_path or model
+            model_input = classify_model_input(model)
             from ..config import generate_build_config
 
             # When ``model`` resolves to an .onnx file (either a local path or
@@ -1031,7 +1046,11 @@ def _patch_device(cfg: WinMLBuildConfig) -> None:
                 resolved_quant, _ = resolve_quant_compile_config(
                     device=device, precision=precision, ep=ep
                 )
-                if cfg.skip_optimize or not quant or resolved_quant is None:
+                if (
+                    not quant
+                    or resolved_quant is None
+                    or (cfg.skip_optimize and resolved_quant.mode != "fp16")
+                ):
                     cfg.quant = None
                 elif cfg.quant is None:
                     # Populate calibration identifiers from the loader/model
@@ -1103,6 +1122,12 @@ def _patch_device(cfg: WinMLBuildConfig) -> None:
         # on the key being present, matching the module-mode path which passes
         # allow_unsupported_nodes explicitly regardless of its value.
         extra_kwargs["allow_unsupported_nodes"] = allow_unsupported_nodes
+        source_kind = source_model_input.kind if source_model_input is not None else None
+        if source_kind is ModelInputKind.HUB_ONNX:
+            assert source_model_input is not None
+            extra_kwargs["source_model_id"] = source_model_input.hf_id
+            extra_kwargs["source_onnx_file"] = source_model_input.artifact_path
+            extra_kwargs["source_revision"] = source_model_input.revision
 
         # ---- OPTIMIZED (GENAI BUNDLE) EXPORT ----
         # ``--export-type optimized`` builds the family's registered
@@ -1779,7 +1804,7 @@ def _run_quantize_stage(
     from ..quant import quantize_onnx
     from ..utils.console import StageLive
 
-    if config.skip_optimize:
+    if config.skip_optimize and (config.quant is None or config.quant.mode != "fp16"):
         config.quant = None
         return current_path
 
@@ -2116,6 +2141,9 @@ def _build_onnx_pipeline(
 
     max_iters: int = extra_kwargs.pop("hack_max_optim_iterations", 3)
     allow_unsupported_nodes: bool = extra_kwargs.pop("allow_unsupported_nodes", False)
+    source_model_id: str | None = extra_kwargs.pop("source_model_id", None)
+    source_onnx_file: str | None = extra_kwargs.pop("source_onnx_file", None)
+    source_revision: str | None = extra_kwargs.pop("source_revision", None)
 
     # ── Validate + setup ─────────────────────────────────────────
     if not onnx_path.exists():
@@ -2204,4 +2232,24 @@ def _build_onnx_pipeline(
     if current_path != final_path:
         copy_onnx_model(current_path, final_path)
 
+    from ..utils import MANIFEST_FILENAME, WinMLManifest
+
+    manifest = WinMLManifest(
+        source="onnx",
+        model_id=source_model_id,
+        task=config.loader.task,
+        runtime=config.runtime.to_dict() if config.runtime is not None else None,
+        input_onnx=str(onnx_path),
+        final_artifact=final_path.name,
+        extras={
+            key: value
+            for key, value in {
+                "hub_onnx_file": source_onnx_file,
+                "hub_revision": source_revision,
+            }.items()
+            if value is not None
+        },
+    )
+    manifest.save(output_dir / MANIFEST_FILENAME)
+
     return stage_timings

src/winml/modelkit/config/__init__.py

@@ -26,6 +26,7 @@
 from .build import (
     SubmoduleClassNotFoundError,
     WinMLBuildConfig,
+    WinMLRuntimeConfig,
     generate_build_config,
     generate_hf_build_config,
     generate_onnx_build_config,
@@ -48,6 +49,7 @@
     "PrecisionPolicy",
     "SubmoduleClassNotFoundError",
     "WinMLBuildConfig",
+    "WinMLRuntimeConfig",
     "expand_precision",
     "extract_activation_bits",
     "extract_weight_bits",