diff --git a/examples/recipes/stanfordmimi_synthpose-vitpose-base-hf/cpu/cpu/keypoint-detection_fp16_config.json b/examples/recipes/stanfordmimi_synthpose-vitpose-base-hf/cpu/cpu/keypoint-detection_fp16_config.json new file mode 100644 index 000000000..a2dc6fe9d --- /dev/null +++ b/examples/recipes/stanfordmimi_synthpose-vitpose-base-hf/cpu/cpu/keypoint-detection_fp16_config.json @@ -0,0 +1,64 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 256, + 192 + ], + "value_range": [ + -2.1179039478302, + 2.640000343322754 + ] + } + ], + "output_tensors": [ + { + "name": "heatmaps" + } + ] + }, + "optim": {}, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "task": "keypoint-detection", + "model_id": "stanfordmimi/synthpose-vitpose-base-hf", + "model_type": "vitpose", + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "keypoint-detection", + "model_class": "VitPoseForPoseEstimation", + "model_type": "vitpose" + } +} diff --git a/examples/recipes/stanfordmimi_synthpose-vitpose-base-hf/cpu/cpu/keypoint-detection_fp32_config.json b/examples/recipes/stanfordmimi_synthpose-vitpose-base-hf/cpu/cpu/keypoint-detection_fp32_config.json new file mode 100644 index 000000000..cb03b176a --- /dev/null +++ b/examples/recipes/stanfordmimi_synthpose-vitpose-base-hf/cpu/cpu/keypoint-detection_fp32_config.json @@ -0,0 +1,42 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 256, + 192 + ], + "value_range": [ + -2.1179039478302, + 2.640000343322754 + ] + } + ], + "output_tensors": [ + { + "name": "heatmaps" + } + ] + }, + "optim": {}, + "quant": null, + "compile": null, + "loader": { + "task": "keypoint-detection", + "model_class": "VitPoseForPoseEstimation", + "model_type": "vitpose" + } +} diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 30d533699..fdb911929 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -262,6 +262,7 @@ def generate_random_inputs( symbolic_shapes = io_config.get("input_symbolic_shapes") or [ [None] * len(s) for s in io_config["input_shapes"] ] + value_ranges = io_config.get("input_value_ranges") or {} overrides = shape_config or {} specs: dict[str, dict[str, Any]] = {} @@ -290,6 +291,11 @@ def generate_random_inputs( "dtype": gen_dtype, "shape": list(resolved_shape), } + if name in value_ranges: + low, high = value_ranges[name] + # Persisted InputTensorSpec ranges are [low, high), while the + # legacy NumPy generator treats integer maxima as inclusive. + specs[name]["range"] = [int(low), int(high) - 1] if gen_dtype == "int" else [low, high] return generate_dummy_inputs_from_specs(specs) diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index be4984977..9c8908185 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -849,6 +849,117 @@ def test_symbolic_override_rejects_non_integer_float_value(self) -> None: ) +class TestGenerateRandomInputs: + def test_honors_negative_float_value_range(self) -> None: + """A configured range outside [0, 1) must replace the float fallback.""" + import numpy as np + + from winml.modelkit.commands.perf import generate_random_inputs + + inputs = generate_random_inputs( + { + "input_names": ["features"], + "input_shapes": [[4, 32]], + "input_types": ["float32"], + "input_value_ranges": {"features": [-9.0, -8.0]}, + } + ) + + assert np.all(inputs["features"] >= -9.0) + assert np.all(inputs["features"] < -8.0) + + def test_honors_singleton_exclusive_integer_value_range(self) -> None: + """The canonical [7, 8) range must generate only the value seven.""" + import numpy as np + + from winml.modelkit.commands.perf import generate_random_inputs + + inputs = generate_random_inputs( + { + "input_names": ["category_ids"], + "input_shapes": [[2, 512]], + "input_types": ["int64"], + "input_value_ranges": {"category_ids": [7, 8]}, + } + ) + + assert inputs["category_ids"].dtype == np.int64 + assert np.all(inputs["category_ids"] == 7) + + def test_adapts_integer_high_exclusive_for_legacy_generator(self) -> None: + """The persisted exclusive high must not become a legal integer value.""" + from winml.modelkit.commands.perf import generate_random_inputs + + with patch( + "winml.modelkit.core.generate_dummy_inputs_from_specs", + return_value={"category_ids": MagicMock()}, + ) as mock_generate: + generate_random_inputs( + { + "input_names": ["category_ids"], + "input_shapes": [[1, 8]], + "input_types": ["int32"], + "input_value_ranges": {"category_ids": [3, 7]}, + } + ) + + assert mock_generate.call_args.args[0]["category_ids"]["range"] == [3, 6] + + def test_unspecified_float_input_keeps_legacy_fallback(self) -> None: + """Ranges are per input; unspecified floats remain in [0, 1).""" + import numpy as np + + from winml.modelkit.commands.perf import generate_random_inputs + + inputs = generate_random_inputs( + { + "input_names": ["ranged", "default"], + "input_shapes": [[4, 32], [4, 32]], + "input_types": ["float32", "float32"], + "input_value_ranges": {"ranged": [-2.0, -1.0]}, + } + ) + + assert np.all(inputs["ranged"] >= -2.0) + assert np.all(inputs["ranged"] < -1.0) + assert np.all(inputs["default"] >= 0.0) + assert np.all(inputs["default"] < 1.0) + + def test_persisted_build_config_range_reaches_perf_generation(self, tmp_path: Path) -> None: + """WinMLSession must carry a co-located build range into perf inputs.""" + import numpy as np + from onnx import TensorProto, helper, save + + from winml.modelkit.commands.perf import generate_random_inputs + from winml.modelkit.session import WinMLSession + + input_info = helper.make_tensor_value_info("features", TensorProto.FLOAT, [1, 16]) + output_info = helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 16]) + graph = helper.make_graph( + [helper.make_node("Identity", ["features"], ["output"])], + "range_forwarding", + [input_info], + [output_info], + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + model.ir_version = 9 + model_path = tmp_path / "model.onnx" + save(model, model_path) + (tmp_path / "winml_build_config.json").write_text( + json.dumps( + {"export": {"input_tensors": [{"name": "features", "value_range": [-9.0, -8.0]}]}} + ), + encoding="utf-8", + ) + + session = WinMLSession(model_path, device="cpu") + assert session.io_config["input_value_ranges"] == {"features": [-9.0, -8.0]} + + inputs = generate_random_inputs(session.io_config) + assert np.all(inputs["features"] >= -9.0) + assert np.all(inputs["features"] < -8.0) + + class TestEffectiveBatchSize: """Throughput must scale by the batch the session actually ran.