Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
{
"export": {
"opset_version": 17,
"batch_size": 1,
"export_params": true,
"do_constant_folding": true,
"verbose": false,
"dynamo": false,
"enable_hierarchy_tags": true,
"clean_onnx": false,
"hierarchy_tag_format": "full",
"input_tensors": [
{
"name": "pixel_values",
"dtype": "float32",
"shape": [
1,
3,
256,
192
],
"value_range": [
-2.1179039478302,
2.640000343322754
]
}
],
"output_tensors": [
{
"name": "heatmaps"
}
]
},
"optim": {},
"quant": {
"mode": "fp16",
"samples": 10,
"calibration_method": "minmax",
"weight_type": "uint8",
"activation_type": "uint8",
"per_channel": false,
"symmetric": false,
"weight_symmetric": null,
"activation_symmetric": null,
"save_calibration": false,
"distribution": "uniform",
"seed": null,
"calibration_load_path": null,
"calibration_save_path": null,
"op_types_to_quantize": null,
"nodes_to_exclude": null,
"task": "keypoint-detection",
"model_id": "stanfordmimi/synthpose-vitpose-base-hf",
"model_type": "vitpose",
"fp16_keep_io_types": true,
"fp16_op_block_list": null
},
"compile": null,
"loader": {
"task": "keypoint-detection",
"model_class": "VitPoseForPoseEstimation",
"model_type": "vitpose"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
{
"export": {
"opset_version": 17,
"batch_size": 1,
"export_params": true,
"do_constant_folding": true,
"verbose": false,
"dynamo": false,
"enable_hierarchy_tags": true,
"clean_onnx": false,
"hierarchy_tag_format": "full",
"input_tensors": [
{
"name": "pixel_values",
"dtype": "float32",
"shape": [
1,
3,
256,
192
],
"value_range": [
-2.1179039478302,
2.640000343322754
]
}
],
"output_tensors": [
{
"name": "heatmaps"
}
]
},
"optim": {},
"quant": null,
"compile": null,
"loader": {
"task": "keypoint-detection",
"model_class": "VitPoseForPoseEstimation",
"model_type": "vitpose"
}
}
6 changes: 6 additions & 0 deletions src/winml/modelkit/commands/perf.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ def generate_random_inputs(
symbolic_shapes = io_config.get("input_symbolic_shapes") or [
[None] * len(s) for s in io_config["input_shapes"]
]
value_ranges = io_config.get("input_value_ranges") or {}
overrides = shape_config or {}

specs: dict[str, dict[str, Any]] = {}
Expand Down Expand Up @@ -290,6 +291,11 @@ def generate_random_inputs(
"dtype": gen_dtype,
"shape": list(resolved_shape),
}
if name in value_ranges:
low, high = value_ranges[name]
# Persisted InputTensorSpec ranges are [low, high), while the
# legacy NumPy generator treats integer maxima as inclusive.
specs[name]["range"] = [int(low), int(high) - 1] if gen_dtype == "int" else [low, high]

return generate_dummy_inputs_from_specs(specs)

Expand Down
111 changes: 111 additions & 0 deletions tests/unit/commands/test_perf_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -849,6 +849,117 @@ def test_symbolic_override_rejects_non_integer_float_value(self) -> None:
)


class TestGenerateRandomInputs:
def test_honors_negative_float_value_range(self) -> None:
"""A configured range outside [0, 1) must replace the float fallback."""
import numpy as np

from winml.modelkit.commands.perf import generate_random_inputs

inputs = generate_random_inputs(
{
"input_names": ["features"],
"input_shapes": [[4, 32]],
"input_types": ["float32"],
"input_value_ranges": {"features": [-9.0, -8.0]},
}
)

assert np.all(inputs["features"] >= -9.0)
assert np.all(inputs["features"] < -8.0)

def test_honors_singleton_exclusive_integer_value_range(self) -> None:
"""The canonical [7, 8) range must generate only the value seven."""
import numpy as np

from winml.modelkit.commands.perf import generate_random_inputs

inputs = generate_random_inputs(
{
"input_names": ["category_ids"],
"input_shapes": [[2, 512]],
"input_types": ["int64"],
"input_value_ranges": {"category_ids": [7, 8]},
}
)

assert inputs["category_ids"].dtype == np.int64
assert np.all(inputs["category_ids"] == 7)

def test_adapts_integer_high_exclusive_for_legacy_generator(self) -> None:
"""The persisted exclusive high must not become a legal integer value."""
from winml.modelkit.commands.perf import generate_random_inputs

with patch(
"winml.modelkit.core.generate_dummy_inputs_from_specs",
return_value={"category_ids": MagicMock()},
) as mock_generate:
generate_random_inputs(
{
"input_names": ["category_ids"],
"input_shapes": [[1, 8]],
"input_types": ["int32"],
"input_value_ranges": {"category_ids": [3, 7]},
}
)

assert mock_generate.call_args.args[0]["category_ids"]["range"] == [3, 6]

def test_unspecified_float_input_keeps_legacy_fallback(self) -> None:
"""Ranges are per input; unspecified floats remain in [0, 1)."""
import numpy as np

from winml.modelkit.commands.perf import generate_random_inputs

inputs = generate_random_inputs(
{
"input_names": ["ranged", "default"],
"input_shapes": [[4, 32], [4, 32]],
"input_types": ["float32", "float32"],
"input_value_ranges": {"ranged": [-2.0, -1.0]},
}
)

assert np.all(inputs["ranged"] >= -2.0)
assert np.all(inputs["ranged"] < -1.0)
assert np.all(inputs["default"] >= 0.0)
assert np.all(inputs["default"] < 1.0)

def test_persisted_build_config_range_reaches_perf_generation(self, tmp_path: Path) -> None:
"""WinMLSession must carry a co-located build range into perf inputs."""
import numpy as np
from onnx import TensorProto, helper, save

from winml.modelkit.commands.perf import generate_random_inputs
from winml.modelkit.session import WinMLSession

input_info = helper.make_tensor_value_info("features", TensorProto.FLOAT, [1, 16])
output_info = helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 16])
graph = helper.make_graph(
[helper.make_node("Identity", ["features"], ["output"])],
"range_forwarding",
[input_info],
[output_info],
)
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)])
model.ir_version = 9
model_path = tmp_path / "model.onnx"
save(model, model_path)
(tmp_path / "winml_build_config.json").write_text(
json.dumps(
{"export": {"input_tensors": [{"name": "features", "value_range": [-9.0, -8.0]}]}}
),
encoding="utf-8",
)

session = WinMLSession(model_path, device="cpu")
assert session.io_config["input_value_ranges"] == {"features": [-9.0, -8.0]}

inputs = generate_random_inputs(session.io_config)
assert np.all(inputs["features"] >= -9.0)
assert np.all(inputs["features"] < -8.0)


class TestEffectiveBatchSize:
"""Throughput must scale by the batch the session actually ran.

Expand Down
Loading