diff --git a/scripts/e2e_eval/README.md b/scripts/e2e_eval/README.md index d858165d3..9030077ec 100644 --- a/scripts/e2e_eval/README.md +++ b/scripts/e2e_eval/README.md @@ -52,13 +52,20 @@ uv run python scripts/e2e_eval/build_registry.py --dry-run ### `run_eval.py` — Run Evaluation (recipe-driven perf + accuracy) For each filtered model the runner builds the model, runs `winml perf`, and — when -perf passes — runs `winml eval`. Builds prefer an **authored recipe** under +perf passes — runs `winml eval`. **Recipes drive the build on every device (they +carry the accuracy eval/dataset config), but quantized precision variants are +NPU-only.** On NPU, builds prefer an **authored recipe** under `examples/recipes//__config*.json` (built with `winml build -c`, once per precision variant that exists, e.g. `fp16` + `w8a16`); -models without a recipe fall back to `winml config` generation. The runner writes -one `eval_result.json` per `(model, task, precision)` containing facts only (perf -output + the winml-eval `metrics`/`dataset`). Delta/verdict grading against the -PyTorch baseline is done by the report site. +an NPU model without a recipe falls back to `winml config` and is expanded into +**two jobs — `w8a8` and `w8a16`** (an explicit per-model `precision` in the +registry overrides this and builds that single precision instead). On CPU/GPU (and +`auto`) the runner builds only the **non-quantized** recipe variants (e.g. `fp16`), +dropping quantized ones, and a model with no applicable recipe variant builds a +single `winml config` fallback. The runner writes one `eval_result.json` per +`(model, task, precision)` containing facts only (perf output + the winml-eval +`metrics`/`dataset`). +Delta/verdict grading against the PyTorch baseline is done by the report site. ```bash # Perf only (default), recipe-driven across precision variants @@ -101,8 +108,8 @@ uv run python scripts/e2e_eval/run_eval.py --update-baseline --eval-type accurac | `--registry` | `testsets/models_all.json` | Model registry file | | `--hf-model` | — | Single model (overrides registry) | | `--output-dir` | `eval_results/{date}` | Output directory | -| `--recipes-dir` | `examples/recipes` | Authored recipe configs; one build per precision variant, `winml config` fallback when a model has none | -| `--no-recipes` | off | Ignore recipes; build every model via `winml config` | +| `--recipes-dir` | `examples/recipes` | Authored recipe configs. NPU builds every precision variant (`winml config` `w8a8`+`w8a16` fallback when a model has none); CPU/GPU build only the non-quantized variants (e.g. `fp16`), else a `winml config` fallback | +| `--no-recipes` | off | Ignore recipes; build every model via `winml config` (on NPU still expands the `w8a8`+`w8a16` fallback) | | `--eval-type` | `perf` | `perf`, `accuracy`, or `both` (perf-gated accuracy) | | `--task` | — | Filter by HF task | | `--priority` | `P0 P1 P2` | Filter: one or more of `P0`, `P1`, `P2`, `P3` (e.g. `--priority P0 P1`). Pass `P3` explicitly to include P3 models. | @@ -244,17 +251,28 @@ no re-run. ### Recipe-driven builds & precision variants +**Recipes drive the build on every device, but multi-precision expansion into +quantized variants applies only when the target device is NPU** (`--device npu`). +On CPU/GPU (and `auto`) the runner uses recipes for their eval/dataset config but +builds only the **non-quantized** variants (e.g. `fp16`), dropping quantized ones; +a model with no applicable recipe variant builds a single `winml config` fallback +(precision by device policy — CPU/GPU omit `--precision`). + A *recipe* is an authored `winml build` config checked into `examples/recipes//` (slug = HF id with `/` → `_`), named `__config.json` (single model) or -`__config_.json` (composite, e.g. encoder/decoder). For -each `(model, task)` the runner builds **every precision variant** present on disk -(`fp16`, `w8a16`, `w8a8` — discovery is data-driven, so adding a `w8a8` recipe is -picked up automatically). Each variant becomes its own job and its own -`eval_result.json`. Models without a recipe fall back to `winml config` generation -(single precision by device policy). Because recipes set `compile: null`, the build -is EP-agnostic; the execution provider is applied at `winml perf`/`winml eval` time -via `--ep`/`--device`. +`__config_.json` (composite, e.g. encoder/decoder). On NPU, +for each `(model, task)` the runner builds **every precision variant** present on +disk (`fp16`, `w8a16`, `w8a8` — discovery is data-driven, so adding a `w8a8` recipe +is picked up automatically). Each variant becomes its own job and its own +`eval_result.json`. An NPU model **without** a recipe falls back to `winml config` +and is expanded into **two jobs, `w8a8` and `w8a16`**; an explicit per-model +`precision` field in the registry overrides this and builds that single precision. +A skip-quant EP (VitisAI, `--ep vitisai`) builds the model unquantized regardless +of precision, so this expansion is suppressed and the recipe-less model builds a +single `winml config` fallback instead. +Because recipes set `compile: null`, the build is EP-agnostic; the execution +provider is applied at `winml perf`/`winml eval` time via `--ep`/`--device`. ### Perf-gated accuracy diff --git a/scripts/e2e_eval/run_eval.py b/scripts/e2e_eval/run_eval.py index 48bb3250e..a8d32f4ec 100644 --- a/scripts/e2e_eval/run_eval.py +++ b/scripts/e2e_eval/run_eval.py @@ -6,9 +6,14 @@ """E2E evaluation runner — unified, recipe-driven perf + accuracy. Batch-builds models and runs winml perf, then (when perf passes) winml eval, -writing one unified eval_result.json per (model, task, precision). Builds prefer -an authored recipe under ``examples/recipes/`` (``winml build -c`` for each -precision variant), falling back to ``winml config`` for models without one. +writing one unified eval_result.json per (model, task, precision). Recipes drive +the build on every device (they carry the accuracy eval/dataset config), but +quantized precision variants are NPU-only. On ``--device npu`` the runner builds +every authored recipe variant under ``examples/recipes/`` (``winml build -c``), +and a recipe-less NPU model falls back to ``winml config`` expanded into ``w8a8`` ++ ``w8a16`` jobs (an explicit per-model precision overrides this). CPU/GPU build +only the non-quantized recipe variants (e.g. ``fp16``), or a single ``winml +config`` fallback when a model has no applicable recipe. The runner records facts only (perf output + the winml-eval metrics/dataset). The per-model report HTML — perf latency and the "Model Accuracy Report" delta @@ -90,6 +95,12 @@ _DEFAULT_SAMPLES = 1000 _DEFAULT_PRECISION_NPU = "w8a16" +# NPU fallback precisions used when a model has no authored recipe: the harness +# expands into one winml-config job per NPU quantization scheme (w8a8 + w8a16). +# An explicit per-model precision (``ModelEntry.precision``) overrides this. This +# expansion is NPU-only; off-NPU devices never build quantized variants. +_NPU_FALLBACK_PRECISIONS: tuple[str, ...] = ("w8a8", "w8a16") + # EPs whose eval track keeps the model unquantized (the "fp" variant) # rather than running winml's QDQ pass on top. This is an eval-setup # choice -- e.g. VitisAI / AMD Ryzen AI is benchmarked on the fp32/fp16 @@ -553,6 +564,11 @@ def _run_build( via ``-o`` (preserving the intermediate export/optimized/quantized ONNX) and skips compile (``--no-compile``) — no execution provider is required. Otherwise the build populates the global cache (``--use-cache``). + + ``precision`` is passed to both ``winml config`` and ``winml build``: since + we always pass ``--device``, ``winml build -c`` re-resolves quant from + device+precision and overwrites what the config baked in, so omitting it + would let the build revert to its auto default (npu → w8a16). """ composite_onnx = getattr(entry, "composite_onnx", None) if isinstance(composite_onnx, dict) and composite_onnx: @@ -652,6 +668,9 @@ def _run_build( else: build_args += ["--use-cache"] build_args += ["--device", device] + # See docstring: forward precision so the build doesn't revert to auto. + if precision: + build_args += ["--precision", precision] if ep: build_args += ["--ep", ep] # Mirror the --no-quant passed to winml config above so the build @@ -2301,25 +2320,71 @@ class EvalJob: ``variant`` is the authored recipe variant to build (one per precision), or ``None`` for the fallback path that generates the build config with - ``winml config``. Each job produces one ``eval_result.json``. + ``winml config``. ``fallback_precision`` pins the precision of a fallback + (``variant is None``) job — used on NPU to expand a recipe-less model into + one job per NPU quantization scheme (see :data:`_NPU_FALLBACK_PRECISIONS`); + it stays ``None`` for the single default-precision fallback. Each job + produces one ``eval_result.json``. """ entry: ModelEntry variant: RecipeVariant | None + fallback_precision: str | None = None @property def precision(self) -> str | None: - """Recipe precision (e.g. ``fp16``/``w8a16``), or None for the fallback.""" - return self.variant.precision if self.variant is not None else None + """Recipe precision, else pinned fallback, else the per-model precision. + Falling back to ``entry.precision`` keeps the result-dir slug, display + label, and recorded precision in sync with what ``_build_for_job`` + actually builds (it uses ``fallback_precision or entry.precision``) for + a recipe-less job that carries an explicit per-model precision. + """ + if self.variant is not None: + return self.variant.precision + return self.fallback_precision or self.entry.precision -def _build_jobs(entries: list[ModelEntry], recipes_dir: Path | None) -> list[EvalJob]: - """Expand entries into jobs, one per recipe precision variant. - For each entry, if ``recipes_dir`` is set and the model has recipe variants - for its task, emit one job per variant (precision); otherwise emit a single - fallback job (``variant=None``) that builds via ``winml config``. +def _is_quantized_precision(precision: str) -> bool: + """True if a recipe precision implies quantization. + + Off-NPU devices skip quantized recipe variants (see :func:`_build_jobs`). + Delegates to winml's precision policy so the classification never drifts + from the CLI (``fp16`` -> False, ``w8a16``/``w8a8`` -> True). + """ + from winml.modelkit.config.precision import is_quantized_precision + + return is_quantized_precision(precision) + + +def _build_jobs( + entries: list[ModelEntry], recipes_dir: Path | None, device: str, ep: str | None = None +) -> list[EvalJob]: + """Expand entries into jobs. Recipes apply on every device; quant is NPU-only. + + Recipes carry the accuracy eval/dataset config, so they are consulted + regardless of device -- but quantized recipe variants (``w8a16``/``w8a8``) + only make sense on the NPU. For each entry: + + * NPU + recipe variants on disk → one job per variant (``fp16`` + any + quantized). + * NPU + no recipe + no per-model precision → one fallback job per NPU + quantization scheme (:data:`_NPU_FALLBACK_PRECISIONS`, i.e. ``w8a8`` + + ``w8a16``). Skipped for skip-quant EPs (VitisAI), which build a single + unquantized fallback -- otherwise both precisions collapse onto the same + unquantized artifact (``_resolve_precision`` forces the flag off). + * NPU + no recipe + explicit per-model precision → a single fallback job + honoring that precision (``winml config``). + * non-NPU + non-quantized recipe variants → one job per such variant + (quantized variants are dropped). + * non-NPU with no applicable recipe variant → a single ``winml config`` + fallback job (``variant=None``). """ + npu = device == "npu" + # Skip-quant EPs (VitisAI) build the model unquantized regardless of + # precision, so the NPU multi-precision expansion would produce duplicate + # artifacts under distinct precision slugs -- fall back to a single job. + expand_npu_quant = npu and not _should_skip_winml_quant(ep) jobs: list[EvalJob] = [] for entry in entries: variants = ( @@ -2327,8 +2392,20 @@ def _build_jobs(entries: list[ModelEntry], recipes_dir: Path | None) -> list[Eva if recipes_dir is not None else [] ) + if not npu: + # Off-NPU still uses recipes for their eval config, but drops + # quantized variants -- quantization is an NPU-only concern here. + variants = [v for v in variants if not _is_quantized_precision(v.precision)] if variants: jobs.extend(EvalJob(entry, variant) for variant in variants) + elif expand_npu_quant and entry.precision is None: + # NPU without a recipe: build both NPU quantization precisions. An + # explicit per-model precision (e.g. fp16) skips this and is honored + # by the single-fallback branch below via _resolve_precision. + jobs.extend( + EvalJob(entry, None, fallback_precision=prec) + for prec in _NPU_FALLBACK_PRECISIONS + ) else: jobs.append(EvalJob(entry, None)) return jobs @@ -2352,10 +2429,13 @@ def _build_for_job( recipe_meta = build_result.get("meta_config") trust = _needs_trust_remote_code(recipe_meta) else: + # Fallback: winml config. A fallback job may pin an NPU precision + # (w8a8/w8a16); otherwise the per-model precision (if any) applies. + explicit_precision = job.fallback_precision or job.entry.precision build_result = _run_build( job.entry, args.device, - _resolve_precision(args.device, job.entry.precision, ep=args.ep), + _resolve_precision(args.device, explicit_precision, ep=args.ep), args.timeout, model_dir, ep=args.ep, @@ -2739,10 +2819,14 @@ def main() -> None: args.continue_run = True retry_types = {t.upper() for t in args.retry_failed} if args.retry_failed else set() - # Expand entries into per-precision recipe jobs; models without a recipe - # for their task fall back to a single winml-config build (variant=None). + # Expand entries into jobs. Recipes drive the build on every device (they + # carry the eval/dataset config), but quantized variants are NPU-only: on + # NPU a model builds one job per recipe precision variant (or, recipe-less + # with no per-model precision, one per NPU quantization scheme -- w8a8 + + # w8a16); off-NPU builds only the non-quantized recipe variants, else a + # single winml-config fallback (variant=None). recipes_dir = None if args.no_recipes else args.recipes_dir - jobs = _build_jobs(entries, recipes_dir) + jobs = _build_jobs(entries, recipes_dir, args.device, ep=args.ep) total_jobs = len(jobs) safe_print(f"E2E Evaluation: {len(entries)} models -> {total_jobs} jobs -> {output_dir}") @@ -2751,8 +2835,16 @@ def main() -> None: f"Device: {args.device} | EP: {ep_label} | Timeout: {args.timeout}s | Eval: {args.eval_type}" ) safe_print(f"Disk free: {_get_disk_free_gb():.1f} GB") - if recipes_dir is not None: - safe_print(f"Recipes: {recipes_dir} (winml config fallback when a model has none)") + if recipes_dir is not None and args.device == "npu": + safe_print( + f"Recipes: {recipes_dir} " + f"(NPU; winml config {'+'.join(_NPU_FALLBACK_PRECISIONS)} fallback when a model has none)" + ) + elif recipes_dir is not None: + safe_print( + f"Recipes: {recipes_dir} " + f"(non-NPU '{args.device}'; non-quantized variants only, winml config fallback)" + ) else: safe_print("Recipes: disabled (winml config for all builds)") if args.clean_cache: diff --git a/tests/unit/eval/test_run_eval_script.py b/tests/unit/eval/test_run_eval_script.py index b6053e246..e21f17022 100644 --- a/tests/unit/eval/test_run_eval_script.py +++ b/tests/unit/eval/test_run_eval_script.py @@ -373,6 +373,67 @@ def fake_subprocess(args, _timeout): assert config_call[idx + 2] == "--overwrite", config_call +class TestRunBuildPrecisionForwarding: + """``_run_build`` must forward ``--precision`` to both ``winml config`` and + ``winml build``. + + Regression guard: since the harness always passes ``--device``, + ``winml build -c`` re-resolves quant from device+precision and overwrites + the config's quant. Omitting ``--precision`` let it revert to the auto + default (npu → w8a16), so w8a8 and w8a16 fallback jobs collided on one + cached artifact. + """ + + @staticmethod + def _make_entry(): + entry = MagicMock() + entry.hf_id = "google-bert/bert-base-uncased" + entry.task = "text-classification" + entry.perf_args = [] + return entry + + def _invoke(self, run_eval, precision, tmp_path, ep=None): + entry = self._make_entry() + (tmp_path / "build_config.json").write_text("{}", encoding="utf-8") + captured: list[list[str]] = [] + + def fake_subprocess(args, _timeout): + captured.append(list(args)) + stdout = "" if "config" in args else "Build cache: model.onnx" + return { + "exit_code": 0, + "stdout": stdout, + "stderr": "", + "elapsed": 0.1, + "command": " ".join(args), + } + + with ( + patch.object(run_eval, "_run_subprocess", side_effect=fake_subprocess), + patch.object(run_eval, "_extract_onnx_path", return_value=str(tmp_path / "model.onnx")), + ): + run_eval._run_build(entry, "npu", precision, 300, tmp_path, ep=ep) + return captured + + def test_precision_forwarded_to_both_config_and_build(self, run_eval, tmp_path): + calls = self._invoke(run_eval, "w8a8", tmp_path) + config_call = next(args for args in calls if "config" in args) + build_call = next(args for args in calls if "build" in args) + for call in (config_call, build_call): + assert "--precision" in call, call + assert call[call.index("--precision") + 1] == "w8a8", call + + def test_distinct_precisions_forwarded_distinctly(self, run_eval, tmp_path): + build_a = next(a for a in self._invoke(run_eval, "w8a8", tmp_path) if "build" in a) + build_b = next(a for a in self._invoke(run_eval, "w8a16", tmp_path) if "build" in a) + assert build_a[build_a.index("--precision") + 1] == "w8a8", build_a + assert build_b[build_b.index("--precision") + 1] == "w8a16", build_b + + def test_precision_omitted_from_build_when_none(self, run_eval, tmp_path): + build_call = next(a for a in self._invoke(run_eval, None, tmp_path) if "build" in a) + assert "--precision" not in build_call, build_call + + class TestFeedVersionForCombo: """``_feed_version_for`` embeds the EP/device combo after the run-stamp.""" @@ -766,7 +827,9 @@ def test_no_trust_without_build_script(self, run_eval, tmp_path): class TestBuildJobs: - """``_build_jobs`` expands entries into one job per recipe precision variant.""" + """``_build_jobs`` uses recipes on every device but drops quantized variants + off-NPU. + """ def _make_single_recipe(self, recipes_dir: Path, slug: str, task: str, precisions: list[str]): model_dir = recipes_dir / slug @@ -776,27 +839,88 @@ def _make_single_recipe(self, recipes_dir: Path, slug: str, task: str, precision json.dumps({"eval": {"task": task, "dataset": {"path": "x"}}}), encoding="utf-8" ) - def test_recipe_expands_to_one_job_per_precision(self, run_eval, tmp_path): + def test_npu_recipe_expands_to_one_job_per_precision(self, run_eval, tmp_path): self._make_single_recipe( tmp_path, "microsoft_resnet-50", "image-classification", ["fp16", "w8a16"] ) entry = _entry() - jobs = run_eval._build_jobs([entry], tmp_path) + jobs = run_eval._build_jobs([entry], tmp_path, "npu") assert [j.precision for j in jobs] == ["fp16", "w8a16"] assert all(j.entry is entry for j in jobs) - def test_no_recipe_falls_back_to_single_job(self, run_eval, tmp_path): + def test_non_npu_keeps_only_non_quantized_variants(self, run_eval, tmp_path): + # A recipe with fp16 + w8a16: off-NPU keeps fp16 (for its eval config) + # and drops the quantized w8a16 variant. + self._make_single_recipe( + tmp_path, "microsoft_resnet-50", "image-classification", ["fp16", "w8a16"] + ) + entry = _entry() + for device in ("cpu", "gpu", "auto"): + jobs = run_eval._build_jobs([entry], tmp_path, device) + assert [j.precision for j in jobs] == ["fp16"] + assert jobs[0].variant is not None + assert all(j.entry is entry for j in jobs) + + def test_non_npu_recipe_only_quantized_falls_back(self, run_eval, tmp_path): + # A recipe with no non-quantized variant leaves nothing to run off-NPU, + # so the model builds a single winml-config fallback. + self._make_single_recipe( + tmp_path, "microsoft_resnet-50", "image-classification", ["w8a16"] + ) + entry = _entry() + jobs = run_eval._build_jobs([entry], tmp_path, "cpu") + assert len(jobs) == 1 + assert jobs[0].variant is None + assert jobs[0].precision is None + + def test_npu_no_recipe_expands_to_w8a8_and_w8a16(self, run_eval, tmp_path): + entry = _entry("some/model", "text-classification") + jobs = run_eval._build_jobs([entry], tmp_path, "npu") + assert [j.precision for j in jobs] == ["w8a8", "w8a16"] + assert all(j.variant is None for j in jobs) + assert all(j.entry is entry for j in jobs) + + def test_npu_no_recipe_honors_explicit_precision(self, run_eval, tmp_path): + # An explicit per-model precision suppresses the w8a8+w8a16 expansion; + # the single fallback job reports that precision so its slug/label match + # the built artifact. + entry = _entry("some/model", "text-classification") + entry.precision = "fp16" + jobs = run_eval._build_jobs([entry], tmp_path, "npu") + assert len(jobs) == 1 + assert jobs[0].variant is None + assert jobs[0].precision == "fp16" + + def test_npu_skip_quant_ep_no_recipe_single_fallback(self, run_eval, tmp_path): + # A skip-quant EP (VitisAI) builds the model unquantized regardless of + # precision, so the w8a8+w8a16 expansion is suppressed to avoid two jobs + # collapsing onto the same unquantized artifact. entry = _entry("some/model", "text-classification") - jobs = run_eval._build_jobs([entry], tmp_path) + jobs = run_eval._build_jobs([entry], tmp_path, "npu", ep="vitisai") assert len(jobs) == 1 assert jobs[0].variant is None assert jobs[0].precision is None - def test_recipes_disabled_yields_fallback_jobs(self, run_eval, tmp_path): + def test_non_npu_no_recipe_single_fallback(self, run_eval, tmp_path): + entry = _entry("some/model", "text-classification") + jobs = run_eval._build_jobs([entry], tmp_path, "cpu") + assert len(jobs) == 1 + assert jobs[0].variant is None + assert jobs[0].precision is None + + def test_npu_recipes_disabled_yields_precision_fallback(self, run_eval, tmp_path): + self._make_single_recipe(tmp_path, "microsoft_resnet-50", "image-classification", ["fp16"]) + entry = _entry() + # recipes_dir=None disables recipe discovery entirely; NPU still expands + # the recipe-less model into the w8a8+w8a16 fallback jobs. + jobs = run_eval._build_jobs([entry], None, "npu") + assert [j.precision for j in jobs] == ["w8a8", "w8a16"] + assert all(j.variant is None for j in jobs) + + def test_non_npu_recipes_disabled_single_fallback(self, run_eval, tmp_path): self._make_single_recipe(tmp_path, "microsoft_resnet-50", "image-classification", ["fp16"]) entry = _entry() - # recipes_dir=None disables recipe discovery entirely. - jobs = run_eval._build_jobs([entry], None) + jobs = run_eval._build_jobs([entry], None, "cpu") assert len(jobs) == 1 assert jobs[0].variant is None