From e970888618a07829e820f59d807630f703bc7c4a Mon Sep 17 00:00:00 2001 From: triasha72 Date: Tue, 25 Aug 2026 20:58:13 -0400 Subject: [PATCH 1/4] Build honest cross-runtime acceptance matrix --- .github/workflows/ci.yml | 6 + .gitignore | 2 + README.md | 6 + reports/portfolio_acceptance.json | 81 ++++++++++ reports/portfolio_acceptance.md | 24 +++ scripts/build_portfolio_acceptance.py | 206 ++++++++++++++++++++++++++ tests/test_portfolio_acceptance.py | 33 +++++ 7 files changed, 358 insertions(+) create mode 100644 reports/portfolio_acceptance.json create mode 100644 reports/portfolio_acceptance.md create mode 100644 scripts/build_portfolio_acceptance.py create mode 100644 tests/test_portfolio_acceptance.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf5fae5..744cbe3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,12 @@ jobs: run: | python scripts/validate_tracked_qualcomm_int8.py + - name: Build cross-runtime portfolio acceptance matrix + run: | + python scripts/build_portfolio_acceptance.py \ + --output-json /tmp/portfolio-acceptance.json \ + --output-markdown /tmp/portfolio-acceptance.md + release-acceptance: name: End-to-end release acceptance runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 4629421..17e3fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ data/processed/* reports/* !reports/.gitkeep !reports/qualcomm_qnn_v0_1.json +!reports/portfolio_acceptance.json +!reports/portfolio_acceptance.md # Deployment artifacts *.onnx diff --git a/README.md b/README.md index 90c388b..26f7e57 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,12 @@ Cross-provider timings should not be interpreted as a direct hardware ranking. ## Qualcomm QNN / Snapdragon deployment +The consolidated [portfolio acceptance matrix](reports/portfolio_acceptance.md) +separates validated CI/device lanes from evidence still pending. It also checks +whether the model hash in the historical Qualcomm report matches the current +ONNX artifact; the current mismatch is intentionally visible and must be +resolved by restoring the reported model or re-running AI Hub. + EdgeGenBench now includes a validated Qualcomm QNN deployment path for the compact neural surrogate. diff --git a/reports/portfolio_acceptance.json b/reports/portfolio_acceptance.json new file mode 100644 index 0000000..3860fb8 --- /dev/null +++ b/reports/portfolio_acceptance.json @@ -0,0 +1,81 @@ +{ + "schema_version": 1, + "project": "EdgeGenBench", + "lanes": { + "native_cpp": { + "status": "validated_in_ci", + "claim": "C++17 reference runtime, tests, CLI, and fused preprocessing acceptance." + }, + "android_reference": { + "status": "validated_physical_device", + "device": "Samsung SM-A356E", + "report": "reports/android_sm_a356e_reference_10_run_v0_1_4.md", + "claim": "Reference JNI/application measurements; not QNN." + }, + "qualcomm_ai_hub_qnn": { + "status": "tracked_ai_hub_report_model_provenance_mismatch", + "device": "Snapdragon 8 Elite QRD", + "backend": "QNN HTP", + "qairt_version": "2.45.0.260326154327", + "context_model_id": "mnl7771jm", + "link_job_id": "jp16x3285", + "graphs": [ + { + "graph": "edgegenbench_batch1", + "batch_size": 1, + "latency_ms": 0.038, + "throughput_samples_per_second": 26315.78947368421, + "peak_memory_bytes": 122822656, + "compute_units": { + "NPU": 9 + }, + "max_normalized_drift": 0.0036358769310459917, + "profile_job_id": "jp4308d85", + "inference_job_id": "jpxx2m63p" + }, + { + "graph": "edgegenbench_batch32", + "batch_size": 32, + "latency_ms": 0.034, + "throughput_samples_per_second": 941176.4705882353, + "peak_memory_bytes": 122888192, + "compute_units": { + "NPU": 9 + }, + "max_normalized_drift": 0.003222448271801646, + "profile_job_id": "jgnk8xmkg", + "inference_job_id": "jprwj9205" + }, + { + "graph": "edgegenbench_batch256", + "batch_size": 256, + "latency_ms": 0.057, + "throughput_samples_per_second": 4491228.070175438, + "peak_memory_bytes": 123211776, + "compute_units": { + "NPU": 9 + }, + "max_normalized_drift": 0.003222448271801646, + "profile_job_id": "jp2enj9rp", + "inference_job_id": "jpym0nj8p" + } + ], + "reported_source_model_sha256": "40f588b329b98fdaa38a7eda202fc89573fce5e91a4193f0a0a93b3142c0382f", + "current_source_model_sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", + "source_model_matches_repository": false, + "claim_boundary": "Physical AI Hub model profiling; not Android APK end-to-end latency." + }, + "android_qnn_apk": { + "status": "implementation_complete_evidence_pending", + "claim": "Build/JNI/capture paths exist; requires a supported Snapdragon APK run." + }, + "android_16kb_runtime": { + "status": "packaging_validated_runtime_pending", + "claim": "ELF/APK alignment passes; runtime PAGE_SIZE=16384 evidence is pending." + }, + "power": { + "status": "not_measured", + "claim": "No power-savings claim is made without a named calibrated tool." + } + } +} diff --git a/reports/portfolio_acceptance.md b/reports/portfolio_acceptance.md new file mode 100644 index 0000000..8484913 --- /dev/null +++ b/reports/portfolio_acceptance.md @@ -0,0 +1,24 @@ +# EdgeGenBench portfolio acceptance + +| Evidence lane | Status | Claim boundary | +|---|---|---| +| `native_cpp` | `validated_in_ci` | C++17 reference runtime, tests, CLI, and fused preprocessing acceptance. | +| `android_reference` | `validated_physical_device` | Reference JNI/application measurements; not QNN. | +| `qualcomm_ai_hub_qnn` | `tracked_ai_hub_report_model_provenance_mismatch` | Physical AI Hub model profiling; not Android APK end-to-end latency. | +| `android_qnn_apk` | `implementation_complete_evidence_pending` | Build/JNI/capture paths exist; requires a supported Snapdragon APK run. | +| `android_16kb_runtime` | `packaging_validated_runtime_pending` | ELF/APK alignment passes; runtime PAGE_SIZE=16384 evidence is pending. | +| `power` | `not_measured` | No power-savings claim is made without a named calibrated tool. | + +## Validated Qualcomm QNN results + +Device: **Snapdragon 8 Elite QRD**; backend: **QNN HTP**; QAIRT: `2.45.0.260326154327`. +Source-model provenance match: **False**. + +| Batch | AI Hub latency (ms) | Throughput (samples/s) | Peak memory (bytes) | Placement | Max normalized drift | +|---:|---:|---:|---:|---|---:| +| 1 | 0.038000 | 26315.789 | 122822656 | NPU × 9 | 0.003635877 | +| 32 | 0.034000 | 941176.471 | 122888192 | NPU × 9 | 0.003222448 | +| 256 | 0.057000 | 4491228.070 | 123211776 | NPU × 9 | 0.003222448 | + +AI Hub measurements are physical-device model profiles, not Android application end-to-end timings. They cannot be promoted as current-model acceptance until the reported model artifact is restored or the current model is re-profiled. +Power remains unmeasured. The two pending proof items are a supported-device QNN APK run and a runtime page size of 16384 bytes. diff --git a/scripts/build_portfolio_acceptance.py b/scripts/build_portfolio_acceptance.py new file mode 100644 index 0000000..c4b39a4 --- /dev/null +++ b/scripts/build_portfolio_acceptance.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Validate tracked deployment evidence and build the final portfolio acceptance matrix.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def _load_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def validate_ai_hub_qnn(report_path: Path, repository_root: Path) -> dict[str, Any]: + report = _load_json(report_path) + if report.get("schema_version") != "0.2": + raise ValueError("unsupported Qualcomm AI Hub report") + source = report.get("source_model") + hardware = report.get("hardware") + linked = report.get("linked_multigraph") + if ( + not isinstance(source, dict) + or not isinstance(hardware, dict) + or not isinstance(linked, dict) + ): + raise ValueError("incomplete Qualcomm AI Hub identity") + source_path = repository_root / str(source.get("path")) + current_source_sha256 = _sha256(source_path) if source_path.is_file() else None + source_model_matches = current_source_sha256 == source.get("sha256") + if hardware.get("backend") != "HTP" or not hardware.get("qairt_version"): + raise ValueError("Qualcomm report must identify the HTP backend and QAIRT version") + if "SUCCESS" not in str(linked.get("link_status")): + raise ValueError("linked QNN context job did not succeed") + validation = linked.get("validation") + if not isinstance(validation, dict) or validation.get("device") != hardware.get("device"): + raise ValueError("linked QNN validation device does not match the report") + graphs = validation.get("graphs") + if not isinstance(graphs, dict) or set(graphs) != { + "edgegenbench_batch1", + "edgegenbench_batch32", + "edgegenbench_batch256", + }: + raise ValueError("linked QNN evidence must contain batch 1, 32, and 256 graphs") + + graph_summaries = [] + for name, graph in graphs.items(): + if not isinstance(graph, dict): + raise ValueError(f"invalid QNN graph evidence: {name}") + profile = graph.get("profile") + parity = graph.get("parity") + if not isinstance(profile, dict) or profile.get("compute_units") != {"NPU": 9}: + raise ValueError(f"{name} does not prove exclusive NPU compute-unit mapping") + if not isinstance(parity, dict) or float(parity.get("max_normalized_drift", 1.0)) > 0.012: + raise ValueError(f"{name} exceeds the predeclared normalized drift limit") + if not graph.get("profile_job_id") or not graph.get("inference_job_id"): + raise ValueError(f"{name} requires profile and inference job IDs") + graph_summaries.append( + { + "graph": name, + "batch_size": graph["batch_size"], + "latency_ms": profile["estimated_inference_latency_ms"], + "throughput_samples_per_second": profile["estimated_throughput_samples_per_second"], + "peak_memory_bytes": profile["estimated_inference_peak_memory_bytes"], + "compute_units": profile["compute_units"], + "max_normalized_drift": parity["max_normalized_drift"], + "profile_job_id": graph["profile_job_id"], + "inference_job_id": graph["inference_job_id"], + } + ) + return { + "status": ( + "validated_ai_hub_physical_qnn" + if source_model_matches + else "tracked_ai_hub_report_model_provenance_mismatch" + ), + "device": hardware["device"], + "backend": "QNN HTP", + "qairt_version": hardware["qairt_version"], + "context_model_id": linked["target_model_id"], + "link_job_id": linked["link_job_id"], + "graphs": sorted(graph_summaries, key=lambda value: value["batch_size"]), + "reported_source_model_sha256": source.get("sha256"), + "current_source_model_sha256": current_source_sha256, + "source_model_matches_repository": source_model_matches, + "claim_boundary": "Physical AI Hub model profiling; not Android APK end-to-end latency.", + } + + +def build_portfolio_acceptance( + *, repository_root: Path, qnn_report: Path, output_json: Path, output_markdown: Path +) -> dict[str, Any]: + qnn = validate_ai_hub_qnn(qnn_report, repository_root) + android_report = repository_root / "reports/android_sm_a356e_reference_10_run_v0_1_4.md" + if not android_report.is_file(): + raise ValueError("tracked Samsung reference report is missing") + matrix = { + "schema_version": 1, + "project": "EdgeGenBench", + "lanes": { + "native_cpp": { + "status": "validated_in_ci", + "claim": "C++17 reference runtime, tests, CLI, and fused preprocessing acceptance.", + }, + "android_reference": { + "status": "validated_physical_device", + "device": "Samsung SM-A356E", + "report": android_report.relative_to(repository_root).as_posix(), + "claim": "Reference JNI/application measurements; not QNN.", + }, + "qualcomm_ai_hub_qnn": qnn, + "android_qnn_apk": { + "status": "implementation_complete_evidence_pending", + "claim": "Build/JNI/capture paths exist; requires a supported Snapdragon APK run.", + }, + "android_16kb_runtime": { + "status": "packaging_validated_runtime_pending", + "claim": "ELF/APK alignment passes; runtime PAGE_SIZE=16384 evidence is pending.", + }, + "power": { + "status": "not_measured", + "claim": "No power-savings claim is made without a named calibrated tool.", + }, + }, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(matrix, indent=2) + "\n", encoding="utf-8") + lines = [ + "# EdgeGenBench portfolio acceptance", + "", + "| Evidence lane | Status | Claim boundary |", + "|---|---|---|", + ] + for name, lane in matrix["lanes"].items(): + claim = lane.get("claim", lane.get("claim_boundary")) + lines.append(f"| `{name}` | `{lane['status']}` | {claim} |") + lines.extend( + [ + "", + "## Validated Qualcomm QNN results", + "", + f"Device: **{qnn['device']}**; backend: **{qnn['backend']}**; " + f"QAIRT: `{qnn['qairt_version']}`.", + f"Source-model provenance match: **{qnn['source_model_matches_repository']}**.", + "", + "| Batch | AI Hub latency (ms) | Throughput (samples/s) | " + "Peak memory (bytes) | Placement | Max normalized drift |", + "|---:|---:|---:|---:|---|---:|", + ] + ) + for graph in qnn["graphs"]: + lines.append( + f"| {graph['batch_size']} | {graph['latency_ms']:.6f} | " + f"{graph['throughput_samples_per_second']:.3f} | {graph['peak_memory_bytes']} | " + f"NPU × 9 | {graph['max_normalized_drift']:.9f} |" + ) + lines.extend( + [ + "", + "AI Hub measurements are physical-device model profiles, not Android " + "application end-to-end timings. They cannot be promoted as current-model " + "acceptance until the reported model artifact is restored or the current model " + "is re-profiled.", + "Power remains unmeasured. The two pending proof items are a supported-device " + "QNN APK run and a runtime page size of 16384 bytes.", + ] + ) + output_markdown.write_text("\n".join(lines) + "\n", encoding="utf-8") + return matrix + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository-root", type=Path, default=Path.cwd()) + parser.add_argument("--qnn-report", type=Path, default=Path("reports/qualcomm_qnn_v0_1.json")) + parser.add_argument( + "--output-json", type=Path, default=Path("reports/portfolio_acceptance.json") + ) + parser.add_argument( + "--output-markdown", type=Path, default=Path("reports/portfolio_acceptance.md") + ) + args = parser.parse_args() + build_portfolio_acceptance( + repository_root=args.repository_root.resolve(), + qnn_report=args.qnn_report, + output_json=args.output_json, + output_markdown=args.output_markdown, + ) + print(f"Portfolio acceptance written to {args.output_markdown}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_portfolio_acceptance.py b/tests/test_portfolio_acceptance.py new file mode 100644 index 0000000..3f826f3 --- /dev/null +++ b/tests/test_portfolio_acceptance.py @@ -0,0 +1,33 @@ +import json +import runpy +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import pytest + +SCRIPT = Path(__file__).parents[1] / "scripts/build_portfolio_acceptance.py" +FUNCTIONS = runpy.run_path(SCRIPT) +ValidateQnn = Callable[[Path, Path], dict[str, object]] +validate_ai_hub_qnn = cast(ValidateQnn, FUNCTIONS["validate_ai_hub_qnn"]) + + +def test_validates_tracked_ai_hub_qnn_evidence() -> None: + root = Path(__file__).parents[1] + result = validate_ai_hub_qnn(root / "reports/qualcomm_qnn_v0_1.json", root) + assert result["status"] == "tracked_ai_hub_report_model_provenance_mismatch" + assert result["backend"] == "QNN HTP" + assert result["source_model_matches_repository"] is False + assert len(cast(list[object], result["graphs"])) == 3 + + +def test_rejects_cpu_compute_unit_in_qnn_report(tmp_path: Path) -> None: + root = Path(__file__).parents[1] + report = json.loads((root / "reports/qualcomm_qnn_v0_1.json").read_text()) + report["linked_multigraph"]["validation"]["graphs"]["edgegenbench_batch1"]["profile"][ + "compute_units" + ] = {"CPU": 1, "NPU": 8} + path = tmp_path / "qnn.json" + path.write_text(json.dumps(report)) + with pytest.raises(ValueError, match="exclusive NPU"): + validate_ai_hub_qnn(path, root) From 00a208a599a57505bf9cf13765b5b8c708b1fd3a Mon Sep 17 00:00:00 2001 From: triasha72 Date: Wed, 26 Aug 2026 13:14:20 -0400 Subject: [PATCH 2/4] Revalidate current model on Qualcomm QNN --- README.md | 22 +- docs/qualcomm_ai_hub_qnn.md | 20 +- reports/portfolio_acceptance.json | 42 +-- reports/portfolio_acceptance.md | 12 +- reports/qualcomm_qnn_v0_1.json | 226 ++++--------- scripts/build_portfolio_acceptance.py | 5 +- scripts/rerun_qualcomm_qnn_current_model.py | 352 ++++++++++++++++++++ tests/test_portfolio_acceptance.py | 4 +- 8 files changed, 468 insertions(+), 215 deletions(-) create mode 100644 scripts/rerun_qualcomm_qnn_current_model.py diff --git a/README.md b/README.md index 26f7e57..76f3d1c 100644 --- a/README.md +++ b/README.md @@ -167,9 +167,8 @@ Cross-provider timings should not be interpreted as a direct hardware ranking. The consolidated [portfolio acceptance matrix](reports/portfolio_acceptance.md) separates validated CI/device lanes from evidence still pending. It also checks -whether the model hash in the historical Qualcomm report matches the current -ONNX artifact; the current mismatch is intentionally visible and must be -resolved by restoring the reported model or re-running AI Hub. +that the model hash in the Qualcomm report matches the current ONNX artifact. +The current-model AI Hub rerun restores that provenance chain. EdgeGenBench now includes a validated Qualcomm QNN deployment path for the compact neural surrogate. @@ -183,13 +182,13 @@ profiled layers were placed on the NPU for every validated graph. | Batch | AI Hub profile latency | Peak memory | Compute units | |---:|---:|---:|---| -| 1 | 38 us | 122,937,344 B | NPU: 9 | -| 32 | 34 us | 122,888,192 B | NPU: 9 | -| 256 | 57 us | 123,211,776 B | NPU: 9 | +| 1 | 38 us | 122,855,424 B | NPU: 9 | +| 32 | 40 us | 122,880,000 B | NPU: 9 | +| 256 | 47 us | 122,896,384 B | NPU: 9 | On all 900 held-out rows, the linked batch-1 QNN deployment retained mean R2 -of 0.996953 versus 0.996955 for the local FP32 ONNX reference. Maximum -normalized deployment drift was 0.003636. +of 0.996954 versus 0.996956 for the local FP32 ONNX reference. Maximum +normalized deployment drift was 0.003234. These values are device-specific AI Hub model-profile measurements. They are not presented as end-to-end Android application latency or as a same-hardware @@ -199,6 +198,13 @@ Complete provenance and linked-graph validation are stored in: `reports/qualcomm_qnn_v0_1.json` +The authenticated current-model run is reproducible with: + +```bash +PYTHONPATH=src python scripts/rerun_qualcomm_qnn_current_model.py +python scripts/build_portfolio_acceptance.py +``` + ### Qualcomm-native INT8 decision The follow-up INT8/QDQ study was executed on the same Snapdragon 8 Elite diff --git a/docs/qualcomm_ai_hub_qnn.md b/docs/qualcomm_ai_hub_qnn.md index f242323..890a7f3 100644 --- a/docs/qualcomm_ai_hub_qnn.md +++ b/docs/qualcomm_ai_hub_qnn.md @@ -90,11 +90,11 @@ containing three statically specialized graph variants: Linked target model: -`mnl7771jm` +`mnjjex79n` Link job: -`jp16x3285` +`jgk4d8lvp` The linked artifact retains: @@ -107,9 +107,9 @@ The linked artifact retains: | Graph | Batch | AI Hub profile latency | Derived model throughput | Peak memory | Compute units | |---|---:|---:|---:|---:|---| -| `edgegenbench_batch1` | 1 | 38 us | 26,315.8 samples/s | 122,937,344 B | NPU: 9 | -| `edgegenbench_batch32` | 32 | 34 us | 941,176.5 samples/s | 122,888,192 B | NPU: 9 | -| `edgegenbench_batch256` | 256 | 57 us | 4,491,228.1 samples/s | 123,211,776 B | NPU: 9 | +| `edgegenbench_batch1` | 1 | 38 us | 26,315.8 samples/s | 122,855,424 B | NPU: 9 | +| `edgegenbench_batch32` | 32 | 40 us | 800,000.0 samples/s | 122,880,000 B | NPU: 9 | +| `edgegenbench_batch256` | 256 | 47 us | 5,446,808.5 samples/s | 122,896,384 B | NPU: 9 | The throughput values are derived from the configured batch size divided by AI Hub's estimated model-inference time. They are not end-to-end Android @@ -125,8 +125,8 @@ All 900 held-out test rows were evaluated through the linked batch-1 graph. | Metric | Local FP32 ONNX | Linked Snapdragon QNN | |---|---:|---:| -| Mean R2 | 0.996955004 | 0.996953249 | -| Mean NRMSE | 0.050432628 | 0.050444571 | +| Mean R2 | 0.996955989 | 0.996954462 | +| Mean NRMSE | 0.050425306 | 0.050449587 | Deployment drift: @@ -193,16 +193,16 @@ Install Qualcomm-specific dependencies with: pip install -e '.[qualcomm]' ``` -Validate the linked multi-graph target: +Compile, link, profile, and validate the current ONNX model: ```bash -python scripts/validate_qualcomm_qnn_multigraph.py +PYTHONPATH=src python scripts/rerun_qualcomm_qnn_current_model.py ``` Build the canonical evidence report: ```bash -python scripts/build_qualcomm_qnn_report.py +python scripts/build_portfolio_acceptance.py ``` The complete machine-readable result is stored in: diff --git a/reports/portfolio_acceptance.json b/reports/portfolio_acceptance.json index 3860fb8..a5f350f 100644 --- a/reports/portfolio_acceptance.json +++ b/reports/portfolio_acceptance.json @@ -13,56 +13,56 @@ "claim": "Reference JNI/application measurements; not QNN." }, "qualcomm_ai_hub_qnn": { - "status": "tracked_ai_hub_report_model_provenance_mismatch", + "status": "validated_ai_hub_physical_qnn", "device": "Snapdragon 8 Elite QRD", "backend": "QNN HTP", "qairt_version": "2.45.0.260326154327", - "context_model_id": "mnl7771jm", - "link_job_id": "jp16x3285", + "context_model_id": "mnjjex79n", + "link_job_id": "jgk4d8lvp", "graphs": [ { "graph": "edgegenbench_batch1", "batch_size": 1, "latency_ms": 0.038, "throughput_samples_per_second": 26315.78947368421, - "peak_memory_bytes": 122822656, + "peak_memory_bytes": 122855424, "compute_units": { "NPU": 9 }, - "max_normalized_drift": 0.0036358769310459917, - "profile_job_id": "jp4308d85", - "inference_job_id": "jpxx2m63p" + "max_normalized_drift": 0.003233889676953719, + "profile_job_id": "j567vw3np", + "inference_job_id": "jg9m8d3q5" }, { "graph": "edgegenbench_batch32", "batch_size": 32, - "latency_ms": 0.034, - "throughput_samples_per_second": 941176.4705882353, - "peak_memory_bytes": 122888192, + "latency_ms": 0.04, + "throughput_samples_per_second": 800000.0, + "peak_memory_bytes": 122880000, "compute_units": { "NPU": 9 }, - "max_normalized_drift": 0.003222448271801646, - "profile_job_id": "jgnk8xmkg", - "inference_job_id": "jprwj9205" + "max_normalized_drift": 0.0028659358491131483, + "profile_job_id": "jpyxnmvr5", + "inference_job_id": "jpez2yl7p" }, { "graph": "edgegenbench_batch256", "batch_size": 256, - "latency_ms": 0.057, - "throughput_samples_per_second": 4491228.070175438, - "peak_memory_bytes": 123211776, + "latency_ms": 0.047, + "throughput_samples_per_second": 5446808.510638298, + "peak_memory_bytes": 122896384, "compute_units": { "NPU": 9 }, - "max_normalized_drift": 0.003222448271801646, - "profile_job_id": "jp2enj9rp", - "inference_job_id": "jpym0nj8p" + "max_normalized_drift": 0.0028659358491131483, + "profile_job_id": "jglx7lllg", + "inference_job_id": "jgk4d82op" } ], - "reported_source_model_sha256": "40f588b329b98fdaa38a7eda202fc89573fce5e91a4193f0a0a93b3142c0382f", + "reported_source_model_sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", "current_source_model_sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", - "source_model_matches_repository": false, + "source_model_matches_repository": true, "claim_boundary": "Physical AI Hub model profiling; not Android APK end-to-end latency." }, "android_qnn_apk": { diff --git a/reports/portfolio_acceptance.md b/reports/portfolio_acceptance.md index 8484913..5ef849c 100644 --- a/reports/portfolio_acceptance.md +++ b/reports/portfolio_acceptance.md @@ -4,7 +4,7 @@ |---|---|---| | `native_cpp` | `validated_in_ci` | C++17 reference runtime, tests, CLI, and fused preprocessing acceptance. | | `android_reference` | `validated_physical_device` | Reference JNI/application measurements; not QNN. | -| `qualcomm_ai_hub_qnn` | `tracked_ai_hub_report_model_provenance_mismatch` | Physical AI Hub model profiling; not Android APK end-to-end latency. | +| `qualcomm_ai_hub_qnn` | `validated_ai_hub_physical_qnn` | Physical AI Hub model profiling; not Android APK end-to-end latency. | | `android_qnn_apk` | `implementation_complete_evidence_pending` | Build/JNI/capture paths exist; requires a supported Snapdragon APK run. | | `android_16kb_runtime` | `packaging_validated_runtime_pending` | ELF/APK alignment passes; runtime PAGE_SIZE=16384 evidence is pending. | | `power` | `not_measured` | No power-savings claim is made without a named calibrated tool. | @@ -12,13 +12,13 @@ ## Validated Qualcomm QNN results Device: **Snapdragon 8 Elite QRD**; backend: **QNN HTP**; QAIRT: `2.45.0.260326154327`. -Source-model provenance match: **False**. +Source-model provenance match: **True**. | Batch | AI Hub latency (ms) | Throughput (samples/s) | Peak memory (bytes) | Placement | Max normalized drift | |---:|---:|---:|---:|---|---:| -| 1 | 0.038000 | 26315.789 | 122822656 | NPU × 9 | 0.003635877 | -| 32 | 0.034000 | 941176.471 | 122888192 | NPU × 9 | 0.003222448 | -| 256 | 0.057000 | 4491228.070 | 123211776 | NPU × 9 | 0.003222448 | +| 1 | 0.038000 | 26315.789 | 122855424 | NPU × 9 | 0.003233890 | +| 32 | 0.040000 | 800000.000 | 122880000 | NPU × 9 | 0.002865936 | +| 256 | 0.047000 | 5446808.511 | 122896384 | NPU × 9 | 0.002865936 | -AI Hub measurements are physical-device model profiles, not Android application end-to-end timings. They cannot be promoted as current-model acceptance until the reported model artifact is restored or the current model is re-profiled. +AI Hub measurements are physical-device model profiles, not Android application end-to-end timings. Current-model acceptance requires source-model provenance to match the repository, as reported above. Power remains unmeasured. The two pending proof items are a supported-device QNN APK run and a runtime page size of 16384 bytes. diff --git a/reports/qualcomm_qnn_v0_1.json b/reports/qualcomm_qnn_v0_1.json index 42e2061..860e1a9 100644 --- a/reports/qualcomm_qnn_v0_1.json +++ b/reports/qualcomm_qnn_v0_1.json @@ -1,11 +1,12 @@ { "schema_version": "0.2", - "experiment": "EdgeGenBench Qualcomm QNN Snapdragon 8 Elite deployment", + "experiment": "EdgeGenBench current-model Qualcomm QNN deployment", "source_model": { "path": "artifacts/neural_onnx/neural_surrogate.onnx", - "sha256": "40f588b329b98fdaa38a7eda202fc89573fce5e91a4193f0a0a93b3142c0382f", + "sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", "input_name": "features", "input_width": 10, + "output_name": "predictions", "output_width": 6, "source_precision": "float32" }, @@ -14,129 +15,37 @@ "device_os": "15", "chipset": "qualcomm-snapdragon-8-elite", "chipset_alias": "sm8750", - "soc_model": "69", "backend": "HTP", "hexagon": "v79", "qairt_version": "2.45.0.260326154327" }, - "batch_specific_baseline": { - "1": { - "batch_size": 1, - "compile_job_id": "j5793jj9g", - "profile_job_id": "jp2en736p", - "target_model_id": "mnz66wwxm", - "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", - "serialized_model_size_bytes": 53248, - "serialized_model_sha256": "fd4dbc74b39607d0126f53f3838737197cc5418c2920786d49e0a01d1d412c00", - "compute_units": { - "NPU": 9 - }, - "estimated_inference_time_us": 32, - "estimated_inference_peak_memory_bytes": 123199488, - "target_metadata": { - "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", - "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", - "ModelMetadataKey.QNN_CONTEXT_BIN_VTCM": "0", - "ModelMetadataKey.QAIRT_SDK_VERSION": "2.45.0.260326154327", - "ModelMetadataKey.QNN_SDK_VARIANT": "default", - "ModelMetadataKey.QNN_CONTEXT_BIN_OPTIMIZATION_LEVEL": "3", - "ModelMetadataKey.QNN_MODEL_GRAPH_NAMES": "graph_x4h9d3zo", - "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" - } - }, - "32": { - "batch_size": 32, - "compile_job_id": "j5m8y4xwp", - "profile_job_id": "jp36280lp", - "target_model_id": "mqvkkk3jn", - "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", - "serialized_model_size_bytes": 53248, - "serialized_model_sha256": "36de3516ebae290dfc445d20d3db435f48aa2b6c531e052d2671d954319f5b99", - "compute_units": { - "NPU": 9 - }, - "estimated_inference_time_us": 35, - "estimated_inference_peak_memory_bytes": 122978304, - "target_metadata": { - "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", - "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", - "ModelMetadataKey.QNN_CONTEXT_BIN_VTCM": "0", - "ModelMetadataKey.QAIRT_SDK_VERSION": "2.45.0.260326154327", - "ModelMetadataKey.QNN_SDK_VARIANT": "default", - "ModelMetadataKey.QNN_CONTEXT_BIN_OPTIMIZATION_LEVEL": "3", - "ModelMetadataKey.QNN_MODEL_GRAPH_NAMES": "graph_f0fyaikk", - "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" - } - }, - "256": { - "batch_size": 256, - "compile_job_id": "jpv7x4kjp", - "profile_job_id": "jpey32m15", - "target_model_id": "mmxgggjrq", - "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", - "serialized_model_size_bytes": 57344, - "serialized_model_sha256": "b1e55d5220240f9d7119fdb72fa12c6aeaf079f260f9e452c4fd01d47f284c18", - "compute_units": { - "NPU": 9 - }, - "estimated_inference_time_us": 54, - "estimated_inference_peak_memory_bytes": 122892288, - "target_metadata": { - "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", - "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", - "ModelMetadataKey.QNN_CONTEXT_BIN_VTCM": "0", - "ModelMetadataKey.QAIRT_SDK_VERSION": "2.45.0.260326154327", - "ModelMetadataKey.QNN_SDK_VARIANT": "default", - "ModelMetadataKey.QNN_CONTEXT_BIN_OPTIMIZATION_LEVEL": "3", - "ModelMetadataKey.QNN_MODEL_GRAPH_NAMES": "graph_bqjx81_9", - "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" - } - } - }, - "heldout_batch1_baseline": { - "sample_count": 900, - "target_model_id": "mnz66wwxm", - "inference_job_id": "jg9de8ll5", - "device": "Snapdragon 8 Elite QRD", - "device_os": "15", - "normalized_mae": 0.0004096813499927521, - "normalized_rmse": 0.000530485762283206, - "normalized_max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.0004119437653571367, - "max_normalized_drift": 0.0036358777433633804, - "allclose_1e3": false, - "local_mean_r2": 0.9969550037922089, - "remote_mean_r2": 0.9969532494078602, - "local_mean_nrmse_std": 0.05043262755109571, - "remote_mean_nrmse_std": 0.05044457139336909, - "r2_delta": -1.754384348684468e-06, - "nrmse_delta": 1.1943842273376415e-05 - }, "linked_multigraph": { "compile_jobs": [ { "batch": 1, "graph_name": "edgegenbench_batch1", - "compile_job_id": "jgd2l0ne5", + "compile_job_id": "jpyxnmd05", "compile_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n" }, { "batch": 32, "graph_name": "edgegenbench_batch32", - "compile_job_id": "j5w4nxw3g", + "compile_job_id": "jp0jk610g", "compile_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n" }, { "batch": 256, "graph_name": "edgegenbench_batch256", - "compile_job_id": "jg9de80w5", + "compile_job_id": "jp8x813qg", "compile_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n" } ], - "link_job_id": "jp16x3285", + "link_job_id": "jgk4d8lvp", "link_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n", - "target_model_id": "mnl7771jm", + "target_model_id": "mnjjex79n", "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", + "serialized_model_size_bytes": 106496, + "serialized_model_sha256": "43d7cb889b0dd97d8de3a48557fdc7dceb322e6c7b72fdb91b19e5473f84b0df", "target_metadata": { "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", @@ -148,8 +57,8 @@ "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" }, "validation": { - "link_job_id": "jp16x3285", - "target_model_id": "mnl7771jm", + "link_job_id": "jgk4d8lvp", + "target_model_id": "mnjjex79n", "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", "target_metadata": { "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", @@ -175,14 +84,14 @@ "graph_name": "edgegenbench_batch1", "batch_size": 1, "sample_count": 900, - "profile_job_id": "jp4308d85", - "inference_job_id": "jpxx2m63p", + "profile_job_id": "j567vw3np", + "inference_job_id": "jg9m8d3q5", "profile": { "batch_size": 1, "estimated_inference_time_us": 38.0, "estimated_inference_latency_ms": 0.038, "estimated_throughput_samples_per_second": 26315.78947368421, - "estimated_inference_peak_memory_bytes": 122822656, + "estimated_inference_peak_memory_bytes": 122855424, "compute_units": { "NPU": 9 } @@ -191,34 +100,34 @@ "parity": { "sample_count": 900, "output_width": 6, - "mae": 0.00040968134033459206, - "rmse": 0.0005304857849774182, - "max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.00041194377879442924, - "max_normalized_drift": 0.0036358769310459917, + "mae": 0.00043518983793479425, + "rmse": 0.0005554710568707642, + "max_abs_error": 0.0033278465270996094, + "mean_normalized_drift": 0.0004384562793800826, + "max_normalized_drift": 0.003233889676953719, "allclose_rtol_1e3_atol_1e3": false }, "predictive_quality": { - "local_mean_r2": 0.9969550037922089, - "remote_mean_r2": 0.9969532494078602, - "r2_delta": -1.754384348684468e-06, - "local_mean_nrmse_std": 0.05043262755109571, - "remote_mean_nrmse_std": 0.05044457139336909, - "nrmse_delta": 1.1943842273376415e-05 + "local_mean_r2": 0.9969559893499255, + "remote_mean_r2": 0.9969544620647843, + "r2_delta": -1.5272851411340937e-06, + "local_mean_nrmse_std": 0.05042530552302998, + "remote_mean_nrmse_std": 0.050449586683723985, + "nrmse_delta": 2.428116069400771e-05 } }, "edgegenbench_batch32": { "graph_name": "edgegenbench_batch32", "batch_size": 32, "sample_count": 256, - "profile_job_id": "jgnk8xmkg", - "inference_job_id": "jprwj9205", + "profile_job_id": "jpyxnmvr5", + "inference_job_id": "jpez2yl7p", "profile": { "batch_size": 32, - "estimated_inference_time_us": 34.0, - "estimated_inference_latency_ms": 0.034, - "estimated_throughput_samples_per_second": 941176.4705882353, - "estimated_inference_peak_memory_bytes": 122888192, + "estimated_inference_time_us": 40.0, + "estimated_inference_latency_ms": 0.04, + "estimated_throughput_samples_per_second": 800000.0, + "estimated_inference_peak_memory_bytes": 122880000, "compute_units": { "NPU": 9 } @@ -227,34 +136,34 @@ "parity": { "sample_count": 256, "output_width": 6, - "mae": 0.00042206487463166315, - "rmse": 0.00055137047774592, - "max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.0004078000437142555, - "max_normalized_drift": 0.003222448271801646, + "mae": 0.0004328816042592128, + "rmse": 0.0005609981854555366, + "max_abs_error": 0.0033278465270996094, + "mean_normalized_drift": 0.00042132448724577204, + "max_normalized_drift": 0.0028659358491131483, "allclose_rtol_1e3_atol_1e3": false }, "predictive_quality": { - "local_mean_r2": 0.9971241567768717, - "remote_mean_r2": 0.9971200615281508, - "r2_delta": -4.0952487209144905e-06, - "local_mean_nrmse_std": 0.04895350495151681, - "remote_mean_nrmse_std": 0.048983762073770726, - "nrmse_delta": 3.0257122253914137e-05 + "local_mean_r2": 0.9971158433783592, + "remote_mean_r2": 0.9971139833803742, + "r2_delta": -1.8599979850764115e-06, + "local_mean_nrmse_std": 0.04900140374890117, + "remote_mean_nrmse_std": 0.04902444807450163, + "nrmse_delta": 2.30443256004631e-05 } }, "edgegenbench_batch256": { "graph_name": "edgegenbench_batch256", "batch_size": 256, "sample_count": 256, - "profile_job_id": "jp2enj9rp", - "inference_job_id": "jpym0nj8p", + "profile_job_id": "jglx7lllg", + "inference_job_id": "jgk4d82op", "profile": { "batch_size": 256, - "estimated_inference_time_us": 57.0, - "estimated_inference_latency_ms": 0.057, - "estimated_throughput_samples_per_second": 4491228.070175438, - "estimated_inference_peak_memory_bytes": 123211776, + "estimated_inference_time_us": 47.0, + "estimated_inference_latency_ms": 0.047, + "estimated_throughput_samples_per_second": 5446808.510638298, + "estimated_inference_peak_memory_bytes": 122896384, "compute_units": { "NPU": 9 } @@ -263,37 +172,24 @@ "parity": { "sample_count": 256, "output_width": 6, - "mae": 0.00042206487463166315, - "rmse": 0.00055137047774592, - "max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.0004078000437142555, - "max_normalized_drift": 0.003222448271801646, + "mae": 0.0004328816042592128, + "rmse": 0.0005609981854555366, + "max_abs_error": 0.0033278465270996094, + "mean_normalized_drift": 0.00042132448724577204, + "max_normalized_drift": 0.0028659358491131483, "allclose_rtol_1e3_atol_1e3": false }, "predictive_quality": { - "local_mean_r2": 0.9971241567768717, - "remote_mean_r2": 0.9971200615281508, - "r2_delta": -4.0952487209144905e-06, - "local_mean_nrmse_std": 0.04895350495151681, - "remote_mean_nrmse_std": 0.048983762073770726, - "nrmse_delta": 3.0257122253914137e-05 + "local_mean_r2": 0.9971158433783592, + "remote_mean_r2": 0.9971139833803742, + "r2_delta": -1.8599979850764115e-06, + "local_mean_nrmse_std": 0.04900140374890117, + "remote_mean_nrmse_std": 0.04902444807450163, + "nrmse_delta": 2.30443256004631e-05 } } } } }, - "measurement_semantics": { - "latency": "Qualcomm AI Hub profile estimated model inference time", - "throughput": "Derived from configured batch size and AI Hub estimated inference time", - "runtime_memory": "Qualcomm AI Hub estimated inference peak memory", - "serialized_model_size": "Downloaded QNN Context Binary byte size", - "allclose": "Diagnostic only; predictive quality and normalized drift are reported separately" - }, - "claim_boundaries": [ - "Results apply to the named Snapdragon 8 Elite QRD configuration.", - "AI Hub profile latency is not end-to-end Android application latency.", - "Derived throughput is based on model-profile latency and is not an end-to-end system throughput measurement.", - "CPU, CoreML, and Snapdragon timings are not treated as same-hardware comparisons.", - "The QNN baseline uses float32 external I/O with HTP FP16-relaxed execution." - ] + "claim_boundary": "Physical Qualcomm AI Hub QNN HTP model measurements; not Android APK end-to-end latency and not a calibrated power measurement." } diff --git a/scripts/build_portfolio_acceptance.py b/scripts/build_portfolio_acceptance.py index c4b39a4..1947c51 100644 --- a/scripts/build_portfolio_acceptance.py +++ b/scripts/build_portfolio_acceptance.py @@ -171,9 +171,8 @@ def build_portfolio_acceptance( [ "", "AI Hub measurements are physical-device model profiles, not Android " - "application end-to-end timings. They cannot be promoted as current-model " - "acceptance until the reported model artifact is restored or the current model " - "is re-profiled.", + "application end-to-end timings. Current-model acceptance requires source-model " + "provenance to match the repository, as reported above.", "Power remains unmeasured. The two pending proof items are a supported-device " "QNN APK run and a runtime page size of 16384 bytes.", ] diff --git a/scripts/rerun_qualcomm_qnn_current_model.py b/scripts/rerun_qualcomm_qnn_current_model.py new file mode 100644 index 0000000..72947d0 --- /dev/null +++ b/scripts/rerun_qualcomm_qnn_current_model.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Compile, link, profile, and validate the current ONNX model on Qualcomm AI Hub.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +import numpy as np +import onnxruntime as ort +import pandas as pd +import qai_hub as hub + +from edgegenbench.deployment.qualcomm_ai_hub import ( + calculate_runtime_parity, + qnn_graph_option, + stringify_metadata, + summarize_profile, +) + +NUMERIC_FEATURES = [ + "passenger_capacity", + "design_range_km", + "cruise_speed_kmh", + "battery_specific_energy_wh_per_kg", + "hydrogen_storage_efficiency", + "hybridization_ratio", +] +CATEGORICAL_FEATURE = "propulsion_architecture" + +MODEL_PATH = Path("artifacts/neural_onnx/neural_surrogate.onnx") +PREPROCESSOR_PATH = Path("artifacts/neural_surrogate/preprocessing.npz") +HELDOUT_PATH = Path("artifacts/neural_surrogate/test_predictions.csv") +OUTPUT_PATH = Path("reports/qualcomm_qnn_v0_1.json") +CONTEXT_PATH = Path("artifacts/qualcomm_ai_hub/current_model/edgegenbench_multigraph.bin") +CHECKPOINT_PATH = Path("artifacts/qualcomm_ai_hub/current_model/rerun_checkpoint.json") + +DEVICE_NAME = "Snapdragon 8 Elite QRD" +DEVICE_OS = "15" +GRAPH_BATCHES = { + "edgegenbench_batch1": 1, + "edgegenbench_batch32": 32, + "edgegenbench_batch256": 256, +} +COMPILE_OPTIONS = "--qnn_options default_graph_htp_precision=FLOAT16" +LINK_OPTIONS = "--qnn_options default_graph_htp_optimizations=O=3" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def _require_success(status: object, label: str) -> None: + if not bool(getattr(status, "success", False)): + raise RuntimeError(f"{label} failed: {status}") + + +def _partition(features: np.ndarray, batch_size: int) -> list[np.ndarray]: + if len(features) % batch_size: + raise ValueError(f"Feature count {len(features)} is not divisible by {batch_size}.") + return [ + features[start : start + batch_size].astype(np.float32, copy=False) + for start in range(0, len(features), batch_size) + ] + + +def _download_output(inference_job: Any) -> tuple[str, np.ndarray]: + output = inference_job.download_output_data() + if not isinstance(output, dict) or len(output) != 1: + raise RuntimeError(f"Expected one AI Hub output tensor, received {type(output)}.") + name = next(iter(output)) + arrays = [] + for value in output[name]: + array = np.asarray(value, dtype=np.float32) + arrays.append(array.reshape(-1, array.shape[-1])) + return name, np.concatenate(arrays, axis=0) + + +def _load_preprocessing() -> dict[str, Any]: + with np.load(PREPROCESSOR_PATH, allow_pickle=False) as data: + return {key: data[key].copy() for key in data.files} + + +def _transform_features(frame: pd.DataFrame, preprocessing: dict[str, Any]) -> np.ndarray: + numeric = frame.loc[:, NUMERIC_FEATURES].to_numpy(dtype=np.float32) + standardized = (numeric - preprocessing["feature_mean"]) / preprocessing["feature_scale"] + categories = [str(value) for value in preprocessing["categories"].tolist()] + category_to_index = {category: index for index, category in enumerate(categories)} + indices = np.asarray( + [category_to_index[str(value)] for value in frame[CATEGORICAL_FEATURE]], dtype=np.int64 + ) + one_hot = np.eye(len(categories), dtype=np.float32)[indices] + return np.concatenate([standardized, one_hot], axis=1).astype(np.float32) + + +def _quality( + actual: np.ndarray, prediction: np.ndarray, preprocessing: dict[str, Any] +) -> dict[str, float]: + physical = (prediction * preprocessing["target_scale"] + preprocessing["target_mean"]).astype( + np.float64 + ) + actual64 = actual.astype(np.float64) + residual = actual64 - physical + squared_error = np.sum(residual * residual, axis=0) + centered = actual64 - np.mean(actual64, axis=0) + total_squared = np.sum(centered * centered, axis=0) + r2 = 1.0 - squared_error / total_squared + rmse = np.sqrt(np.mean(residual * residual, axis=0)) + actual_std = np.std(actual64, axis=0) + nrmse_std = rmse / actual_std + return { + "mean_r2": float(np.nanmean(r2)), + "mean_nrmse_std": float(np.nanmean(nrmse_std)), + } + + +def _checkpoint(client: hub.Client, model_sha256: str) -> dict[str, Any]: + if not CHECKPOINT_PATH.is_file(): + return {"source_model_sha256": model_sha256, "graphs": {}} + value = json.loads(CHECKPOINT_PATH.read_text(encoding="utf-8")) + if value.get("source_model_sha256") != model_sha256: + raise RuntimeError("Checkpoint belongs to a different source model; remove it to rerun.") + # Validate that the saved linked model remains accessible before resuming. + if value.get("target_model_id"): + client.get_model(str(value["target_model_id"])) + return value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--resume", action="store_true", help="Reuse successful saved job IDs.") + args = parser.parse_args() + + for path in (MODEL_PATH, PREPROCESSOR_PATH, HELDOUT_PATH): + if not path.is_file(): + raise FileNotFoundError(f"Required artifact is missing: {path}") + + model_sha256 = _sha256(MODEL_PATH) + client = hub.Client() + device = hub.Device(DEVICE_NAME, DEVICE_OS) + checkpoint = ( + _checkpoint(client, model_sha256) + if args.resume + else { + "source_model_sha256": model_sha256, + "graphs": {}, + } + ) + + if checkpoint.get("target_model_id"): + target_model = client.get_model(str(checkpoint["target_model_id"])) + compile_jobs = [client.get_job(job_id) for job_id in checkpoint["compile_job_ids"]] + link_job = client.get_job(str(checkpoint["link_job_id"])) + print(f"Resuming linked model {target_model.model_id}", flush=True) + else: + graph_names = list(GRAPH_BATCHES) + print("Submitting three QNN DLC compiles and the dependent link job...", flush=True) + compile_jobs, link_job = client.submit_compile_and_link_jobs( + models=[MODEL_PATH] * len(graph_names), + device=device, + name="EdgeGenBench current-model QNN multigraph", + input_specs=[ + {"features": ((batch_size, 10), "float32")} for batch_size in GRAPH_BATCHES.values() + ], + graph_names=graph_names, + compile_options=COMPILE_OPTIONS, + link_options=LINK_OPTIONS, + ) + if link_job is None: + raise RuntimeError("AI Hub did not create the QNN link job.") + checkpoint.update( + compile_job_ids=[job.job_id for job in compile_jobs], + link_job_id=link_job.job_id, + ) + _write_json(CHECKPOINT_PATH, checkpoint) + print(f"Compile jobs: {checkpoint['compile_job_ids']}", flush=True) + print(f"Link job: {link_job.job_id}", flush=True) + status = link_job.wait() + _require_success(status, "QNN context link") + target_model = link_job.get_target_model() + if target_model is None: + raise RuntimeError("Successful link job did not return a target model.") + checkpoint["target_model_id"] = target_model.model_id + checkpoint["link_status"] = str(status) + _write_json(CHECKPOINT_PATH, checkpoint) + + CONTEXT_PATH.parent.mkdir(parents=True, exist_ok=True) + target_model.download(str(CONTEXT_PATH)) + + frame = pd.read_csv(HELDOUT_PATH) + if len(frame) != 900: + raise RuntimeError(f"Expected 900 held-out rows, received {len(frame)}.") + preprocessing = _load_preprocessing() + targets = [str(value) for value in preprocessing["targets"].tolist()] + features = _transform_features(frame, preprocessing) + session = ort.InferenceSession(str(MODEL_PATH), providers=["CPUExecutionProvider"]) + input_name = session.get_inputs()[0].name + output_name = session.get_outputs()[0].name + local = np.asarray(session.run([output_name], {input_name: features})[0], dtype=np.float32) + + graph_records: dict[str, Any] = {} + for graph_name, batch_size in GRAPH_BATCHES.items(): + saved = checkpoint["graphs"].get(graph_name, {}) if args.resume else {} + if saved.get("profile_job_id"): + profile_job = client.get_job(str(saved["profile_job_id"])) + else: + print(f"Submitting profile for {graph_name}...", flush=True) + profile_job = client.submit_profile_job( + model=target_model, + device=device, + name=f"EdgeGenBench current-model {graph_name} profile", + options=qnn_graph_option(graph_name), + ) + if isinstance(profile_job, list): + raise RuntimeError("Expected one profile job.") + saved["profile_job_id"] = profile_job.job_id + checkpoint["graphs"][graph_name] = saved + _write_json(CHECKPOINT_PATH, checkpoint) + profile_status = profile_job.wait() + _require_success(profile_status, f"Profile {graph_name}") + profile = summarize_profile(profile_job.download_profile(), batch_size=batch_size) + + sample_count = 900 if batch_size == 1 else 256 + if saved.get("inference_job_id"): + inference_job = client.get_job(str(saved["inference_job_id"])) + else: + print(f"Submitting inference for {graph_name}...", flush=True) + inference_job = client.submit_inference_job( + model=target_model, + device=device, + inputs={input_name: _partition(features[:sample_count], batch_size)}, + name=f"EdgeGenBench current-model {graph_name} inference", + options=qnn_graph_option(graph_name), + ) + if isinstance(inference_job, list): + raise RuntimeError("Expected one inference job.") + saved["inference_job_id"] = inference_job.job_id + _write_json(CHECKPOINT_PATH, checkpoint) + inference_status = inference_job.wait() + _require_success(inference_status, f"Inference {graph_name}") + remote_output_name, remote = _download_output(inference_job) + reference = local[:sample_count] + parity = calculate_runtime_parity(reference, remote) + actual = frame.loc[: sample_count - 1, targets].to_numpy(dtype=np.float32) + local_quality = _quality(actual, reference, preprocessing) + remote_quality = _quality(actual, remote, preprocessing) + graph_records[graph_name] = { + "graph_name": graph_name, + "batch_size": batch_size, + "sample_count": sample_count, + "profile_job_id": profile_job.job_id, + "inference_job_id": inference_job.job_id, + "profile": profile.to_dict(), + "remote_output_name": remote_output_name, + "parity": parity.to_dict(), + "predictive_quality": { + "local_mean_r2": local_quality["mean_r2"], + "remote_mean_r2": remote_quality["mean_r2"], + "r2_delta": remote_quality["mean_r2"] - local_quality["mean_r2"], + "local_mean_nrmse_std": local_quality["mean_nrmse_std"], + "remote_mean_nrmse_std": remote_quality["mean_nrmse_std"], + "nrmse_delta": (remote_quality["mean_nrmse_std"] - local_quality["mean_nrmse_std"]), + }, + } + print(f"Completed {graph_name}: {profile.to_dict()}", flush=True) + + metadata = stringify_metadata(target_model.metadata) + qairt_version = next( + (value for key, value in metadata.items() if "QAIRT_SDK_VERSION" in key), "unknown" + ) + backend = next((value for key, value in metadata.items() if key.endswith("BACKEND")), "HTP") + hexagon = next( + (value for key, value in metadata.items() if "HEXAGON_VERSION" in key), "unknown" + ) + report = { + "schema_version": "0.2", + "experiment": "EdgeGenBench current-model Qualcomm QNN deployment", + "source_model": { + "path": str(MODEL_PATH), + "sha256": model_sha256, + "input_name": input_name, + "input_width": 10, + "output_name": output_name, + "output_width": 6, + "source_precision": "float32", + }, + "hardware": { + "device": DEVICE_NAME, + "device_os": DEVICE_OS, + "chipset": "qualcomm-snapdragon-8-elite", + "chipset_alias": "sm8750", + "backend": backend, + "hexagon": hexagon, + "qairt_version": qairt_version, + }, + "linked_multigraph": { + "compile_jobs": [ + { + "batch": batch, + "graph_name": graph, + "compile_job_id": job.job_id, + "compile_status": str(job.get_status()), + } + for (graph, batch), job in zip(GRAPH_BATCHES.items(), compile_jobs, strict=True) + ], + "link_job_id": link_job.job_id, + "link_status": str(link_job.get_status()), + "target_model_id": target_model.model_id, + "target_model_type": str(target_model.model_type), + "serialized_model_size_bytes": CONTEXT_PATH.stat().st_size, + "serialized_model_sha256": _sha256(CONTEXT_PATH), + "target_metadata": metadata, + "validation": { + "link_job_id": link_job.job_id, + "target_model_id": target_model.model_id, + "target_model_type": str(target_model.model_type), + "target_metadata": metadata, + "target_input_spec": str(target_model.input_spec), + "device": DEVICE_NAME, + "device_os": DEVICE_OS, + "local_reference": { + "model_path": str(MODEL_PATH), + "input_name": input_name, + "output_name": output_name, + "heldout_rows": len(frame), + }, + "graphs": graph_records, + }, + }, + "claim_boundary": ( + "Physical Qualcomm AI Hub QNN HTP model measurements; not Android APK " + "end-to-end latency and not a calibrated power measurement." + ), + } + _write_json(OUTPUT_PATH, report) + print(f"Wrote {OUTPUT_PATH}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/test_portfolio_acceptance.py b/tests/test_portfolio_acceptance.py index 3f826f3..83613d3 100644 --- a/tests/test_portfolio_acceptance.py +++ b/tests/test_portfolio_acceptance.py @@ -15,9 +15,9 @@ def test_validates_tracked_ai_hub_qnn_evidence() -> None: root = Path(__file__).parents[1] result = validate_ai_hub_qnn(root / "reports/qualcomm_qnn_v0_1.json", root) - assert result["status"] == "tracked_ai_hub_report_model_provenance_mismatch" + assert result["status"] == "validated_ai_hub_physical_qnn" assert result["backend"] == "QNN HTP" - assert result["source_model_matches_repository"] is False + assert result["source_model_matches_repository"] is True assert len(cast(list[object], result["graphs"])) == 3 From 10361fbf3bd29c873f83d26469451edafe38f2cc Mon Sep 17 00:00:00 2001 From: triasha72 Date: Wed, 26 Aug 2026 13:29:36 -0400 Subject: [PATCH 3/4] Bind QNN evidence to committed ONNX model --- artifacts/neural_onnx/neural_surrogate.onnx | Bin 0 -> 26365 bytes reports/portfolio_acceptance.json | 1 + scripts/build_portfolio_acceptance.py | 23 +++++++++++++++++++- tests/test_portfolio_acceptance.py | 1 + 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 artifacts/neural_onnx/neural_surrogate.onnx diff --git a/artifacts/neural_onnx/neural_surrogate.onnx b/artifacts/neural_onnx/neural_surrogate.onnx new file mode 100644 index 0000000000000000000000000000000000000000..3fff1cb72e06bce856da4db4299f23e9b3a58ba7 GIT binary patch literal 26365 zcmeIbc|2C%yZ>*FTV~0WDN`~Jk$YbkNs=;_xk4g@WJ;72g(MjwDVj73rHQ?k=2DvH zStHV{IpudZz2Ch*?>=Yve9rlP&N;ugKRoVz-+S$~u63>TTF<@K+SeB4$o7ewnK&&z zG(uU#+}O;**wnV|BtIu#oFf_@7Lu40AC|z8jtxtkF)eURI^Qm$VXF}sn-mk2ln@r50A>?3KqE9HIyxvkDK<1Q za$0O)Vo3ayutfM?2XP6;zsPHXKJTOiUQ&IXAC1mXLnkiiyVrtDgTA%9zxnb@(;H-{ zF?GiG)M>K^xj1r^|E9=aT2D|y=(M=71XTjZMECd^3q>`C{QeT_zzh zBy?&Juj%rd;WHtp$mlQ)EfeoAR+}Wo^9m9Xk`NL9g-(l!iw^rzUnU9hp(a1K2$OG3 zoYy4(risVR)H2lgQeq8j8$*rASdH*$@w}Rt%+cWe#fR~_ZmdQ^SadkAbA0W+`ai06 z5buotzVaIVq?FU_N5P+ z{N4wDH_F_~;!jpZ#>OQj@|wxFboyTp|L0lry?6a?ikXGwpG^7gAgAw|@y!;)Zw~v> z*uQ%3Ya{2p661W=SXrU?u;?UZv9C(|<)fC67w^Bz?mr0SrvJx6Y4!`D4B-hta>$oP zZR`~`#@pmeVZRIImygE(<7M)14ERkbe=a>wEHeMTOh)Q_Irksb$e*+MNg=;AFT=kT zK8HUk*3ZJH{da}W?2B1nvieK*{u$Z+OZI;1Ek8-0<)2mjH|hJ*>V60xPs^iuLo0mM zyzd8tI0fRIuX6W0f|~ycLCuZ7BB)@+{{pZ6*Mn&7-&fd@A0QfJ_7BSJ_lo>J8vP`+Uz+D1h|~O67-}B$D-3n| zH`Q4G@2au+?-&|n_Ln06rN}>S3;vPu<$KHiNsa$`5S_>2{HC)=diR`obz4!eg{*_KY^*`AHmf8k6>#4e*sJ_ zf1$JAN1ZW$08<{p{$B*s|Huq!kokWHnDWH-=NVGVU!kexe<_+;{)DFHf1&ALX!<`G zP0hcfsrk=n%2Qk~9_Iavmz1o2A(=moQNN?<56S$Qx&G(g$^VTBU$o+@;Qw5Do@o60 zX!?(;<Bb{3OU!c<=zmVEr(kXAc;QzODYVqGfr{;g4(;>g2(;@$*bUNfGI<@$V zPXD6Q|6z10%ZcIbG76d!9}*W~P$4gBFUpbTclguJDwhoT?$I-`L17I(s@w#t{ns;> z{Q7fkHaVj;qe=(NO>BC9?j_bws-#wS#pu+#5pBJd(a}ZT<;7+k?UF&2 zmX9TT!HW1+VlefrnE`d3h1B$D0BR+UBJFRsz^iZtdh5_Ba=|K(rjHJw3s#F`2>*Fn z8Lfcl8VteKQ<8cJ#loWv#kf59C=PqR4-5u9gnU&sv{cLn{jEW$d1e_FN!X&_i#aqU z;RE`F=h3zG2KaHJF3q{8)*P;|naC?=k!4Lf*wGkEZf??mwy0E`e6J7adoLv8b3U^v z{gt4hxF3z(IT4bV__4+>IB;+50y0cX*G{%amU_N%peyGRa%jh1yq*?}+HcA+qQsO6 zU&~=HZA&B<4R4TPhvi_?i7I?I@grJaJ3t?j{q*!~5nOj;Ei0fl6NG0(LE633;BxOc z@h@?pqlKTs#-JAPn)L=8HP?YzK{;uCJ&rh+S%sQda*~f5c>}Z-hNs6wB zoyuCT?#5U4g3V*&i-@#CJ}urT1N-AoF@^KqvsXV3=hjWrBQ@H>M0nzI?Arc@HeEEd z3yU5~cMCs5_0ij*{R)Sg&Nu}-Ch3r4DpSCFxDpP!hRw7{27@oegW(}j=9Jq`dT?kX z-0=#h1A8AewT#MO@-!&>uJ|LS#%N%0)-K|lyMis$GJ_GrKd^NZ1*mz}QMS*Km$$ns-iO@iSVHbj z>ZPw_Gl-+1F_z2ck!vDdtV%5zXZ#S{yyPKO&<-HkA8w*gZV@y*1?n-d zl$mF*hwAbya_@!G7IO)_TWt@AEVO7uWe3UsV2h1$R71FwI!)dW z!Je`>?p_`HN?|EoRQ;9+oLNLI?K&{Y%K^Q6jBw1TiNyQZJ?4tiO6WG`G(SA;M&)%c z(Da**xN&0|btr9zC|4(P{qi;YpkX3*2F6oW>!FO8lvF16Lh>`8atYE6;1Wt zGDDq$XjuAtV$+#P)l6kV5SYQJNiOMeP60FAc~xdFNaMWZ7R9jhc;-6 z*u81|$n0r93rSw{xp!*h&{B43QwBGdj@S3%#)Q46`&Tq$@Y_Z7f#Yj3%V!>C)pEG6 zrT1gM;fPs50@xICnb?H8!Yliuc;BIgin^6>hr(?L3EWPN8hfakdJdI4*A5w%YZz<( zCb*kiO~hY4Vee+T&@wS?IxlcLsvSLyk$#7Wg57mSYO^ibG_Qi?kNXI-Goo;$kr!5} z4WdamH=ywQTKY<38>l}QK-&q$WOQsP_FnA7aqH7)W$_N2S-cKT&K^LWk22JMP6mdR zlwt6;t<3t9I#j}G2jTNkr+HUgP`5sqYOgy&E?;6%uWLWt(CtIt@h{;@?At>HnscdH zRTEt{I*-_L8>rs%1GFH&jLf21d~3BJXGIms6q-o8V)h~JIYMpxhR|-anK;(4 zAG>V(MJg@P#um;_L;w9lF>{j;-R2$x#{~~_!xR>iXk&l8tY3n23U)KMY-O6x?bIjH z7o4%Z5Sz&kQ`~aAnO+gufD7GRVQBq3TyXy!GuiSqXuG5{hp7*()>=vP-T3WHn>2Bj z%{BVibRlzl#BQ>#MUy(mIWoc740#^Eg%RHIgxwu2ZMRy?iB3uorW37qH=eyZ1btu2 zgUEz)6qet!oiCDv=Dhs2Faw(N9&E$ldSlF-SO9Wu8F<)nAuPS*K-JB3nYM>}aF|ya z^&9Bh?B^ARX0FO?W4R#i&l&+s*&rf1$Cfysh$VMFJfzNSJ@TiEH$V4o#@Wklv9@On zTz*%8ZzqT$S4tdmM~@|5ug?Q+6Gl#knw`>pEO%Hp0-x2wbeMz8Z6zyLjWp|-ds(EQ+ zA*RoGKq9>uIB3#xZtgbVJs zz%#4{L%U9b0U>N&#}!8R{ax%Ss)m*ImdwJCZOqUs?#xP`SB$hr9+o#qqSF0?n6G#M z2USYiIj861j^P~Ix1xmAIrWyjxb8%IO={@dxIVOHe195f9Y`0;MA0i6V)SL7b-)?S zFpt9ffQQ|EDp7P7y>biabr}bGVOudOO7zev@6W((Nf)wgx;WXdD+ApXzL?X$o&@w< zAjLt>lq}37tz8>XytabOSZ2a({A@vH7_Xqcw=^(umQnM{*eX~(?jlZ|!1vyBXpf8wy(60dBN)nBPdE?l z0S8gjYXLQO8h|=Migs7q)aYh%5^NSvr`gN*l7h!am=&gb$of?R%`ax3z*`NJT1;83N4?^$buDk3mb5oqc@7~w07 z$MQCj@iUIVydqh;^yzDS_VFc+3rVEgQo<;oKo*g0slqFHJ-GXcLgUI6&+!y-g44>$ z=$&7UvzIJG^V2r8Y_2NW3_Xl8uH$eB4r!D;oJ9>P>_s)c*3EC+{9`+qONRY(&BfR+ zPL$>-Ij|y!jG9K;?xB{^wxlCHi&mUaMfE&YGBD*S)o31zF6v6?^idmMa2OizKtB+dH5_`aA_B)JD3eqC3z;kM z5d2^1V9dG!*i?3xtc@uH*=d^1cVwh-hPVP)cMVq`e#C*6bG zlq^zpXdrE$qD`5*8_8Xhp)}Ao3of77Oh#yXv&)=$++~vq`)mqGf3b2{H}EcTs96tP z@u#`Z zD;03iqib-WW+-;J93q|`!lY@A1ld!4oIKxMOk3o4a8tL8rrJ7k*dVOSG)5kR1iLs?`sCK3nRcg*$ceaQru0*JVBo$>3d zhPE05P+FajTB^9?XftWNuxT~P`81ux-nJ#r41MALpv#Q1b}RX$zJu$L(+^)czG5_M zJyHAgNEiz@$$*aC#ARhB)EDVu^QWW4NlF9_J&u4azdK3ob-+UvtBGE~RCqqQo|H!9 zH)R*FB;d#qkhqW!0d2$K)4EhN*DGn78Jq`mZTQe)#4#q-vYeS5%a04&oKfNg;-~8a zu+ou>;X>wk>+u6N^t~(Yj^9L_stI`X-iH37p18|n3h%QUo-ks`Lg5OCtMJDiwfy+B zN)}Uv#=s1%1#FDtXSmDA;j2Bl?9!VeRC}yADzDK8{-+Z$uwo>RaOcCD%aoB*ABrE> z3E)W)6Dsch8X8-ZVE@&tj8(4@mR6R*rU^DExxpWOTJqs@-e?%UU4YmeIS)@gUXsS8 z#~?CjFfwRIt2@jQ^AYr#Z^heZW9WE*QuR@ zJITE4VX2o~F6&VH^Np8VpONnfg`gqo#_VC_D#%Hn{!NLbPLFCucz$? z=2E$jOQ=#zJ{~(a6nqZ#kg@t<^h1X{`Mh5YJ6bHs9<_7OEvrjNR|_-lf-HTyu##?H zF_-H1nxaLL3VA#J47pQhN?KgD5$DP*u3z$g=K9k@RDEy-R*cXfpK3JlNbf50jI#hu zUu|F)oE3z-d0JF=tPwsbUqN0C`bgd{i9w0Wez%m1=4|Q$R^oOfD&Jg0>RwKO zDWVzVWV9R1jm~2Pxvk{Roe6BuppQ(z3w3Tvfi~Emkzr2RxLbo1EV@!*w+&bYGIf z41e$$J}R68zvLH0^UMHR61tejkniCYrwme}KQld-(uwhf!FXtRGs*MjP-nfq*bB2! zmoFZ6boz4b^)!H+P|9#0785U}agIfxH;x?;GQ4GjLgku8e7M9i3*P&9BgY*^z;)R;G9{|9^O?(>u^%^nX$dpEHL z%j%)7|6rK%Nf5Vh%*4FMt03Ckoot!Ip(Xq&&=x)rqhC{!(l8!+WJ_?K@Lnj1Sq$>= z^$_>Dp4>mv4o|cWz_JPz90l{4F``4Tts((tl`UazJetG2+VBx7y9>Biu5xg^g$B)t zd_e9DeZV}@5yP9w7l7NUh0!?=xy@IiafF8s_AD&`op!ffq~;lYwz(d3qj!907YYTUyn4&X_dr*(pcM zPbRWsI)j-rkB36@28PX*Qm4$Zqiki>6?Ssg1MrA^OcXMOfO*;mQIhjX`-2<^oOO}W z6&rxJQeHF5e2>G&BioVjN`~8fpvx$%^qq(YjpO`DYO)rMA){{vle6b&Wjtvm$=$L|CV)V7jo8 z4;=ShBQ?ejP}2I5QO${CWIrv35TEmn4k~U~S}F_5ai7`yh6Z>x>kc_5z>lVJcbFB+ z%gEG$H<*Hl&!J~U3^cklz`Nr@%)Pf&%#;oHp~GwibT5~OH>GaqxnG}l=RP8fO-7?! zuq0j)Sj;@DoeDb-ytv~GH{L@LKb>Sf}5fXwP{%m$CdX2y)+RYHmQ+a zlls#=<+bE;gb?-HPy~IeXVIxynY1sd$55rEW zAhCtiTr6M?wCj_4@tH)YY$R_i^@EPxBen+_Gjm7M532sUw6i4GLuR z#9Gp4g*d%GD~Elw)QrqkkjK6j-e7Vvj&91#f)@kLsJ%3gJRc;JTk{nuo+v}6pb)Q( zFM$lzwXkN50^L8M6^c13RQj1PXofss{mqa*%Pe6YMX8hO8Y#NBUKlUcdeF$#%i;B| zcihNWZB#z4&agK{?DAUYlR~KmGJlB(wH|4M$)0EN{HO>r?)hC}sFnp| zTSf4g@F`T7E(|k`wlLhH1#qN#9Ev&G;rJ&vKs#ju_P?SIXLr>xSzUhk?1mVduw@12 z>ZCAzN=}2i<9yIe2*+06P=NsAPsg*-dO@G+?IX;NKn zNjP!V6ZZ++fu8GzFg;!z=9Pt_bd46(h@3~PY^OrSwQyQjCF8KamuZky!8c|eFhAif)3x{`6Y+3&qtB=lVAIgaOs`%` z953}j{=>6~Mv?)M^Q|Jqoh!-s6PB3Xq|de;BTQZPWaOlNW=<~U%@daug0Hp~dgcOn zw@E?Kg-B3ObwpQDZS=D)gu~A5@G)2%GgmZ0O4>bGH2oU%)XI?oJ)U^;!#ddd{xR$c zSc6$d0#VPshS3{46ID+(LFpwqEW10A)xI>JdMBuX#bsyQmAMNpTjdfJA1SD1c=O12 z1<8F`Qd9`hvv@JUoA;hvVk6D3|yEai#GpLRh1LpcYzR2%!HeckI>hAgL;En9j|I zV35H()=E>3wAdd3zDZ&@DRK_8-ztZuTMQ&M&xf-rk>lvJPCit_L2;e`d1> zZ={k=1#reHn+C;h0ybUqGpR93UI4FsF# zsHxHkc{?yLc_+7x=i{Rz3c+dZL^Q24M5g8<@W06$3Wj#DXx(Yx26)pJ=cU})Dk3C6 z=^%?Al$mwOT};JRSsJt336fJdB+z^ZtZx}Z1#H!Dk*Pmu%!>zW-6$%O(?T}95WzLW z*OFw#c5?cT309@%uv4t>gF=QbUA)x{$*0yP@%Ytrg}5WCiA0jmm-6741P3df2C~og z_Q#Dj>xsN=69`r80jymH_lMTPh6if6YLY9&@2z5Lw(?6e`_$p0Y|WIx!@rEk|k!Grhgpz{ORoECG8>obzu)h`}5I|Vg45552* zZT!6Z#9-X-bcAVmsbDL{{Rqka3Uo^EYE~sdkiOF03eY8uIYN72%WO(s+@B6dmrsTw zISpJbT}1-66p=|`#hBA+gv1Ve065nmm6M1=8IDu%7EmJkmNy`ON&LS<&WjvAZAoKXfBSL5grP<}5in zy8^Ig}+sXYhNPE;^~dU?#T%P(Qt{!2`aFQs}sl`#}MN55w3Z%$+C?u|nCX=9om zTqq}jFHS*n-wU8Wsue~T#P7{mRSqJ@cfr}!`KYLq4g*sAk^>`#V@YlqCM>VP+k@pH zZ}=j%LrNNxN4y|LZ=}c@aUS2T;PvTF1!DPHk3<;x;h1c7ay#V%xp>;DNmkkt#lR7_ z3u@A~s1HopRB@bmiyyai-UZF;%DCW;Hfh~89c9KFFzJE1ICNwKMAWphal3_4adIE@ zo_iS#9<@N#$Bi)W^cv7O^9sE4#c{ETG}dvqaAiezG`XLffvZo?#wU}E$h5*oynD8B z;0nIvL9q_K?EIOrO*zXhtUm_=rJ3yD6Pk2^&U|8Dc#DxuGekeBdhVTP_AqK*7hCDd z0%v3$3`y33Q~9dE{!GFh0|0b-to9VP`S+nx&%ATAw*n4G=L4jVNe_uO-$(+wo3 zYRnNfUqFZnvR?(WxzFJE$Pp;XzmP;0Y-PQdDdCV4fh2X{8(V{oCipl*7oMEThMGaQ z*xBWK$)we(P_$A3ZpMnymiqqG&)pckYSxl7KKe}A+XgVt4kwjL!R(}$PeEv8Us6<~ zf>SfJ;qvZbjK^EV(ltGd>dfURx40dYCVwW0m(*dnnm>vKC%_@+3=kf77+f_*VZd$) zR7sIQ)5qD|;m=i|$4#7`U&BRnGZUP?WB?prbeGwF_dKgKQ3oXt`C=z70l$szB3M>X5eai?GdB0Nv+uaqZ#BXgj|j@#=EHjh-^}gYyZ- zK{O0MZ&$|Gz87Jps|5DQJFp|F4Twp9Ih64WLx}^+A+u#HKCV~! z>m_jUI50&eL`72%n>11xgDQCx)ILdMx|OK;2T!n{VTx_qzWA#99djwI87hMMKwH^0 zViv5yE%;c=Y)%`4dt8>0u8k|_o!O0t5H~R0{1(L(HDX)a^_JShinyw{B77oS(*18xQHI~E$FT$t!t1!-l zk8E^w22Jl^k~!QJ)>MSUpglB$xzN599DMvqf2I$HaJhIV;1WDHnNL@8)`3aY9bgNRsqD%yuo6ncR@F=< zpkXz)US0s7H}#`i8?|xo&;x+4E;KE5pTM*YoQ1x35#6`AQd?IeqV@=(YSJ7`oO2RP zj5pI0-FlQy^*OsMbU0Nmpj7z~5U1#^r1{`7a#ha?SIHhHgR)#OHY1Z!y0(g{u24YV z!3OBIu8!HSun5-c2-4@{vvI7E93AaC7&{W&=*H*!A%CbjrY>7W*WFzRAwjE2Eli=F zQ&+=e|7BFR>?F)FDuoNOm%!K1i4xvi^yl$IN#le>c=W~{>m?{ZUJl0^*EBY7%*S)#olL7>1e}{wg6CH_ z<6)mf+-aB%3pZ3Tr}tW-+vCk}rs*9sxatGsj64tWud;}OXfQ0gRED4W^fZ|{+JZix z9s~0qK~MZt?5#Tu87qfURZvBNH(DSO*1`-5+(k0(3_;E36JTZjHdcI>Gz1A4Byn_jt^^oPQ@se(B zGcxRL1if@(DBSo^ONI=#!dn`WxXdCMqGc|ETDu0m8kj}&BN&A1OCthk>nDpWqDC=+yGOIlx)nAHM*|)=1#U1c*qC8nCI}>a#_s0c6 zQp9(d1k1g;23nFR6ZzH+E}cyUhX?1`ttCdVeNPp+s3VU@?uj+k^!)@qBimpWA)tL; z1xHTpV9uRtXCgaoaO(TRplrGuY>$fN2(zu`F%^~Q}I$E)IBzd<(2l=gYnPXLBiSr!Z+)43va+=>BLkbvN<7Q5k+hnnN z?IWf+yM!D&T7)*cmC>oyAIk2nWcCFQMD>%6VACfDedE`mRhuR)7py0%&+E~$Ezu-L z?Gs~5@`-v~9-(_Bu|{hSv&V27duqvgQrD>gl_D##=20p18?g;E>&0;cZ;rC~!(0p* zxdN1LG%~IMLfmIdA4QBj~|AD_xn=T zT#0Lbq8?<@L*b?Qa>9T1F!%8rTYNWeH#@Y;3=fUn4g;UT{Lu^GqTSwfzB%QOBJfwr!tJuhcgrDqst-9Izi=9bOE0 zbu_u1Mu@-m6(aUx2kEHwBKmK0L4UCx)cZ{*Q<}EH*^oN2uIe87aJ7k9p`TAy7CvM? zCO5%Gjd*fu?RFx#L>nyUwv)Jk)3D^kbkZk&3asks;H_2Ek$uZTm_P-2SX;LPRFyA* zePLIV^{6B4iBZ#;^c6}ZI5v-aVERj9eZG!W)SU+@9v8v+S~@-z(A~RRq8C11?gb;C zRjkc3O&Gmx0rz?%z_a*dnx4K4oVJRbZ zjW8wKMPYL61hjs`GA}-5Fb>0RGa{Y~$@EX>2!Hox$O^PTOT9c&lD`we>$_Ny*H>81 z}sJJ?UD1+Am)Al`ifMqjICf)*cc95&1pV>0wCxGb7O;_5aq6OXGx zc~^hfJ@X-n+93lvn^oxh)oSF7hYx+wxRaf(-@phO@Ke2y37C7O2&Nvt3%i&2vh3DT zu+`%g8KF5G(oW2voxL02)vle)^$ELSV{r^4n_f%0i_*!nIium6)o40;r!kYULK72Z z8=zv=Vj5Xk#rVFu)Z~zS3ATK?LsF-1Ag0mEXeGvviAUGsK)r5wb+ryYT=gfI4)T z*+rpXX1)jP*44AQi*7^CjAhV%ID_1e52l45`_h-Z_1Jf36)^siKf3#v;`?(6I4eMn zIpA@e+qr27jQa2yd~OH9;6wRLl)obBj_1OJ^?hjR5^apToeZOM^+DV5HTd?-0^h}A z(C>pByJ=wotR7p*45|irRKEhW1@oY0jSAM8WP(n*Wn(FCo-}c8RpY|!YG!qZaZ_DV z3#4yf1Mi0jaU*WXQ9ZjFND7u@H%!U_gED?mji2{Tdmci@d>`eW@Y6M}@F+ z&kAT=F^qLx;6xuCY$ZKUPC6|BK1>)%Wavh`Ae1l=OMKec2 zw-QdlU`E%5Lchuq)Z)!&)pnHOz_X`#yb^+f+ZN!-1%=o$f>*M~a(M637Aj_UAGSSI z2K$g=EMuhbu`^+M500cmq&AUpGy7ti${V&+tcUe0T86I+&l17Dt?cv1w@GpD1qhz2 zK)1p!n5kL{Cwcq-N>Zfoh+-$4Qcq?+DmvoJyGyCvrc`_sdxL$ZE=i4cd1Kj51*&Aa z2?x&U!;Nc6rn+|JBw^(YHgo!KefhT zGV@8r@G8>LZGj<+129%n3U}tl;lU{`EVotzOYaw>|A|a|yweTuX?U^uE&Qy0gg8zp zF~+_JYGB{!`)uXW2om9)&L(P}A%mnlVZfUnk~Ss;ZAYcR>^pbJ^%0Tm3vD0jwyBYA zSBZm-=_ANy$-Y>RFPH(-v!O7_2A5`Qput9OoSvgX-8L>KKBoH6aVC_!9QgpcBtEn2 zy2g@-6eX&4PXnu}%+Zp!7r}h!Zcv?e4DwiejIRzS8{PRSe{TWxzdn$f84tx?g@Ncb ztOz=j)G?K}hccor2a;^Z;D;S~#XcJ!Lta%u2ZjN!vK|rBXZl@^cfh+onrH zd$ov%_a34#))y{U%ttMyWiYNfAEP!d1dm2(mS{%c*+pGs!t1MeUnm1Vwb;-Z+`jm1 zv=O{KB2L{zr5Jvna6G%oj*LwijCI2X5y|Q%-db!G_)E;AXIqqUaIOxrJ(XB1E)4~) zs$}seF=)7aiv%w5hJ{;#(6B6zym%%`C8;vKy7WG1?iXj8Ee$X&?k+RTUY(vcl>wnk zs@U3i60YW2f{)xM;@JNxaZGyv7dG-!T?Zj}G3_HOZ!;CT=K-UVCP_1%uO=6Y`r*{^ zQ<%pc8$e;m5fb7mg{8svBw8wwSR3AF2JM|oURtdsQLER&D>HX?#_rKHPg}P6sJ|e6 zvO5>!PmaT0pFEONI-UAe_k)OlMeyXrQP!$`Irb|VN!wGMu_bRLR)9KcAKOWmPPV4Y zeWy_wZYvJiKOAd@eI`ZkYGC|lN%A?|8P?%>xNj)dJf<}WhK?_SVF#s~kLk@~GZxAr zcjYiB4Ge*nJF&ET+5m#8?*( z-mS%kMnfbEv@v&PF4uM0Te?nS1Y~&eVW8(6oDfk+lsw1L**1bGRi;C8)@tD=|2x1q z>EOF%*7RonOy;6MC{$j41p2aR`UN)-N!3hNc%d!vEG~xO!fizR z_*}AFXf!<(*B6(eC~*nvPX%n2u!ZLjv72?bG6{11iMYuQDCo^5=)H(|KFTHU?l&@L zA|Aq1M`89TA3v7Ji(tviM`Ws~93j_O(re|0vjg2>bD}LKbnb;!?^nRB1S{GbYen@k zSCZh{SKyN69F!XngxUAAs6ask?B*+>N#(B4u+Rk!TlDF2#d0)TB1|89ZAG#1m%&lH z5NDedq12Yy%w&ml5OIme>*m7Lz|$4OjGmLI3o9|z&xg9dI7|Ba=E8`rZdA!L5gjMH zquk`lxbC(VeJp$dKE0U5m~7hyM#_`e{o97)+IQQ5KcbQA_*k5N;PH{Gg$TqqOvEQ# zb)p_`3!RfTFb8Mb(5nYK$az(UE(=5OY3ESwp%u_U7Le>cA^73WC=5!Fpox=`&{Bkh zpH!0RquC7ie3T=0i#4L{I}Yu$j2}0L2jZ!`Y*hU)82Q&&&>d$|uugOVW{q0g?wxuX%df8#c&RVswEvbW^&@-;B^;Q|T@!FZ)a4N@&VC|tivth()~-FO2!^HdSu zYrMo>OxXwL?Rk4%m)pUJ@IG|vP(L&|8Oiu64QD*>55=L z1aCUW!1O|Yaz}hFZZg=5v*!<@;x~-x72f=dt=x5Z-YtVU+5@;HH-_QSi34$oKZmxh zo`zGL7D1p-AtTdxoSe38Vh)a1fL3QaBBnhby-kAYF7fGP{+aCu4_| zH&g~cBvYmDlPDcqyRO}?*xx&bcIzI5>nWmiHv3Xl z3GLb0K`u0(C6_9;5H;aSXn1Hz98ZMuo?JOf#@YA5zJY?2?U+P{by{Q5%4Lvs>=c>P z5{pr{BlES&Xc%D^s*-m46!G@1Rv>)Ahsw;x9gB{xQ;$@@sq} z-!4Jnh8PWK<^Me*x2zmTMwDMxg40JOK#e(VGP$y*x#dc*#+-ue+>@(|N7^7{MACg80ANN#h0+j9=!K?#D~QEEyu_{7vv1&z4L%K@+WY99vHrb&bQV0 zZNSy|{{D#{?|k^}9VU8zNcUwx`L|PjIpgOvKeYPaO7&w@`eV9(FRyQc|8trjO7m~! z^+QwtnC{=p>ziEtoaRSA`M2`=p$LCW_wVKPt+VNIe!PdmR^xa2(^b literal 0 HcmV?d00001 diff --git a/reports/portfolio_acceptance.json b/reports/portfolio_acceptance.json index a5f350f..492b718 100644 --- a/reports/portfolio_acceptance.json +++ b/reports/portfolio_acceptance.json @@ -62,6 +62,7 @@ ], "reported_source_model_sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", "current_source_model_sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", + "source_model_hash_origin": "committed_git_blob", "source_model_matches_repository": true, "claim_boundary": "Physical AI Hub model profiling; not Android APK end-to-end latency." }, diff --git a/scripts/build_portfolio_acceptance.py b/scripts/build_portfolio_acceptance.py index 1947c51..223b255 100644 --- a/scripts/build_portfolio_acceptance.py +++ b/scripts/build_portfolio_acceptance.py @@ -6,6 +6,7 @@ import argparse import hashlib import json +import subprocess from pathlib import Path from typing import Any @@ -25,6 +26,25 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _repository_sha256(repository_root: Path, source_path: Path) -> tuple[str | None, str]: + """Hash the committed artifact, falling back to the worktree outside Git.""" + try: + relative_path = source_path.relative_to(repository_root).as_posix() + except ValueError as exc: + raise ValueError("source model must be inside the repository") from exc + + result = subprocess.run( + ["git", "-C", str(repository_root), "show", f"HEAD:{relative_path}"], + check=False, + capture_output=True, + ) + if result.returncode == 0: + return hashlib.sha256(result.stdout).hexdigest(), "committed_git_blob" + if source_path.is_file(): + return _sha256(source_path), "worktree_file" + return None, "missing" + + def validate_ai_hub_qnn(report_path: Path, repository_root: Path) -> dict[str, Any]: report = _load_json(report_path) if report.get("schema_version") != "0.2": @@ -39,7 +59,7 @@ def validate_ai_hub_qnn(report_path: Path, repository_root: Path) -> dict[str, A ): raise ValueError("incomplete Qualcomm AI Hub identity") source_path = repository_root / str(source.get("path")) - current_source_sha256 = _sha256(source_path) if source_path.is_file() else None + current_source_sha256, source_hash_origin = _repository_sha256(repository_root, source_path) source_model_matches = current_source_sha256 == source.get("sha256") if hardware.get("backend") != "HTP" or not hardware.get("qairt_version"): raise ValueError("Qualcomm report must identify the HTP backend and QAIRT version") @@ -95,6 +115,7 @@ def validate_ai_hub_qnn(report_path: Path, repository_root: Path) -> dict[str, A "graphs": sorted(graph_summaries, key=lambda value: value["batch_size"]), "reported_source_model_sha256": source.get("sha256"), "current_source_model_sha256": current_source_sha256, + "source_model_hash_origin": source_hash_origin, "source_model_matches_repository": source_model_matches, "claim_boundary": "Physical AI Hub model profiling; not Android APK end-to-end latency.", } diff --git a/tests/test_portfolio_acceptance.py b/tests/test_portfolio_acceptance.py index 83613d3..d0e73a6 100644 --- a/tests/test_portfolio_acceptance.py +++ b/tests/test_portfolio_acceptance.py @@ -18,6 +18,7 @@ def test_validates_tracked_ai_hub_qnn_evidence() -> None: assert result["status"] == "validated_ai_hub_physical_qnn" assert result["backend"] == "QNN HTP" assert result["source_model_matches_repository"] is True + assert result["source_model_hash_origin"] == "committed_git_blob" assert len(cast(list[object], result["graphs"])) == 3 From e5180c79e641db9a3a5ac4ccc8ff48b51c817e36 Mon Sep 17 00:00:00 2001 From: triasha72 Date: Wed, 26 Aug 2026 13:35:05 -0400 Subject: [PATCH 4/4] Trigger CI after provenance fix