diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf5fae5..744cbe3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,12 @@ jobs: run: | python scripts/validate_tracked_qualcomm_int8.py + - name: Build cross-runtime portfolio acceptance matrix + run: | + python scripts/build_portfolio_acceptance.py \ + --output-json /tmp/portfolio-acceptance.json \ + --output-markdown /tmp/portfolio-acceptance.md + release-acceptance: name: End-to-end release acceptance runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 4629421..17e3fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ data/processed/* reports/* !reports/.gitkeep !reports/qualcomm_qnn_v0_1.json +!reports/portfolio_acceptance.json +!reports/portfolio_acceptance.md # Deployment artifacts *.onnx diff --git a/README.md b/README.md index 90c388b..76f3d1c 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,11 @@ Cross-provider timings should not be interpreted as a direct hardware ranking. ## Qualcomm QNN / Snapdragon deployment +The consolidated [portfolio acceptance matrix](reports/portfolio_acceptance.md) +separates validated CI/device lanes from evidence still pending. It also checks +that the model hash in the Qualcomm report matches the current ONNX artifact. +The current-model AI Hub rerun restores that provenance chain. + EdgeGenBench now includes a validated Qualcomm QNN deployment path for the compact neural surrogate. @@ -177,13 +182,13 @@ profiled layers were placed on the NPU for every validated graph. | Batch | AI Hub profile latency | Peak memory | Compute units | |---:|---:|---:|---| -| 1 | 38 us | 122,937,344 B | NPU: 9 | -| 32 | 34 us | 122,888,192 B | NPU: 9 | -| 256 | 57 us | 123,211,776 B | NPU: 9 | +| 1 | 38 us | 122,855,424 B | NPU: 9 | +| 32 | 40 us | 122,880,000 B | NPU: 9 | +| 256 | 47 us | 122,896,384 B | NPU: 9 | On all 900 held-out rows, the linked batch-1 QNN deployment retained mean R2 -of 0.996953 versus 0.996955 for the local FP32 ONNX reference. Maximum -normalized deployment drift was 0.003636. +of 0.996954 versus 0.996956 for the local FP32 ONNX reference. Maximum +normalized deployment drift was 0.003234. These values are device-specific AI Hub model-profile measurements. They are not presented as end-to-end Android application latency or as a same-hardware @@ -193,6 +198,13 @@ Complete provenance and linked-graph validation are stored in: `reports/qualcomm_qnn_v0_1.json` +The authenticated current-model run is reproducible with: + +```bash +PYTHONPATH=src python scripts/rerun_qualcomm_qnn_current_model.py +python scripts/build_portfolio_acceptance.py +``` + ### Qualcomm-native INT8 decision The follow-up INT8/QDQ study was executed on the same Snapdragon 8 Elite diff --git a/artifacts/neural_onnx/neural_surrogate.onnx b/artifacts/neural_onnx/neural_surrogate.onnx new file mode 100644 index 0000000..3fff1cb Binary files /dev/null and b/artifacts/neural_onnx/neural_surrogate.onnx differ diff --git a/docs/qualcomm_ai_hub_qnn.md b/docs/qualcomm_ai_hub_qnn.md index f242323..890a7f3 100644 --- a/docs/qualcomm_ai_hub_qnn.md +++ b/docs/qualcomm_ai_hub_qnn.md @@ -90,11 +90,11 @@ containing three statically specialized graph variants: Linked target model: -`mnl7771jm` +`mnjjex79n` Link job: -`jp16x3285` +`jgk4d8lvp` The linked artifact retains: @@ -107,9 +107,9 @@ The linked artifact retains: | Graph | Batch | AI Hub profile latency | Derived model throughput | Peak memory | Compute units | |---|---:|---:|---:|---:|---| -| `edgegenbench_batch1` | 1 | 38 us | 26,315.8 samples/s | 122,937,344 B | NPU: 9 | -| `edgegenbench_batch32` | 32 | 34 us | 941,176.5 samples/s | 122,888,192 B | NPU: 9 | -| `edgegenbench_batch256` | 256 | 57 us | 4,491,228.1 samples/s | 123,211,776 B | NPU: 9 | +| `edgegenbench_batch1` | 1 | 38 us | 26,315.8 samples/s | 122,855,424 B | NPU: 9 | +| `edgegenbench_batch32` | 32 | 40 us | 800,000.0 samples/s | 122,880,000 B | NPU: 9 | +| `edgegenbench_batch256` | 256 | 47 us | 5,446,808.5 samples/s | 122,896,384 B | NPU: 9 | The throughput values are derived from the configured batch size divided by AI Hub's estimated model-inference time. They are not end-to-end Android @@ -125,8 +125,8 @@ All 900 held-out test rows were evaluated through the linked batch-1 graph. | Metric | Local FP32 ONNX | Linked Snapdragon QNN | |---|---:|---:| -| Mean R2 | 0.996955004 | 0.996953249 | -| Mean NRMSE | 0.050432628 | 0.050444571 | +| Mean R2 | 0.996955989 | 0.996954462 | +| Mean NRMSE | 0.050425306 | 0.050449587 | Deployment drift: @@ -193,16 +193,16 @@ Install Qualcomm-specific dependencies with: pip install -e '.[qualcomm]' ``` -Validate the linked multi-graph target: +Compile, link, profile, and validate the current ONNX model: ```bash -python scripts/validate_qualcomm_qnn_multigraph.py +PYTHONPATH=src python scripts/rerun_qualcomm_qnn_current_model.py ``` Build the canonical evidence report: ```bash -python scripts/build_qualcomm_qnn_report.py +python scripts/build_portfolio_acceptance.py ``` The complete machine-readable result is stored in: diff --git a/reports/portfolio_acceptance.json b/reports/portfolio_acceptance.json new file mode 100644 index 0000000..492b718 --- /dev/null +++ b/reports/portfolio_acceptance.json @@ -0,0 +1,82 @@ +{ + "schema_version": 1, + "project": "EdgeGenBench", + "lanes": { + "native_cpp": { + "status": "validated_in_ci", + "claim": "C++17 reference runtime, tests, CLI, and fused preprocessing acceptance." + }, + "android_reference": { + "status": "validated_physical_device", + "device": "Samsung SM-A356E", + "report": "reports/android_sm_a356e_reference_10_run_v0_1_4.md", + "claim": "Reference JNI/application measurements; not QNN." + }, + "qualcomm_ai_hub_qnn": { + "status": "validated_ai_hub_physical_qnn", + "device": "Snapdragon 8 Elite QRD", + "backend": "QNN HTP", + "qairt_version": "2.45.0.260326154327", + "context_model_id": "mnjjex79n", + "link_job_id": "jgk4d8lvp", + "graphs": [ + { + "graph": "edgegenbench_batch1", + "batch_size": 1, + "latency_ms": 0.038, + "throughput_samples_per_second": 26315.78947368421, + "peak_memory_bytes": 122855424, + "compute_units": { + "NPU": 9 + }, + "max_normalized_drift": 0.003233889676953719, + "profile_job_id": "j567vw3np", + "inference_job_id": "jg9m8d3q5" + }, + { + "graph": "edgegenbench_batch32", + "batch_size": 32, + "latency_ms": 0.04, + "throughput_samples_per_second": 800000.0, + "peak_memory_bytes": 122880000, + "compute_units": { + "NPU": 9 + }, + "max_normalized_drift": 0.0028659358491131483, + "profile_job_id": "jpyxnmvr5", + "inference_job_id": "jpez2yl7p" + }, + { + "graph": "edgegenbench_batch256", + "batch_size": 256, + "latency_ms": 0.047, + "throughput_samples_per_second": 5446808.510638298, + "peak_memory_bytes": 122896384, + "compute_units": { + "NPU": 9 + }, + "max_normalized_drift": 0.0028659358491131483, + "profile_job_id": "jglx7lllg", + "inference_job_id": "jgk4d82op" + } + ], + "reported_source_model_sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", + "current_source_model_sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", + "source_model_hash_origin": "committed_git_blob", + "source_model_matches_repository": true, + "claim_boundary": "Physical AI Hub model profiling; not Android APK end-to-end latency." + }, + "android_qnn_apk": { + "status": "implementation_complete_evidence_pending", + "claim": "Build/JNI/capture paths exist; requires a supported Snapdragon APK run." + }, + "android_16kb_runtime": { + "status": "packaging_validated_runtime_pending", + "claim": "ELF/APK alignment passes; runtime PAGE_SIZE=16384 evidence is pending." + }, + "power": { + "status": "not_measured", + "claim": "No power-savings claim is made without a named calibrated tool." + } + } +} diff --git a/reports/portfolio_acceptance.md b/reports/portfolio_acceptance.md new file mode 100644 index 0000000..5ef849c --- /dev/null +++ b/reports/portfolio_acceptance.md @@ -0,0 +1,24 @@ +# EdgeGenBench portfolio acceptance + +| Evidence lane | Status | Claim boundary | +|---|---|---| +| `native_cpp` | `validated_in_ci` | C++17 reference runtime, tests, CLI, and fused preprocessing acceptance. | +| `android_reference` | `validated_physical_device` | Reference JNI/application measurements; not QNN. | +| `qualcomm_ai_hub_qnn` | `validated_ai_hub_physical_qnn` | Physical AI Hub model profiling; not Android APK end-to-end latency. | +| `android_qnn_apk` | `implementation_complete_evidence_pending` | Build/JNI/capture paths exist; requires a supported Snapdragon APK run. | +| `android_16kb_runtime` | `packaging_validated_runtime_pending` | ELF/APK alignment passes; runtime PAGE_SIZE=16384 evidence is pending. | +| `power` | `not_measured` | No power-savings claim is made without a named calibrated tool. | + +## Validated Qualcomm QNN results + +Device: **Snapdragon 8 Elite QRD**; backend: **QNN HTP**; QAIRT: `2.45.0.260326154327`. +Source-model provenance match: **True**. + +| Batch | AI Hub latency (ms) | Throughput (samples/s) | Peak memory (bytes) | Placement | Max normalized drift | +|---:|---:|---:|---:|---|---:| +| 1 | 0.038000 | 26315.789 | 122855424 | NPU × 9 | 0.003233890 | +| 32 | 0.040000 | 800000.000 | 122880000 | NPU × 9 | 0.002865936 | +| 256 | 0.047000 | 5446808.511 | 122896384 | NPU × 9 | 0.002865936 | + +AI Hub measurements are physical-device model profiles, not Android application end-to-end timings. Current-model acceptance requires source-model provenance to match the repository, as reported above. +Power remains unmeasured. The two pending proof items are a supported-device QNN APK run and a runtime page size of 16384 bytes. diff --git a/reports/qualcomm_qnn_v0_1.json b/reports/qualcomm_qnn_v0_1.json index 42e2061..860e1a9 100644 --- a/reports/qualcomm_qnn_v0_1.json +++ b/reports/qualcomm_qnn_v0_1.json @@ -1,11 +1,12 @@ { "schema_version": "0.2", - "experiment": "EdgeGenBench Qualcomm QNN Snapdragon 8 Elite deployment", + "experiment": "EdgeGenBench current-model Qualcomm QNN deployment", "source_model": { "path": "artifacts/neural_onnx/neural_surrogate.onnx", - "sha256": "40f588b329b98fdaa38a7eda202fc89573fce5e91a4193f0a0a93b3142c0382f", + "sha256": "191927e05b5f82a539f0ad35c78dafe7f969cb4e57e9c76556cb9f35053e658e", "input_name": "features", "input_width": 10, + "output_name": "predictions", "output_width": 6, "source_precision": "float32" }, @@ -14,129 +15,37 @@ "device_os": "15", "chipset": "qualcomm-snapdragon-8-elite", "chipset_alias": "sm8750", - "soc_model": "69", "backend": "HTP", "hexagon": "v79", "qairt_version": "2.45.0.260326154327" }, - "batch_specific_baseline": { - "1": { - "batch_size": 1, - "compile_job_id": "j5793jj9g", - "profile_job_id": "jp2en736p", - "target_model_id": "mnz66wwxm", - "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", - "serialized_model_size_bytes": 53248, - "serialized_model_sha256": "fd4dbc74b39607d0126f53f3838737197cc5418c2920786d49e0a01d1d412c00", - "compute_units": { - "NPU": 9 - }, - "estimated_inference_time_us": 32, - "estimated_inference_peak_memory_bytes": 123199488, - "target_metadata": { - "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", - "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", - "ModelMetadataKey.QNN_CONTEXT_BIN_VTCM": "0", - "ModelMetadataKey.QAIRT_SDK_VERSION": "2.45.0.260326154327", - "ModelMetadataKey.QNN_SDK_VARIANT": "default", - "ModelMetadataKey.QNN_CONTEXT_BIN_OPTIMIZATION_LEVEL": "3", - "ModelMetadataKey.QNN_MODEL_GRAPH_NAMES": "graph_x4h9d3zo", - "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" - } - }, - "32": { - "batch_size": 32, - "compile_job_id": "j5m8y4xwp", - "profile_job_id": "jp36280lp", - "target_model_id": "mqvkkk3jn", - "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", - "serialized_model_size_bytes": 53248, - "serialized_model_sha256": "36de3516ebae290dfc445d20d3db435f48aa2b6c531e052d2671d954319f5b99", - "compute_units": { - "NPU": 9 - }, - "estimated_inference_time_us": 35, - "estimated_inference_peak_memory_bytes": 122978304, - "target_metadata": { - "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", - "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", - "ModelMetadataKey.QNN_CONTEXT_BIN_VTCM": "0", - "ModelMetadataKey.QAIRT_SDK_VERSION": "2.45.0.260326154327", - "ModelMetadataKey.QNN_SDK_VARIANT": "default", - "ModelMetadataKey.QNN_CONTEXT_BIN_OPTIMIZATION_LEVEL": "3", - "ModelMetadataKey.QNN_MODEL_GRAPH_NAMES": "graph_f0fyaikk", - "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" - } - }, - "256": { - "batch_size": 256, - "compile_job_id": "jpv7x4kjp", - "profile_job_id": "jpey32m15", - "target_model_id": "mmxgggjrq", - "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", - "serialized_model_size_bytes": 57344, - "serialized_model_sha256": "b1e55d5220240f9d7119fdb72fa12c6aeaf079f260f9e452c4fd01d47f284c18", - "compute_units": { - "NPU": 9 - }, - "estimated_inference_time_us": 54, - "estimated_inference_peak_memory_bytes": 122892288, - "target_metadata": { - "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", - "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", - "ModelMetadataKey.QNN_CONTEXT_BIN_VTCM": "0", - "ModelMetadataKey.QAIRT_SDK_VERSION": "2.45.0.260326154327", - "ModelMetadataKey.QNN_SDK_VARIANT": "default", - "ModelMetadataKey.QNN_CONTEXT_BIN_OPTIMIZATION_LEVEL": "3", - "ModelMetadataKey.QNN_MODEL_GRAPH_NAMES": "graph_bqjx81_9", - "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" - } - } - }, - "heldout_batch1_baseline": { - "sample_count": 900, - "target_model_id": "mnz66wwxm", - "inference_job_id": "jg9de8ll5", - "device": "Snapdragon 8 Elite QRD", - "device_os": "15", - "normalized_mae": 0.0004096813499927521, - "normalized_rmse": 0.000530485762283206, - "normalized_max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.0004119437653571367, - "max_normalized_drift": 0.0036358777433633804, - "allclose_1e3": false, - "local_mean_r2": 0.9969550037922089, - "remote_mean_r2": 0.9969532494078602, - "local_mean_nrmse_std": 0.05043262755109571, - "remote_mean_nrmse_std": 0.05044457139336909, - "r2_delta": -1.754384348684468e-06, - "nrmse_delta": 1.1943842273376415e-05 - }, "linked_multigraph": { "compile_jobs": [ { "batch": 1, "graph_name": "edgegenbench_batch1", - "compile_job_id": "jgd2l0ne5", + "compile_job_id": "jpyxnmd05", "compile_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n" }, { "batch": 32, "graph_name": "edgegenbench_batch32", - "compile_job_id": "j5w4nxw3g", + "compile_job_id": "jp0jk610g", "compile_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n" }, { "batch": 256, "graph_name": "edgegenbench_batch256", - "compile_job_id": "jg9de80w5", + "compile_job_id": "jp8x813qg", "compile_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n" } ], - "link_job_id": "jp16x3285", + "link_job_id": "jgk4d8lvp", "link_status": "JobStatus\n---------\ncode : SUCCESS\nmessage : \n", - "target_model_id": "mnl7771jm", + "target_model_id": "mnjjex79n", "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", + "serialized_model_size_bytes": 106496, + "serialized_model_sha256": "43d7cb889b0dd97d8de3a48557fdc7dceb322e6c7b72fdb91b19e5473f84b0df", "target_metadata": { "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", "ModelMetadataKey.QNN_CONTEXT_BIN_BACKEND": "HTP", @@ -148,8 +57,8 @@ "ModelMetadataKey.QNN_CONTEXT_BIN_HEXAGON_VERSION": "v79" }, "validation": { - "link_job_id": "jp16x3285", - "target_model_id": "mnl7771jm", + "link_job_id": "jgk4d8lvp", + "target_model_id": "mnjjex79n", "target_model_type": "SourceModelType.QNN_CONTEXT_BINARY", "target_metadata": { "ModelMetadataKey.QNN_CONTEXT_BIN_SOC_MODEL": "69", @@ -175,14 +84,14 @@ "graph_name": "edgegenbench_batch1", "batch_size": 1, "sample_count": 900, - "profile_job_id": "jp4308d85", - "inference_job_id": "jpxx2m63p", + "profile_job_id": "j567vw3np", + "inference_job_id": "jg9m8d3q5", "profile": { "batch_size": 1, "estimated_inference_time_us": 38.0, "estimated_inference_latency_ms": 0.038, "estimated_throughput_samples_per_second": 26315.78947368421, - "estimated_inference_peak_memory_bytes": 122822656, + "estimated_inference_peak_memory_bytes": 122855424, "compute_units": { "NPU": 9 } @@ -191,34 +100,34 @@ "parity": { "sample_count": 900, "output_width": 6, - "mae": 0.00040968134033459206, - "rmse": 0.0005304857849774182, - "max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.00041194377879442924, - "max_normalized_drift": 0.0036358769310459917, + "mae": 0.00043518983793479425, + "rmse": 0.0005554710568707642, + "max_abs_error": 0.0033278465270996094, + "mean_normalized_drift": 0.0004384562793800826, + "max_normalized_drift": 0.003233889676953719, "allclose_rtol_1e3_atol_1e3": false }, "predictive_quality": { - "local_mean_r2": 0.9969550037922089, - "remote_mean_r2": 0.9969532494078602, - "r2_delta": -1.754384348684468e-06, - "local_mean_nrmse_std": 0.05043262755109571, - "remote_mean_nrmse_std": 0.05044457139336909, - "nrmse_delta": 1.1943842273376415e-05 + "local_mean_r2": 0.9969559893499255, + "remote_mean_r2": 0.9969544620647843, + "r2_delta": -1.5272851411340937e-06, + "local_mean_nrmse_std": 0.05042530552302998, + "remote_mean_nrmse_std": 0.050449586683723985, + "nrmse_delta": 2.428116069400771e-05 } }, "edgegenbench_batch32": { "graph_name": "edgegenbench_batch32", "batch_size": 32, "sample_count": 256, - "profile_job_id": "jgnk8xmkg", - "inference_job_id": "jprwj9205", + "profile_job_id": "jpyxnmvr5", + "inference_job_id": "jpez2yl7p", "profile": { "batch_size": 32, - "estimated_inference_time_us": 34.0, - "estimated_inference_latency_ms": 0.034, - "estimated_throughput_samples_per_second": 941176.4705882353, - "estimated_inference_peak_memory_bytes": 122888192, + "estimated_inference_time_us": 40.0, + "estimated_inference_latency_ms": 0.04, + "estimated_throughput_samples_per_second": 800000.0, + "estimated_inference_peak_memory_bytes": 122880000, "compute_units": { "NPU": 9 } @@ -227,34 +136,34 @@ "parity": { "sample_count": 256, "output_width": 6, - "mae": 0.00042206487463166315, - "rmse": 0.00055137047774592, - "max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.0004078000437142555, - "max_normalized_drift": 0.003222448271801646, + "mae": 0.0004328816042592128, + "rmse": 0.0005609981854555366, + "max_abs_error": 0.0033278465270996094, + "mean_normalized_drift": 0.00042132448724577204, + "max_normalized_drift": 0.0028659358491131483, "allclose_rtol_1e3_atol_1e3": false }, "predictive_quality": { - "local_mean_r2": 0.9971241567768717, - "remote_mean_r2": 0.9971200615281508, - "r2_delta": -4.0952487209144905e-06, - "local_mean_nrmse_std": 0.04895350495151681, - "remote_mean_nrmse_std": 0.048983762073770726, - "nrmse_delta": 3.0257122253914137e-05 + "local_mean_r2": 0.9971158433783592, + "remote_mean_r2": 0.9971139833803742, + "r2_delta": -1.8599979850764115e-06, + "local_mean_nrmse_std": 0.04900140374890117, + "remote_mean_nrmse_std": 0.04902444807450163, + "nrmse_delta": 2.30443256004631e-05 } }, "edgegenbench_batch256": { "graph_name": "edgegenbench_batch256", "batch_size": 256, "sample_count": 256, - "profile_job_id": "jp2enj9rp", - "inference_job_id": "jpym0nj8p", + "profile_job_id": "jglx7lllg", + "inference_job_id": "jgk4d82op", "profile": { "batch_size": 256, - "estimated_inference_time_us": 57.0, - "estimated_inference_latency_ms": 0.057, - "estimated_throughput_samples_per_second": 4491228.070175438, - "estimated_inference_peak_memory_bytes": 123211776, + "estimated_inference_time_us": 47.0, + "estimated_inference_latency_ms": 0.047, + "estimated_throughput_samples_per_second": 5446808.510638298, + "estimated_inference_peak_memory_bytes": 122896384, "compute_units": { "NPU": 9 } @@ -263,37 +172,24 @@ "parity": { "sample_count": 256, "output_width": 6, - "mae": 0.00042206487463166315, - "rmse": 0.00055137047774592, - "max_abs_error": 0.0037398338317871094, - "mean_normalized_drift": 0.0004078000437142555, - "max_normalized_drift": 0.003222448271801646, + "mae": 0.0004328816042592128, + "rmse": 0.0005609981854555366, + "max_abs_error": 0.0033278465270996094, + "mean_normalized_drift": 0.00042132448724577204, + "max_normalized_drift": 0.0028659358491131483, "allclose_rtol_1e3_atol_1e3": false }, "predictive_quality": { - "local_mean_r2": 0.9971241567768717, - "remote_mean_r2": 0.9971200615281508, - "r2_delta": -4.0952487209144905e-06, - "local_mean_nrmse_std": 0.04895350495151681, - "remote_mean_nrmse_std": 0.048983762073770726, - "nrmse_delta": 3.0257122253914137e-05 + "local_mean_r2": 0.9971158433783592, + "remote_mean_r2": 0.9971139833803742, + "r2_delta": -1.8599979850764115e-06, + "local_mean_nrmse_std": 0.04900140374890117, + "remote_mean_nrmse_std": 0.04902444807450163, + "nrmse_delta": 2.30443256004631e-05 } } } } }, - "measurement_semantics": { - "latency": "Qualcomm AI Hub profile estimated model inference time", - "throughput": "Derived from configured batch size and AI Hub estimated inference time", - "runtime_memory": "Qualcomm AI Hub estimated inference peak memory", - "serialized_model_size": "Downloaded QNN Context Binary byte size", - "allclose": "Diagnostic only; predictive quality and normalized drift are reported separately" - }, - "claim_boundaries": [ - "Results apply to the named Snapdragon 8 Elite QRD configuration.", - "AI Hub profile latency is not end-to-end Android application latency.", - "Derived throughput is based on model-profile latency and is not an end-to-end system throughput measurement.", - "CPU, CoreML, and Snapdragon timings are not treated as same-hardware comparisons.", - "The QNN baseline uses float32 external I/O with HTP FP16-relaxed execution." - ] + "claim_boundary": "Physical Qualcomm AI Hub QNN HTP model measurements; not Android APK end-to-end latency and not a calibrated power measurement." } diff --git a/scripts/build_portfolio_acceptance.py b/scripts/build_portfolio_acceptance.py new file mode 100644 index 0000000..223b255 --- /dev/null +++ b/scripts/build_portfolio_acceptance.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Validate tracked deployment evidence and build the final portfolio acceptance matrix.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path +from typing import Any + + +def _load_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _repository_sha256(repository_root: Path, source_path: Path) -> tuple[str | None, str]: + """Hash the committed artifact, falling back to the worktree outside Git.""" + try: + relative_path = source_path.relative_to(repository_root).as_posix() + except ValueError as exc: + raise ValueError("source model must be inside the repository") from exc + + result = subprocess.run( + ["git", "-C", str(repository_root), "show", f"HEAD:{relative_path}"], + check=False, + capture_output=True, + ) + if result.returncode == 0: + return hashlib.sha256(result.stdout).hexdigest(), "committed_git_blob" + if source_path.is_file(): + return _sha256(source_path), "worktree_file" + return None, "missing" + + +def validate_ai_hub_qnn(report_path: Path, repository_root: Path) -> dict[str, Any]: + report = _load_json(report_path) + if report.get("schema_version") != "0.2": + raise ValueError("unsupported Qualcomm AI Hub report") + source = report.get("source_model") + hardware = report.get("hardware") + linked = report.get("linked_multigraph") + if ( + not isinstance(source, dict) + or not isinstance(hardware, dict) + or not isinstance(linked, dict) + ): + raise ValueError("incomplete Qualcomm AI Hub identity") + source_path = repository_root / str(source.get("path")) + current_source_sha256, source_hash_origin = _repository_sha256(repository_root, source_path) + source_model_matches = current_source_sha256 == source.get("sha256") + if hardware.get("backend") != "HTP" or not hardware.get("qairt_version"): + raise ValueError("Qualcomm report must identify the HTP backend and QAIRT version") + if "SUCCESS" not in str(linked.get("link_status")): + raise ValueError("linked QNN context job did not succeed") + validation = linked.get("validation") + if not isinstance(validation, dict) or validation.get("device") != hardware.get("device"): + raise ValueError("linked QNN validation device does not match the report") + graphs = validation.get("graphs") + if not isinstance(graphs, dict) or set(graphs) != { + "edgegenbench_batch1", + "edgegenbench_batch32", + "edgegenbench_batch256", + }: + raise ValueError("linked QNN evidence must contain batch 1, 32, and 256 graphs") + + graph_summaries = [] + for name, graph in graphs.items(): + if not isinstance(graph, dict): + raise ValueError(f"invalid QNN graph evidence: {name}") + profile = graph.get("profile") + parity = graph.get("parity") + if not isinstance(profile, dict) or profile.get("compute_units") != {"NPU": 9}: + raise ValueError(f"{name} does not prove exclusive NPU compute-unit mapping") + if not isinstance(parity, dict) or float(parity.get("max_normalized_drift", 1.0)) > 0.012: + raise ValueError(f"{name} exceeds the predeclared normalized drift limit") + if not graph.get("profile_job_id") or not graph.get("inference_job_id"): + raise ValueError(f"{name} requires profile and inference job IDs") + graph_summaries.append( + { + "graph": name, + "batch_size": graph["batch_size"], + "latency_ms": profile["estimated_inference_latency_ms"], + "throughput_samples_per_second": profile["estimated_throughput_samples_per_second"], + "peak_memory_bytes": profile["estimated_inference_peak_memory_bytes"], + "compute_units": profile["compute_units"], + "max_normalized_drift": parity["max_normalized_drift"], + "profile_job_id": graph["profile_job_id"], + "inference_job_id": graph["inference_job_id"], + } + ) + return { + "status": ( + "validated_ai_hub_physical_qnn" + if source_model_matches + else "tracked_ai_hub_report_model_provenance_mismatch" + ), + "device": hardware["device"], + "backend": "QNN HTP", + "qairt_version": hardware["qairt_version"], + "context_model_id": linked["target_model_id"], + "link_job_id": linked["link_job_id"], + "graphs": sorted(graph_summaries, key=lambda value: value["batch_size"]), + "reported_source_model_sha256": source.get("sha256"), + "current_source_model_sha256": current_source_sha256, + "source_model_hash_origin": source_hash_origin, + "source_model_matches_repository": source_model_matches, + "claim_boundary": "Physical AI Hub model profiling; not Android APK end-to-end latency.", + } + + +def build_portfolio_acceptance( + *, repository_root: Path, qnn_report: Path, output_json: Path, output_markdown: Path +) -> dict[str, Any]: + qnn = validate_ai_hub_qnn(qnn_report, repository_root) + android_report = repository_root / "reports/android_sm_a356e_reference_10_run_v0_1_4.md" + if not android_report.is_file(): + raise ValueError("tracked Samsung reference report is missing") + matrix = { + "schema_version": 1, + "project": "EdgeGenBench", + "lanes": { + "native_cpp": { + "status": "validated_in_ci", + "claim": "C++17 reference runtime, tests, CLI, and fused preprocessing acceptance.", + }, + "android_reference": { + "status": "validated_physical_device", + "device": "Samsung SM-A356E", + "report": android_report.relative_to(repository_root).as_posix(), + "claim": "Reference JNI/application measurements; not QNN.", + }, + "qualcomm_ai_hub_qnn": qnn, + "android_qnn_apk": { + "status": "implementation_complete_evidence_pending", + "claim": "Build/JNI/capture paths exist; requires a supported Snapdragon APK run.", + }, + "android_16kb_runtime": { + "status": "packaging_validated_runtime_pending", + "claim": "ELF/APK alignment passes; runtime PAGE_SIZE=16384 evidence is pending.", + }, + "power": { + "status": "not_measured", + "claim": "No power-savings claim is made without a named calibrated tool.", + }, + }, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(matrix, indent=2) + "\n", encoding="utf-8") + lines = [ + "# EdgeGenBench portfolio acceptance", + "", + "| Evidence lane | Status | Claim boundary |", + "|---|---|---|", + ] + for name, lane in matrix["lanes"].items(): + claim = lane.get("claim", lane.get("claim_boundary")) + lines.append(f"| `{name}` | `{lane['status']}` | {claim} |") + lines.extend( + [ + "", + "## Validated Qualcomm QNN results", + "", + f"Device: **{qnn['device']}**; backend: **{qnn['backend']}**; " + f"QAIRT: `{qnn['qairt_version']}`.", + f"Source-model provenance match: **{qnn['source_model_matches_repository']}**.", + "", + "| Batch | AI Hub latency (ms) | Throughput (samples/s) | " + "Peak memory (bytes) | Placement | Max normalized drift |", + "|---:|---:|---:|---:|---|---:|", + ] + ) + for graph in qnn["graphs"]: + lines.append( + f"| {graph['batch_size']} | {graph['latency_ms']:.6f} | " + f"{graph['throughput_samples_per_second']:.3f} | {graph['peak_memory_bytes']} | " + f"NPU × 9 | {graph['max_normalized_drift']:.9f} |" + ) + lines.extend( + [ + "", + "AI Hub measurements are physical-device model profiles, not Android " + "application end-to-end timings. Current-model acceptance requires source-model " + "provenance to match the repository, as reported above.", + "Power remains unmeasured. The two pending proof items are a supported-device " + "QNN APK run and a runtime page size of 16384 bytes.", + ] + ) + output_markdown.write_text("\n".join(lines) + "\n", encoding="utf-8") + return matrix + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository-root", type=Path, default=Path.cwd()) + parser.add_argument("--qnn-report", type=Path, default=Path("reports/qualcomm_qnn_v0_1.json")) + parser.add_argument( + "--output-json", type=Path, default=Path("reports/portfolio_acceptance.json") + ) + parser.add_argument( + "--output-markdown", type=Path, default=Path("reports/portfolio_acceptance.md") + ) + args = parser.parse_args() + build_portfolio_acceptance( + repository_root=args.repository_root.resolve(), + qnn_report=args.qnn_report, + output_json=args.output_json, + output_markdown=args.output_markdown, + ) + print(f"Portfolio acceptance written to {args.output_markdown}") + + +if __name__ == "__main__": + main() diff --git a/scripts/rerun_qualcomm_qnn_current_model.py b/scripts/rerun_qualcomm_qnn_current_model.py new file mode 100644 index 0000000..72947d0 --- /dev/null +++ b/scripts/rerun_qualcomm_qnn_current_model.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Compile, link, profile, and validate the current ONNX model on Qualcomm AI Hub.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +import numpy as np +import onnxruntime as ort +import pandas as pd +import qai_hub as hub + +from edgegenbench.deployment.qualcomm_ai_hub import ( + calculate_runtime_parity, + qnn_graph_option, + stringify_metadata, + summarize_profile, +) + +NUMERIC_FEATURES = [ + "passenger_capacity", + "design_range_km", + "cruise_speed_kmh", + "battery_specific_energy_wh_per_kg", + "hydrogen_storage_efficiency", + "hybridization_ratio", +] +CATEGORICAL_FEATURE = "propulsion_architecture" + +MODEL_PATH = Path("artifacts/neural_onnx/neural_surrogate.onnx") +PREPROCESSOR_PATH = Path("artifacts/neural_surrogate/preprocessing.npz") +HELDOUT_PATH = Path("artifacts/neural_surrogate/test_predictions.csv") +OUTPUT_PATH = Path("reports/qualcomm_qnn_v0_1.json") +CONTEXT_PATH = Path("artifacts/qualcomm_ai_hub/current_model/edgegenbench_multigraph.bin") +CHECKPOINT_PATH = Path("artifacts/qualcomm_ai_hub/current_model/rerun_checkpoint.json") + +DEVICE_NAME = "Snapdragon 8 Elite QRD" +DEVICE_OS = "15" +GRAPH_BATCHES = { + "edgegenbench_batch1": 1, + "edgegenbench_batch32": 32, + "edgegenbench_batch256": 256, +} +COMPILE_OPTIONS = "--qnn_options default_graph_htp_precision=FLOAT16" +LINK_OPTIONS = "--qnn_options default_graph_htp_optimizations=O=3" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def _require_success(status: object, label: str) -> None: + if not bool(getattr(status, "success", False)): + raise RuntimeError(f"{label} failed: {status}") + + +def _partition(features: np.ndarray, batch_size: int) -> list[np.ndarray]: + if len(features) % batch_size: + raise ValueError(f"Feature count {len(features)} is not divisible by {batch_size}.") + return [ + features[start : start + batch_size].astype(np.float32, copy=False) + for start in range(0, len(features), batch_size) + ] + + +def _download_output(inference_job: Any) -> tuple[str, np.ndarray]: + output = inference_job.download_output_data() + if not isinstance(output, dict) or len(output) != 1: + raise RuntimeError(f"Expected one AI Hub output tensor, received {type(output)}.") + name = next(iter(output)) + arrays = [] + for value in output[name]: + array = np.asarray(value, dtype=np.float32) + arrays.append(array.reshape(-1, array.shape[-1])) + return name, np.concatenate(arrays, axis=0) + + +def _load_preprocessing() -> dict[str, Any]: + with np.load(PREPROCESSOR_PATH, allow_pickle=False) as data: + return {key: data[key].copy() for key in data.files} + + +def _transform_features(frame: pd.DataFrame, preprocessing: dict[str, Any]) -> np.ndarray: + numeric = frame.loc[:, NUMERIC_FEATURES].to_numpy(dtype=np.float32) + standardized = (numeric - preprocessing["feature_mean"]) / preprocessing["feature_scale"] + categories = [str(value) for value in preprocessing["categories"].tolist()] + category_to_index = {category: index for index, category in enumerate(categories)} + indices = np.asarray( + [category_to_index[str(value)] for value in frame[CATEGORICAL_FEATURE]], dtype=np.int64 + ) + one_hot = np.eye(len(categories), dtype=np.float32)[indices] + return np.concatenate([standardized, one_hot], axis=1).astype(np.float32) + + +def _quality( + actual: np.ndarray, prediction: np.ndarray, preprocessing: dict[str, Any] +) -> dict[str, float]: + physical = (prediction * preprocessing["target_scale"] + preprocessing["target_mean"]).astype( + np.float64 + ) + actual64 = actual.astype(np.float64) + residual = actual64 - physical + squared_error = np.sum(residual * residual, axis=0) + centered = actual64 - np.mean(actual64, axis=0) + total_squared = np.sum(centered * centered, axis=0) + r2 = 1.0 - squared_error / total_squared + rmse = np.sqrt(np.mean(residual * residual, axis=0)) + actual_std = np.std(actual64, axis=0) + nrmse_std = rmse / actual_std + return { + "mean_r2": float(np.nanmean(r2)), + "mean_nrmse_std": float(np.nanmean(nrmse_std)), + } + + +def _checkpoint(client: hub.Client, model_sha256: str) -> dict[str, Any]: + if not CHECKPOINT_PATH.is_file(): + return {"source_model_sha256": model_sha256, "graphs": {}} + value = json.loads(CHECKPOINT_PATH.read_text(encoding="utf-8")) + if value.get("source_model_sha256") != model_sha256: + raise RuntimeError("Checkpoint belongs to a different source model; remove it to rerun.") + # Validate that the saved linked model remains accessible before resuming. + if value.get("target_model_id"): + client.get_model(str(value["target_model_id"])) + return value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--resume", action="store_true", help="Reuse successful saved job IDs.") + args = parser.parse_args() + + for path in (MODEL_PATH, PREPROCESSOR_PATH, HELDOUT_PATH): + if not path.is_file(): + raise FileNotFoundError(f"Required artifact is missing: {path}") + + model_sha256 = _sha256(MODEL_PATH) + client = hub.Client() + device = hub.Device(DEVICE_NAME, DEVICE_OS) + checkpoint = ( + _checkpoint(client, model_sha256) + if args.resume + else { + "source_model_sha256": model_sha256, + "graphs": {}, + } + ) + + if checkpoint.get("target_model_id"): + target_model = client.get_model(str(checkpoint["target_model_id"])) + compile_jobs = [client.get_job(job_id) for job_id in checkpoint["compile_job_ids"]] + link_job = client.get_job(str(checkpoint["link_job_id"])) + print(f"Resuming linked model {target_model.model_id}", flush=True) + else: + graph_names = list(GRAPH_BATCHES) + print("Submitting three QNN DLC compiles and the dependent link job...", flush=True) + compile_jobs, link_job = client.submit_compile_and_link_jobs( + models=[MODEL_PATH] * len(graph_names), + device=device, + name="EdgeGenBench current-model QNN multigraph", + input_specs=[ + {"features": ((batch_size, 10), "float32")} for batch_size in GRAPH_BATCHES.values() + ], + graph_names=graph_names, + compile_options=COMPILE_OPTIONS, + link_options=LINK_OPTIONS, + ) + if link_job is None: + raise RuntimeError("AI Hub did not create the QNN link job.") + checkpoint.update( + compile_job_ids=[job.job_id for job in compile_jobs], + link_job_id=link_job.job_id, + ) + _write_json(CHECKPOINT_PATH, checkpoint) + print(f"Compile jobs: {checkpoint['compile_job_ids']}", flush=True) + print(f"Link job: {link_job.job_id}", flush=True) + status = link_job.wait() + _require_success(status, "QNN context link") + target_model = link_job.get_target_model() + if target_model is None: + raise RuntimeError("Successful link job did not return a target model.") + checkpoint["target_model_id"] = target_model.model_id + checkpoint["link_status"] = str(status) + _write_json(CHECKPOINT_PATH, checkpoint) + + CONTEXT_PATH.parent.mkdir(parents=True, exist_ok=True) + target_model.download(str(CONTEXT_PATH)) + + frame = pd.read_csv(HELDOUT_PATH) + if len(frame) != 900: + raise RuntimeError(f"Expected 900 held-out rows, received {len(frame)}.") + preprocessing = _load_preprocessing() + targets = [str(value) for value in preprocessing["targets"].tolist()] + features = _transform_features(frame, preprocessing) + session = ort.InferenceSession(str(MODEL_PATH), providers=["CPUExecutionProvider"]) + input_name = session.get_inputs()[0].name + output_name = session.get_outputs()[0].name + local = np.asarray(session.run([output_name], {input_name: features})[0], dtype=np.float32) + + graph_records: dict[str, Any] = {} + for graph_name, batch_size in GRAPH_BATCHES.items(): + saved = checkpoint["graphs"].get(graph_name, {}) if args.resume else {} + if saved.get("profile_job_id"): + profile_job = client.get_job(str(saved["profile_job_id"])) + else: + print(f"Submitting profile for {graph_name}...", flush=True) + profile_job = client.submit_profile_job( + model=target_model, + device=device, + name=f"EdgeGenBench current-model {graph_name} profile", + options=qnn_graph_option(graph_name), + ) + if isinstance(profile_job, list): + raise RuntimeError("Expected one profile job.") + saved["profile_job_id"] = profile_job.job_id + checkpoint["graphs"][graph_name] = saved + _write_json(CHECKPOINT_PATH, checkpoint) + profile_status = profile_job.wait() + _require_success(profile_status, f"Profile {graph_name}") + profile = summarize_profile(profile_job.download_profile(), batch_size=batch_size) + + sample_count = 900 if batch_size == 1 else 256 + if saved.get("inference_job_id"): + inference_job = client.get_job(str(saved["inference_job_id"])) + else: + print(f"Submitting inference for {graph_name}...", flush=True) + inference_job = client.submit_inference_job( + model=target_model, + device=device, + inputs={input_name: _partition(features[:sample_count], batch_size)}, + name=f"EdgeGenBench current-model {graph_name} inference", + options=qnn_graph_option(graph_name), + ) + if isinstance(inference_job, list): + raise RuntimeError("Expected one inference job.") + saved["inference_job_id"] = inference_job.job_id + _write_json(CHECKPOINT_PATH, checkpoint) + inference_status = inference_job.wait() + _require_success(inference_status, f"Inference {graph_name}") + remote_output_name, remote = _download_output(inference_job) + reference = local[:sample_count] + parity = calculate_runtime_parity(reference, remote) + actual = frame.loc[: sample_count - 1, targets].to_numpy(dtype=np.float32) + local_quality = _quality(actual, reference, preprocessing) + remote_quality = _quality(actual, remote, preprocessing) + graph_records[graph_name] = { + "graph_name": graph_name, + "batch_size": batch_size, + "sample_count": sample_count, + "profile_job_id": profile_job.job_id, + "inference_job_id": inference_job.job_id, + "profile": profile.to_dict(), + "remote_output_name": remote_output_name, + "parity": parity.to_dict(), + "predictive_quality": { + "local_mean_r2": local_quality["mean_r2"], + "remote_mean_r2": remote_quality["mean_r2"], + "r2_delta": remote_quality["mean_r2"] - local_quality["mean_r2"], + "local_mean_nrmse_std": local_quality["mean_nrmse_std"], + "remote_mean_nrmse_std": remote_quality["mean_nrmse_std"], + "nrmse_delta": (remote_quality["mean_nrmse_std"] - local_quality["mean_nrmse_std"]), + }, + } + print(f"Completed {graph_name}: {profile.to_dict()}", flush=True) + + metadata = stringify_metadata(target_model.metadata) + qairt_version = next( + (value for key, value in metadata.items() if "QAIRT_SDK_VERSION" in key), "unknown" + ) + backend = next((value for key, value in metadata.items() if key.endswith("BACKEND")), "HTP") + hexagon = next( + (value for key, value in metadata.items() if "HEXAGON_VERSION" in key), "unknown" + ) + report = { + "schema_version": "0.2", + "experiment": "EdgeGenBench current-model Qualcomm QNN deployment", + "source_model": { + "path": str(MODEL_PATH), + "sha256": model_sha256, + "input_name": input_name, + "input_width": 10, + "output_name": output_name, + "output_width": 6, + "source_precision": "float32", + }, + "hardware": { + "device": DEVICE_NAME, + "device_os": DEVICE_OS, + "chipset": "qualcomm-snapdragon-8-elite", + "chipset_alias": "sm8750", + "backend": backend, + "hexagon": hexagon, + "qairt_version": qairt_version, + }, + "linked_multigraph": { + "compile_jobs": [ + { + "batch": batch, + "graph_name": graph, + "compile_job_id": job.job_id, + "compile_status": str(job.get_status()), + } + for (graph, batch), job in zip(GRAPH_BATCHES.items(), compile_jobs, strict=True) + ], + "link_job_id": link_job.job_id, + "link_status": str(link_job.get_status()), + "target_model_id": target_model.model_id, + "target_model_type": str(target_model.model_type), + "serialized_model_size_bytes": CONTEXT_PATH.stat().st_size, + "serialized_model_sha256": _sha256(CONTEXT_PATH), + "target_metadata": metadata, + "validation": { + "link_job_id": link_job.job_id, + "target_model_id": target_model.model_id, + "target_model_type": str(target_model.model_type), + "target_metadata": metadata, + "target_input_spec": str(target_model.input_spec), + "device": DEVICE_NAME, + "device_os": DEVICE_OS, + "local_reference": { + "model_path": str(MODEL_PATH), + "input_name": input_name, + "output_name": output_name, + "heldout_rows": len(frame), + }, + "graphs": graph_records, + }, + }, + "claim_boundary": ( + "Physical Qualcomm AI Hub QNN HTP model measurements; not Android APK " + "end-to-end latency and not a calibrated power measurement." + ), + } + _write_json(OUTPUT_PATH, report) + print(f"Wrote {OUTPUT_PATH}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/test_portfolio_acceptance.py b/tests/test_portfolio_acceptance.py new file mode 100644 index 0000000..d0e73a6 --- /dev/null +++ b/tests/test_portfolio_acceptance.py @@ -0,0 +1,34 @@ +import json +import runpy +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import pytest + +SCRIPT = Path(__file__).parents[1] / "scripts/build_portfolio_acceptance.py" +FUNCTIONS = runpy.run_path(SCRIPT) +ValidateQnn = Callable[[Path, Path], dict[str, object]] +validate_ai_hub_qnn = cast(ValidateQnn, FUNCTIONS["validate_ai_hub_qnn"]) + + +def test_validates_tracked_ai_hub_qnn_evidence() -> None: + root = Path(__file__).parents[1] + result = validate_ai_hub_qnn(root / "reports/qualcomm_qnn_v0_1.json", root) + assert result["status"] == "validated_ai_hub_physical_qnn" + assert result["backend"] == "QNN HTP" + assert result["source_model_matches_repository"] is True + assert result["source_model_hash_origin"] == "committed_git_blob" + assert len(cast(list[object], result["graphs"])) == 3 + + +def test_rejects_cpu_compute_unit_in_qnn_report(tmp_path: Path) -> None: + root = Path(__file__).parents[1] + report = json.loads((root / "reports/qualcomm_qnn_v0_1.json").read_text()) + report["linked_multigraph"]["validation"]["graphs"]["edgegenbench_batch1"]["profile"][ + "compute_units" + ] = {"CPU": 1, "NPU": 8} + path = tmp_path / "qnn.json" + path.write_text(json.dumps(report)) + with pytest.raises(ValueError, match="exclusive NPU"): + validate_ai_hub_qnn(path, root)