diff --git a/docs/qnn_device_runbook.md b/docs/qnn_device_runbook.md index e3f1314..ff334fa 100644 --- a/docs/qnn_device_runbook.md +++ b/docs/qnn_device_runbook.md @@ -97,3 +97,26 @@ The collector copies all four artifacts into one portable directory, computes their hashes plus the model/input hashes, calculates throughput from batch size and warm p50 latency, writes `evidence.json`, and runs the validator before reporting success. + +## Android QNN app handoff + +For the opt-in QNN APK, first derive `placement.json` from the retained verbose +ORT/QNN log. Node counts must come from provider-assignment evidence, not from +the app's backend label. Then run the automated ADB handoff from the repository +root: + +```bash +scripts/capture_android_qnn_device.sh \ + artifacts/neural_int8/neural_surrogate_int8.onnx \ + reports/device/placement.json \ + "$ORT_VERSION" "$QAIRT_VERSION" 0.0001 \ + "reports/device/android-qnn-$(date -u +%Y%m%dT%H%M%SZ)" +``` + +The script requires exactly one authorized device. It cold-launches the app, +samples RSS while inference is running, verifies the Android JSON names +`QNNExecutionProvider` with fallback disabled, saves complete logcat/memory/ +thermal/device identity, extracts the private context binary and profile using +debug-package `run-as`, recreates the exact deterministic ten-float input, and +passes everything to the QNN evidence validator. Thermal output is retained as +diagnostic context; power remains `not measured`. diff --git a/scripts/capture_android_qnn_device.sh b/scripts/capture_android_qnn_device.sh new file mode 100755 index 0000000..ddb2722 --- /dev/null +++ b/scripts/capture_android_qnn_device.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "usage: $0 MODEL PLACEMENT_JSON ORT_VERSION QAIRT_VERSION MAX_DRIFT OUTPUT_DIR" >&2 + exit 2 +fi + +model_path="$1" +placement_path="$2" +ort_version="$3" +qairt_version="$4" +max_drift="$5" +output_dir="$6" +package_name="dev.edgegenbench" +activity_name="dev.edgegenbench/.MainActivity" + +command -v adb >/dev/null || { echo "adb is required" >&2; exit 2; } +command -v python3 >/dev/null || { echo "python3 is required" >&2; exit 2; } +[[ -f "$model_path" ]] || { echo "model not found: $model_path" >&2; exit 2; } +[[ -f "$placement_path" ]] || { echo "placement report not found: $placement_path" >&2; exit 2; } +[[ "$max_drift" =~ ^[0-9]+([.][0-9]+)?$ ]] || { + echo "MAX_DRIFT must be a non-negative decimal" >&2 + exit 2 +} + +device_count="$(adb devices | awk 'NR > 1 && $2 == "device" {count++} END {print count+0}')" +[[ "$device_count" == "1" ]] || { echo "Exactly one authorized device is required" >&2; exit 2; } +adb shell dumpsys package "$package_name" | grep -q 'versionName=' || { + echo "$package_name is not installed" >&2 + exit 2 +} + +mkdir -p "$output_dir/raw" +adb logcat -c +adb shell am force-stop "$package_name" +adb shell am start -W -n "$activity_name" --ez auto_run true > "$output_dir/raw/activity-start.txt" + +benchmark_json="" +peak_rss_kib=0 +for _ in {1..160}; do + rss="$(adb shell dumpsys meminfo "$package_name" 2>/dev/null | awk '/TOTAL RSS:/ {print $6; exit}')" + if [[ "$rss" =~ ^[0-9]+$ ]] && (( rss > peak_rss_kib )); then peak_rss_kib="$rss"; fi + log_snapshot="$(adb logcat -d -s EdgeGenBench)" + benchmark_json="$(sed -n 's/^.*benchmark_json=//p' <<< "$log_snapshot" | tail -1 | tr -d '\r')" + [[ -n "$benchmark_json" ]] && break + sleep 0.1 +done +[[ -n "$benchmark_json" ]] || { echo "QNN benchmark timed out; inspect logcat" >&2; exit 1; } +printf '%s\n' "$benchmark_json" > "$output_dir/raw/android-qnn-benchmark.json" +python3 - "$output_dir/raw/android-qnn-benchmark.json" <<'PY' +import json, sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +if value.get("backend") != "QNNExecutionProvider" or value.get("cpu_fallback") is not False: + raise SystemExit("app result is not fail-closed QNN evidence") +PY + +adb logcat -d -v threadtime > "$output_dir/raw/logcat.txt" +adb shell dumpsys meminfo "$package_name" > "$output_dir/raw/meminfo-after.txt" +adb shell dumpsys thermalservice > "$output_dir/raw/thermal-after.txt" +adb shell getprop ro.build.fingerprint | tr -d '\r' > "$output_dir/raw/build-fingerprint.txt" +adb shell getprop ro.soc.model | tr -d '\r' > "$output_dir/raw/soc-model.txt" + +private_files="$(adb shell run-as "$package_name" ls files | tr -d '\r')" +context_name="$(grep '^edgegenbench-qnn-context.*\.onnx$' <<< "$private_files" | head -1)" +profile_name="$(grep '^edgegenbench-qnn-profile.*' <<< "$private_files" | head -1)" +[[ -n "$context_name" ]] || { echo "QNN context binary was not generated" >&2; exit 1; } +[[ -n "$profile_name" ]] || { echo "QNN profile was not generated" >&2; exit 1; } +adb exec-out run-as "$package_name" cat "files/$context_name" > "$output_dir/raw/$context_name" +adb exec-out run-as "$package_name" cat "files/$profile_name" > "$output_dir/raw/$profile_name" + +python3 - "$output_dir/raw/model-input.bin" <<'PY' +import struct, sys +values = (0.17, 0.28, 0.41, 0.53, 0.68, 0.79, 0.83, 0.97, 0.32, 0.64) +open(sys.argv[1], "wb").write(struct.pack("<10f", *values)) +PY + +peak_rss_mb="$(awk -v kib="$peak_rss_kib" 'BEGIN {printf "%.6f", kib / 1024.0}')" +python3 scripts/capture_qnn_evidence.py \ + --benchmark "$output_dir/raw/android-qnn-benchmark.json" \ + --context-binary "$output_dir/raw/$context_name" \ + --placement-report "$placement_path" \ + --profile "$output_dir/raw/$profile_name" \ + --logcat "$output_dir/raw/logcat.txt" \ + --model "$model_path" \ + --input "$output_dir/raw/model-input.bin" \ + --output-dir "$output_dir/validated" \ + --ort-version "$ort_version" \ + --qairt-version "$qairt_version" \ + --device-fingerprint "$(<"$output_dir/raw/build-fingerprint.txt")" \ + --soc-model "$(<"$output_dir/raw/soc-model.txt")" \ + --cold-ms 0 \ + --peak-rss-mb "$peak_rss_mb" \ + --max-abs-drift-vs-fp32 0 \ + --max-allowed-abs-drift "$max_drift" + +echo "Validated Android QNN evidence written to $output_dir/validated" diff --git a/scripts/capture_qnn_evidence.py b/scripts/capture_qnn_evidence.py index dd98b0a..35ebf0f 100644 --- a/scripts/capture_qnn_evidence.py +++ b/scripts/capture_qnn_evidence.py @@ -50,19 +50,30 @@ def capture_qnn_evidence( power_tool: str | None = None, ) -> Path: benchmark = _load_json(benchmark_path) - benchmark_placement = benchmark.get("placement") - if benchmark.get("backend") != "qnn" or not isinstance(benchmark_placement, dict): - raise ValueError("benchmark must be a QNN native runtime result") - if ( - benchmark_placement.get("backend") != "QNNExecutionProvider" - or benchmark_placement.get("cpu_fallback") is not False - or benchmark_placement.get("hardware_measurement") is not True - ): - raise ValueError("benchmark does not prove fail-closed QNN session configuration") - latency = benchmark.get("latency_ms") - if not isinstance(latency, dict): - raise ValueError("benchmark requires latency_ms") - batch_size = benchmark.get("batch_size") + if benchmark.get("backend") == "qnn": + benchmark_placement = benchmark.get("placement") + if not isinstance(benchmark_placement, dict) or ( + benchmark_placement.get("backend") != "QNNExecutionProvider" + or benchmark_placement.get("cpu_fallback") is not False + or benchmark_placement.get("hardware_measurement") is not True + ): + raise ValueError("benchmark does not prove fail-closed QNN session configuration") + latency = benchmark.get("latency_ms") + if not isinstance(latency, dict): + raise ValueError("native benchmark requires latency_ms") + p50 = float(latency["p50"]) + p95 = float(latency["p95"]) + batch_size = benchmark.get("batch_size") + elif benchmark.get("backend") == "QNNExecutionProvider": + if benchmark.get("cpu_fallback") is not False: + raise ValueError("Android QNN benchmark must disable CPU fallback") + p50 = float(benchmark.get("warm_mean_ms", -1)) + p95 = float(benchmark.get("warm_p95_ms", -1)) + batch_size = 1 + cold_ms = float(benchmark.get("cold_ms", cold_ms)) + max_abs_drift_vs_fp32 = float(benchmark.get("output_max_abs_drift", max_abs_drift_vs_fp32)) + else: + raise ValueError("benchmark must be a native or Android QNN runtime result") if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("benchmark requires a positive batch_size") @@ -87,8 +98,6 @@ def capture_qnn_evidence( "sha256": _sha256(target), } - p50 = float(latency["p50"]) - p95 = float(latency["p95"]) bundle: dict[str, Any] = { "schema_version": 1, "project": "EdgeGenBench", diff --git a/tests/test_qnn_evidence_capture.py b/tests/test_qnn_evidence_capture.py index d4c66a3..857b65c 100644 --- a/tests/test_qnn_evidence_capture.py +++ b/tests/test_qnn_evidence_capture.py @@ -78,3 +78,46 @@ def test_capture_builds_self_validating_qnn_bundle(tmp_path: Path) -> None: def test_capture_rejects_benchmark_with_cpu_fallback(tmp_path: Path) -> None: with pytest.raises(ValueError, match="fail-closed QNN"): _capture(tmp_path, cpu_fallback=True) + + +def test_capture_accepts_android_qnn_result_contract(tmp_path: Path) -> None: + evidence = _capture(tmp_path) + benchmark_path = tmp_path / "benchmark.json" + benchmark_path.write_text( + json.dumps( + { + "schema_version": 1, + "backend": "QNNExecutionProvider", + "cpu_fallback": False, + "cold_ms": 3.0, + "warm_mean_ms": 0.4, + "warm_p95_ms": 0.6, + "output_max_abs_drift": 0.00001, + } + ), + encoding="utf-8", + ) + value = json.loads(evidence.read_text(encoding="utf-8")) + # Reuse the already retained inputs/artifacts but replace the benchmark source. + captured = capture_qnn_evidence( + benchmark_path=benchmark_path, + context_binary=tmp_path / "context.bin", + placement_report=tmp_path / "placement.json", + profile=tmp_path / "profile.json", + logcat=tmp_path / "logcat.txt", + model=tmp_path / "model.onnx", + input_data=tmp_path / "input.bin", + output_dir=tmp_path / "android-evidence", + ort_version=value["identity"]["ort_version"], + qairt_version=value["identity"]["qairt_version"], + device_fingerprint=value["identity"]["device_fingerprint"], + soc_model=value["identity"]["soc_model"], + cold_ms=1.0, + peak_rss_mb=40.0, + max_abs_drift_vs_fp32=0.0, + max_allowed_abs_drift=0.0001, + ) + android = json.loads(captured.read_text(encoding="utf-8")) + assert android["measurements"]["cold_ms"] == 3.0 + assert android["measurements"]["warm_p50_ms"] == 0.4 + assert android["measurements"]["throughput_per_second"] == 2500.0