Capture Android QNN runtime artifacts

docs/qnn_device_runbook.md

@@ -97,3 +97,26 @@ The collector copies all four artifacts into one portable directory, computes
 their hashes plus the model/input hashes, calculates throughput from batch size
 and warm p50 latency, writes `evidence.json`, and runs the validator before
 reporting success.
+
+## Android QNN app handoff
+
+For the opt-in QNN APK, first derive `placement.json` from the retained verbose
+ORT/QNN log. Node counts must come from provider-assignment evidence, not from
+the app's backend label. Then run the automated ADB handoff from the repository
+root:
+
+```bash
+scripts/capture_android_qnn_device.sh \
+  artifacts/neural_int8/neural_surrogate_int8.onnx \
+  reports/device/placement.json \
+  "$ORT_VERSION" "$QAIRT_VERSION" 0.0001 \
+  "reports/device/android-qnn-$(date -u +%Y%m%dT%H%M%SZ)"
+```
+
+The script requires exactly one authorized device. It cold-launches the app,
+samples RSS while inference is running, verifies the Android JSON names
+`QNNExecutionProvider` with fallback disabled, saves complete logcat/memory/
+thermal/device identity, extracts the private context binary and profile using
+debug-package `run-as`, recreates the exact deterministic ten-float input, and
+passes everything to the QNN evidence validator. Thermal output is retained as
+diagnostic context; power remains `not measured`.

scripts/capture_android_qnn_device.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 6 ]]; then
+  echo "usage: $0 MODEL PLACEMENT_JSON ORT_VERSION QAIRT_VERSION MAX_DRIFT OUTPUT_DIR" >&2
+  exit 2
+fi
+
+model_path="$1"
+placement_path="$2"
+ort_version="$3"
+qairt_version="$4"
+max_drift="$5"
+output_dir="$6"
+package_name="dev.edgegenbench"
+activity_name="dev.edgegenbench/.MainActivity"
+
+command -v adb >/dev/null || { echo "adb is required" >&2; exit 2; }
+command -v python3 >/dev/null || { echo "python3 is required" >&2; exit 2; }
+[[ -f "$model_path" ]] || { echo "model not found: $model_path" >&2; exit 2; }
+[[ -f "$placement_path" ]] || { echo "placement report not found: $placement_path" >&2; exit 2; }
+[[ "$max_drift" =~ ^[0-9]+([.][0-9]+)?$ ]] || {
+  echo "MAX_DRIFT must be a non-negative decimal" >&2
+  exit 2
+}
+
+device_count="$(adb devices | awk 'NR > 1 && $2 == "device" {count++} END {print count+0}')"
+[[ "$device_count" == "1" ]] || { echo "Exactly one authorized device is required" >&2; exit 2; }
+adb shell dumpsys package "$package_name" | grep -q 'versionName=' || {
+  echo "$package_name is not installed" >&2
+  exit 2
+}
+
+mkdir -p "$output_dir/raw"
+adb logcat -c
+adb shell am force-stop "$package_name"
+adb shell am start -W -n "$activity_name" --ez auto_run true > "$output_dir/raw/activity-start.txt"
+
+benchmark_json=""
+peak_rss_kib=0
+for _ in {1..160}; do
+  rss="$(adb shell dumpsys meminfo "$package_name" 2>/dev/null | awk '/TOTAL RSS:/ {print $6; exit}')"
+  if [[ "$rss" =~ ^[0-9]+$ ]] && (( rss > peak_rss_kib )); then peak_rss_kib="$rss"; fi
+  log_snapshot="$(adb logcat -d -s EdgeGenBench)"
+  benchmark_json="$(sed -n 's/^.*benchmark_json=//p' <<< "$log_snapshot" | tail -1 | tr -d '\r')"
+  [[ -n "$benchmark_json" ]] && break
+  sleep 0.1
+done
+[[ -n "$benchmark_json" ]] || { echo "QNN benchmark timed out; inspect logcat" >&2; exit 1; }
+printf '%s\n' "$benchmark_json" > "$output_dir/raw/android-qnn-benchmark.json"
+python3 - "$output_dir/raw/android-qnn-benchmark.json" <<'PY'
+import json, sys
+value = json.load(open(sys.argv[1], encoding="utf-8"))
+if value.get("backend") != "QNNExecutionProvider" or value.get("cpu_fallback") is not False:
+    raise SystemExit("app result is not fail-closed QNN evidence")
+PY
+
+adb logcat -d -v threadtime > "$output_dir/raw/logcat.txt"
+adb shell dumpsys meminfo "$package_name" > "$output_dir/raw/meminfo-after.txt"
+adb shell dumpsys thermalservice > "$output_dir/raw/thermal-after.txt"
+adb shell getprop ro.build.fingerprint | tr -d '\r' > "$output_dir/raw/build-fingerprint.txt"
+adb shell getprop ro.soc.model | tr -d '\r' > "$output_dir/raw/soc-model.txt"
+
+private_files="$(adb shell run-as "$package_name" ls files | tr -d '\r')"
+context_name="$(grep '^edgegenbench-qnn-context.*\.onnx$' <<< "$private_files" | head -1)"
+profile_name="$(grep '^edgegenbench-qnn-profile.*' <<< "$private_files" | head -1)"
+[[ -n "$context_name" ]] || { echo "QNN context binary was not generated" >&2; exit 1; }
+[[ -n "$profile_name" ]] || { echo "QNN profile was not generated" >&2; exit 1; }
+adb exec-out run-as "$package_name" cat "files/$context_name" > "$output_dir/raw/$context_name"
+adb exec-out run-as "$package_name" cat "files/$profile_name" > "$output_dir/raw/$profile_name"
+
+python3 - "$output_dir/raw/model-input.bin" <<'PY'
+import struct, sys
+values = (0.17, 0.28, 0.41, 0.53, 0.68, 0.79, 0.83, 0.97, 0.32, 0.64)
+open(sys.argv[1], "wb").write(struct.pack("<10f", *values))
+PY
+
+peak_rss_mb="$(awk -v kib="$peak_rss_kib" 'BEGIN {printf "%.6f", kib / 1024.0}')"
+python3 scripts/capture_qnn_evidence.py \
+  --benchmark "$output_dir/raw/android-qnn-benchmark.json" \
+  --context-binary "$output_dir/raw/$context_name" \
+  --placement-report "$placement_path" \
+  --profile "$output_dir/raw/$profile_name" \
+  --logcat "$output_dir/raw/logcat.txt" \
+  --model "$model_path" \
+  --input "$output_dir/raw/model-input.bin" \
+  --output-dir "$output_dir/validated" \
+  --ort-version "$ort_version" \
+  --qairt-version "$qairt_version" \
+  --device-fingerprint "$(<"$output_dir/raw/build-fingerprint.txt")" \
+  --soc-model "$(<"$output_dir/raw/soc-model.txt")" \
+  --cold-ms 0 \
+  --peak-rss-mb "$peak_rss_mb" \
+  --max-abs-drift-vs-fp32 0 \
+  --max-allowed-abs-drift "$max_drift"
+
+echo "Validated Android QNN evidence written to $output_dir/validated"

scripts/capture_qnn_evidence.py

@@ -50,19 +50,30 @@ def capture_qnn_evidence(
     power_tool: str | None = None,
 ) -> Path:
     benchmark = _load_json(benchmark_path)
-    benchmark_placement = benchmark.get("placement")
-    if benchmark.get("backend") != "qnn" or not isinstance(benchmark_placement, dict):
-        raise ValueError("benchmark must be a QNN native runtime result")
-    if (
-        benchmark_placement.get("backend") != "QNNExecutionProvider"
-        or benchmark_placement.get("cpu_fallback") is not False
-        or benchmark_placement.get("hardware_measurement") is not True
-    ):
-        raise ValueError("benchmark does not prove fail-closed QNN session configuration")
-    latency = benchmark.get("latency_ms")
-    if not isinstance(latency, dict):
-        raise ValueError("benchmark requires latency_ms")
-    batch_size = benchmark.get("batch_size")
+    if benchmark.get("backend") == "qnn":
+        benchmark_placement = benchmark.get("placement")
+        if not isinstance(benchmark_placement, dict) or (
+            benchmark_placement.get("backend") != "QNNExecutionProvider"
+            or benchmark_placement.get("cpu_fallback") is not False
+            or benchmark_placement.get("hardware_measurement") is not True
+        ):
+            raise ValueError("benchmark does not prove fail-closed QNN session configuration")
+        latency = benchmark.get("latency_ms")
+        if not isinstance(latency, dict):
+            raise ValueError("native benchmark requires latency_ms")
+        p50 = float(latency["p50"])
+        p95 = float(latency["p95"])
+        batch_size = benchmark.get("batch_size")
+    elif benchmark.get("backend") == "QNNExecutionProvider":
+        if benchmark.get("cpu_fallback") is not False:
+            raise ValueError("Android QNN benchmark must disable CPU fallback")
+        p50 = float(benchmark.get("warm_mean_ms", -1))
+        p95 = float(benchmark.get("warm_p95_ms", -1))
+        batch_size = 1
+        cold_ms = float(benchmark.get("cold_ms", cold_ms))
+        max_abs_drift_vs_fp32 = float(benchmark.get("output_max_abs_drift", max_abs_drift_vs_fp32))
+    else:
+        raise ValueError("benchmark must be a native or Android QNN runtime result")
     if not isinstance(batch_size, int) or batch_size <= 0:
         raise ValueError("benchmark requires a positive batch_size")
 
@@ -87,8 +98,6 @@ def capture_qnn_evidence(
             "sha256": _sha256(target),
         }
 
-    p50 = float(latency["p50"])
-    p95 = float(latency["p95"])
     bundle: dict[str, Any] = {
         "schema_version": 1,
         "project": "EdgeGenBench",

tests/test_qnn_evidence_capture.py

@@ -78,3 +78,46 @@ def test_capture_builds_self_validating_qnn_bundle(tmp_path: Path) -> None:
 def test_capture_rejects_benchmark_with_cpu_fallback(tmp_path: Path) -> None:
     with pytest.raises(ValueError, match="fail-closed QNN"):
         _capture(tmp_path, cpu_fallback=True)
+
+
+def test_capture_accepts_android_qnn_result_contract(tmp_path: Path) -> None:
+    evidence = _capture(tmp_path)
+    benchmark_path = tmp_path / "benchmark.json"
+    benchmark_path.write_text(
+        json.dumps(
+            {
+                "schema_version": 1,
+                "backend": "QNNExecutionProvider",
+                "cpu_fallback": False,
+                "cold_ms": 3.0,
+                "warm_mean_ms": 0.4,
+                "warm_p95_ms": 0.6,
+                "output_max_abs_drift": 0.00001,
+            }
+        ),
+        encoding="utf-8",
+    )
+    value = json.loads(evidence.read_text(encoding="utf-8"))
+    # Reuse the already retained inputs/artifacts but replace the benchmark source.
+    captured = capture_qnn_evidence(
+        benchmark_path=benchmark_path,
+        context_binary=tmp_path / "context.bin",
+        placement_report=tmp_path / "placement.json",
+        profile=tmp_path / "profile.json",
+        logcat=tmp_path / "logcat.txt",
+        model=tmp_path / "model.onnx",
+        input_data=tmp_path / "input.bin",
+        output_dir=tmp_path / "android-evidence",
+        ort_version=value["identity"]["ort_version"],
+        qairt_version=value["identity"]["qairt_version"],
+        device_fingerprint=value["identity"]["device_fingerprint"],
+        soc_model=value["identity"]["soc_model"],
+        cold_ms=1.0,
+        peak_rss_mb=40.0,
+        max_abs_drift_vs_fp32=0.0,
+        max_allowed_abs_drift=0.0001,
+    )
+    android = json.loads(captured.read_text(encoding="utf-8"))
+    assert android["measurements"]["cold_ms"] == 3.0
+    assert android["measurements"]["warm_p50_ms"] == 0.4
+    assert android["measurements"]["throughput_per_second"] == 2500.0