Build end-to-end iOS Core ML acceptance

.github/workflows/ci.yml

@@ -96,6 +96,80 @@ jobs:
           path: build/evidence/android
           if-no-files-found: error
 
+  ios:
+    name: iOS Core ML simulator
+    runs-on: macos-15
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install Core ML export dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install -e ".[neural,coreml]"
+
+      - name: Export current model and stage iOS resources
+        run: python scripts/prepare_ios_resources.py
+
+      - name: Install XcodeGen
+        run: brew install xcodegen
+
+      - name: Generate Xcode project
+        working-directory: ios
+        run: xcodegen generate
+
+      - name: Build unsigned iOS simulator app
+        working-directory: ios
+        run: |
+          xcodebuild \
+            -project EdgeGenBenchDemo.xcodeproj \
+            -scheme EdgeGenBenchDemo \
+            -sdk iphonesimulator \
+            -destination 'generic/platform=iOS Simulator' \
+            -derivedDataPath DerivedData \
+            CODE_SIGNING_ALLOWED=NO \
+            build
+
+      - name: Run XCTest on an available iPhone simulator
+        working-directory: ios
+        run: |
+          DEVICE_ID="$(xcrun simctl list devices available -j | python -c \
+            'import json,sys; d=json.load(sys.stdin)["devices"]; print(next(x["udid"] for values in d.values() for x in values if x["name"].startswith("iPhone")))')"
+          xcodebuild \
+            -project EdgeGenBenchDemo.xcodeproj \
+            -scheme EdgeGenBenchDemo \
+            -destination "platform=iOS Simulator,id=$DEVICE_ID" \
+            -derivedDataPath DerivedData \
+            -resultBundlePath ../build/ios-tests.xcresult \
+            CODE_SIGNING_ALLOWED=NO \
+            test
+
+      - name: Package simulator acceptance evidence
+        run: |
+          mkdir -p build/ios-evidence
+          ditto -c -k --sequesterRsrc --keepParent \
+            ios/DerivedData/Build/Products/Debug-iphonesimulator/EdgeGenBenchDemo.app \
+            build/ios-evidence/EdgeGenBench-ios-simulator-app.zip
+          ditto -c -k --sequesterRsrc --keepParent \
+            build/ios-tests.xcresult \
+            build/ios-evidence/ios-tests.xcresult.zip
+          xcodebuild -version > build/ios-evidence/xcode-version.txt
+          shasum -a 256 build/ios-evidence/* > build/ios-evidence/checksums.txt
+
+      - name: Upload iOS simulator evidence
+        uses: actions/upload-artifact@v4
+        with:
+          name: ios-coreml-simulator-evidence
+          path: build/ios-evidence
+          if-no-files-found: error
+
   test:
     name: Python 3.12 checks
     runs-on: ubuntu-latest
@@ -150,7 +224,7 @@ jobs:
   release-acceptance:
     name: End-to-end release acceptance
     runs-on: ubuntu-latest
-    needs: [native, android, test]
+    needs: [native, android, ios, test]
 
     steps:
       - name: Check out repository
@@ -174,13 +248,20 @@ jobs:
           name: android-verification-evidence
           path: build/input/android
 
+      - name: Download iOS simulator evidence
+        uses: actions/download-artifact@v4
+        with:
+          name: ios-coreml-simulator-evidence
+          path: build/input/ios
+
       - name: Validate and assemble release evidence
         run: |
           python scripts/build_release_evidence.py \
             --baseline build/input/native/baseline.json \
             --fused build/input/native/fused.json \
             --apk build/input/android/EdgeGenBench-0.1.7-device-evidence-debug.apk \
             --alignment-report build/input/android/16kb-alignment.txt \
+            --ios-simulator-evidence build/input/ios \
             --output-dir build/release-evidence \
             --git-revision "$GITHUB_SHA" \
             --version 0.1.7

.gitignore

@@ -13,6 +13,12 @@ __pycache__/
 
 # Generated data, models, and reports
 artifacts/
+!artifacts/
+artifacts/*
+!artifacts/neural_surrogate/
+artifacts/neural_surrogate/*
+!artifacts/neural_surrogate/model.pt
+!artifacts/neural_surrogate/preprocessing.npz
 /models/
 data/raw/*
 !data/raw/.gitkeep
@@ -29,12 +35,17 @@ reports/*
 *.joblib
 *.pt
 *.pth
+!artifacts/neural_surrogate/model.pt
 
 # Native and Android build outputs
 build/
 android/.gradle/
 android/local.properties
 android/**/build/
+ios/EdgeGenBenchDemo/Resources/
+ios/EdgeGenBenchDemo/.generated-coreml/
+ios/*.xcodeproj/
+ios/DerivedData/
 .idea/
 
 # Local backup files

README.md

@@ -31,7 +31,8 @@ The project combines:
 - mixed-precision INT8/FP32 static-QDQ deployment;
 - quantization calibration and drift analysis;
 - CPU and CoreML execution-provider benchmarking;
-- native iOS 17 SwiftUI inference with an exported Core ML model contract;
+- native iOS 17 SwiftUI/Core ML integration with CI simulator acceptance and a
+  physical-device evidence contract;
 - installable iPhone browser inference with ONNX Runtime Web;
 - repeated latency benchmarking;
 - reproducible testing, type checking, and continuous integration.
@@ -42,8 +43,10 @@ proprietary aircraft-manufacturer data, software, or design information.
 The [browser demo](web/README.md) provides the usable iPhone path without
 Xcode: GitHub Pages serves an installable web app and inference runs locally in
 Safari. The separate [native app](ios/README.md) preserves the Core ML route,
-but physical-device latency and energy remain unclaimed until a signed build is
-measured on an iPhone.
+automates current-model export, builds and tests on an unsigned iOS simulator
+in CI, and exports validation-ready physical-iPhone evidence. Device latency,
+ANE placement, and energy remain unclaimed until their respective evidence is
+captured and validated.
 
 ## Native C++ and Android runtime
 

docs/ios_device_validation.md

@@ -0,0 +1,18 @@
+# iPhone evidence acceptance
+
+EdgeGenBench separates three iOS proof levels:
+
+| Level | What it proves | What it does not prove |
+|---|---|---|
+| CI simulator build + XCTest | Core ML resources compile, the Swift app links, and evidence contracts pass tests | Physical-device performance, ANE placement, power |
+| Validated physical-iPhone JSON | Current-model Core ML app latency, device/OS identity, thermal boundary, deterministic repeated output | ANE placement or calibrated power |
+| Retained Instruments capture | Only the metrics and placement visible in the named Instruments templates | Claims outside that measured boundary |
+
+Use [`ios/README.md`](../ios/README.md) for the physical run. The evidence
+validator compares the hashes embedded by the Core ML exporter with the tracked
+checkpoint and preprocessing artifacts. A screenshot alone is supporting
+visual evidence and cannot replace the JSON export.
+
+Do not publish an Apple Neural Engine or energy-saving claim merely because the
+app requests `MLComputeUnits.all`. Core ML remains free to choose an available
+compute unit, and device power requires an appropriate named measurement tool.

ios/EdgeGenBenchDemo/BenchmarkEvidence.swift

@@ -0,0 +1,139 @@
+import Foundation
+import UIKit
+
+struct LatencySummary: Codable {
+    let coldMs: Double
+    let warmMeanMs: Double
+    let warmP95Ms: Double
+    let warmRuns: Int
+}
+
+struct IOSDeviceIdentity: Codable {
+    let model: String
+    let systemName: String
+    let systemVersion: String
+    let simulator: Bool
+}
+
+struct IOSBenchmarkEvidence: Codable {
+    let schemaVersion: String
+    let capturedAtUTC: String
+    let appVersion: String
+    let backend: String
+    let requestedComputeUnits: String
+    let neuralEnginePlacement: String
+    let powerMeasurement: String
+    let thermalStateBefore: String
+    let thermalStateAfter: String
+    let lowPowerMode: Bool
+    let sourceModelSha256: String
+    let preprocessingSha256: String
+    let contractSha256: String
+    let device: IOSDeviceIdentity
+    let latency: LatencySummary
+    let outputMaxAbsDrift: Double
+    let outputs: [PredictionValue]
+}
+
+struct PredictionValue: Codable {
+    let name: String
+    let value: Double
+}
+
+enum BenchmarkStatistics {
+    static func mean(_ values: [Double]) -> Double {
+        values.reduce(0, +) / Double(values.count)
+    }
+
+    static func percentile95(_ values: [Double]) -> Double {
+        let sorted = values.sorted()
+        let index = min(sorted.count - 1, Int(ceil(Double(sorted.count) * 0.95)) - 1)
+        return sorted[index]
+    }
+}
+
+enum IOSBenchmarkRunner {
+    static func run(numericValues: [Double], category: String, warmRuns: Int = 100) throws -> IOSBenchmarkEvidence {
+        precondition(warmRuns > 0)
+        let thermalBefore = ProcessInfo.processInfo.thermalState.label
+        let coldStart = DispatchTime.now().uptimeNanoseconds
+        let predictor = try SurrogatePredictor()
+        let coldOutput = try predictor.predict(numericValues: numericValues, category: category)
+        let coldEnd = DispatchTime.now().uptimeNanoseconds
+
+        var latencies = [Double]()
+        var maxDrift = 0.0
+        for _ in 0..<warmRuns {
+            let start = DispatchTime.now().uptimeNanoseconds
+            let output = try predictor.predict(numericValues: numericValues, category: category)
+            let end = DispatchTime.now().uptimeNanoseconds
+            latencies.append(Double(end - start) / 1_000_000)
+            for (expected, actual) in zip(coldOutput, output) {
+                maxDrift = max(maxDrift, abs(expected.value - actual.value))
+            }
+        }
+
+        return IOSBenchmarkEvidence(
+            schemaVersion: "1.0",
+            capturedAtUTC: ISO8601DateFormatter().string(from: Date()),
+            appVersion: Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "unknown",
+            backend: "CoreML",
+            requestedComputeUnits: "all",
+            neuralEnginePlacement: "not_measured",
+            powerMeasurement: "not_measured",
+            thermalStateBefore: thermalBefore,
+            thermalStateAfter: ProcessInfo.processInfo.thermalState.label,
+            lowPowerMode: ProcessInfo.processInfo.isLowPowerModeEnabled,
+            sourceModelSha256: predictor.contract.sourceModelSha256,
+            preprocessingSha256: predictor.contract.preprocessingSha256,
+            contractSha256: predictor.contractSHA256,
+            device: IOSDeviceIdentity(
+                model: hardwareIdentifier(),
+                systemName: UIDevice.current.systemName,
+                systemVersion: UIDevice.current.systemVersion,
+                simulator: ProcessInfo.processInfo.environment["SIMULATOR_DEVICE_NAME"] != nil
+            ),
+            latency: LatencySummary(
+                coldMs: Double(coldEnd - coldStart) / 1_000_000,
+                warmMeanMs: BenchmarkStatistics.mean(latencies),
+                warmP95Ms: BenchmarkStatistics.percentile95(latencies),
+                warmRuns: warmRuns
+            ),
+            outputMaxAbsDrift: maxDrift,
+            outputs: coldOutput.map { PredictionValue(name: $0.name, value: $0.value) }
+        )
+    }
+
+    private static func hardwareIdentifier() -> String {
+        if let simulatedModel = ProcessInfo.processInfo.environment["SIMULATOR_MODEL_IDENTIFIER"] {
+            return simulatedModel
+        }
+        var systemInfo = utsname()
+        uname(&systemInfo)
+        return withUnsafePointer(to: &systemInfo.machine) {
+            $0.withMemoryRebound(to: CChar.self, capacity: 1) { String(cString: $0) }
+        }
+    }
+}
+
+extension ProcessInfo.ThermalState {
+    var label: String {
+        switch self {
+        case .nominal: return "nominal"
+        case .fair: return "fair"
+        case .serious: return "serious"
+        case .critical: return "critical"
+        @unknown default: return "unknown"
+        }
+    }
+}
+
+extension IOSBenchmarkEvidence {
+    func writeTemporaryJSON() throws -> URL {
+        let encoder = JSONEncoder()
+        encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+        let url = FileManager.default.temporaryDirectory.appendingPathComponent("EdgeGenBench-iOS-evidence.json")
+        try encoder.encode(self).write(to: url, options: .atomic)
+        return url
+    }
+}

ios/EdgeGenBenchDemo/ContentView.swift

@@ -9,7 +9,10 @@ struct ContentView: View {
     @State private var values = [4.0, 250.0, 180.0, 300.0, 0.65, 0.5]
     @State private var category = "battery_electric"
     @State private var predictions: [Prediction] = []
-    @State private var message = "Export and add the model resources, then run a design point."
+    @State private var message = "Run the bundled Core ML model and capture cold + warm evidence."
+    @State private var evidence: IOSBenchmarkEvidence?
+    @State private var evidenceURL: URL?
+    @State private var isRunning = false
 
     var body: some View {
         NavigationStack {
@@ -21,27 +24,49 @@ struct ContentView: View {
                     }
                     TextField("propulsion_architecture", text: $category)
                 }
-                Section { Button("Run on device", action: runPrediction) }
+                Section {
+                    Button(isRunning ? "Benchmarking…" : "Run cold + warm benchmark", action: runBenchmark)
+                        .disabled(isRunning)
+                    if let evidenceURL {
+                        ShareLink(item: evidenceURL) {
+                            Label("Export evidence JSON", systemImage: "square.and.arrow.up")
+                        }
+                    }
+                }
                 Section("Result") {
                     Text(message)
                     ForEach(predictions) { prediction in
                         LabeledContent(prediction.name, value: prediction.value.formatted(.number.precision(.fractionLength(3))))
                     }
+                    if let evidence {
+                        LabeledContent("Backend", value: evidence.backend)
+                        LabeledContent("Cold", value: "\(evidence.latency.coldMs.formatted(.number.precision(.fractionLength(3)))) ms")
+                        LabeledContent("Warm mean", value: "\(evidence.latency.warmMeanMs.formatted(.number.precision(.fractionLength(3)))) ms")
+                        LabeledContent("Warm p95", value: "\(evidence.latency.warmP95Ms.formatted(.number.precision(.fractionLength(3)))) ms")
+                        Text("ANE placement and power are not inferred; use Instruments for those claims.")
+                            .font(.footnote)
+                            .foregroundStyle(.secondary)
+                    }
                 }
             }
             .navigationTitle("EdgeGenBench")
         }
     }
 
-    private func runPrediction() {
+    private func runBenchmark() {
+        isRunning = true
         do {
-            let predictor = try SurrogatePredictor()
-            predictions = try predictor.predict(numericValues: values, category: category)
-            message = "Inference completed with Core ML."
+            let result = try IOSBenchmarkRunner.run(numericValues: values, category: category)
+            evidence = result
+            predictions = result.outputs.map { Prediction(name: $0.name, value: $0.value) }
+            evidenceURL = try result.writeTemporaryJSON()
+            message = "Core ML benchmark completed (1 cold + \(result.latency.warmRuns) warm runs)."
         } catch {
             predictions = []
+            evidence = nil
+            evidenceURL = nil
             message = error.localizedDescription
         }
+        isRunning = false
     }
 }
-