diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 744cbe3..cba96d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,80 @@ jobs: path: build/evidence/android if-no-files-found: error + ios: + name: iOS Core ML simulator + runs-on: macos-15 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install Core ML export dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -e ".[neural,coreml]" + + - name: Export current model and stage iOS resources + run: python scripts/prepare_ios_resources.py + + - name: Install XcodeGen + run: brew install xcodegen + + - name: Generate Xcode project + working-directory: ios + run: xcodegen generate + + - name: Build unsigned iOS simulator app + working-directory: ios + run: | + xcodebuild \ + -project EdgeGenBenchDemo.xcodeproj \ + -scheme EdgeGenBenchDemo \ + -sdk iphonesimulator \ + -destination 'generic/platform=iOS Simulator' \ + -derivedDataPath DerivedData \ + CODE_SIGNING_ALLOWED=NO \ + build + + - name: Run XCTest on an available iPhone simulator + working-directory: ios + run: | + DEVICE_ID="$(xcrun simctl list devices available -j | python -c \ + 'import json,sys; d=json.load(sys.stdin)["devices"]; print(next(x["udid"] for values in d.values() for x in values if x["name"].startswith("iPhone")))')" + xcodebuild \ + -project EdgeGenBenchDemo.xcodeproj \ + -scheme EdgeGenBenchDemo \ + -destination "platform=iOS Simulator,id=$DEVICE_ID" \ + -derivedDataPath DerivedData \ + -resultBundlePath ../build/ios-tests.xcresult \ + CODE_SIGNING_ALLOWED=NO \ + test + + - name: Package simulator acceptance evidence + run: | + mkdir -p build/ios-evidence + ditto -c -k --sequesterRsrc --keepParent \ + ios/DerivedData/Build/Products/Debug-iphonesimulator/EdgeGenBenchDemo.app \ + build/ios-evidence/EdgeGenBench-ios-simulator-app.zip + ditto -c -k --sequesterRsrc --keepParent \ + build/ios-tests.xcresult \ + build/ios-evidence/ios-tests.xcresult.zip + xcodebuild -version > build/ios-evidence/xcode-version.txt + shasum -a 256 build/ios-evidence/* > build/ios-evidence/checksums.txt + + - name: Upload iOS simulator evidence + uses: actions/upload-artifact@v4 + with: + name: ios-coreml-simulator-evidence + path: build/ios-evidence + if-no-files-found: error + test: name: Python 3.12 checks runs-on: ubuntu-latest @@ -150,7 +224,7 @@ jobs: release-acceptance: name: End-to-end release acceptance runs-on: ubuntu-latest - needs: [native, android, test] + needs: [native, android, ios, test] steps: - name: Check out repository @@ -174,6 +248,12 @@ jobs: name: android-verification-evidence path: build/input/android + - name: Download iOS simulator evidence + uses: actions/download-artifact@v4 + with: + name: ios-coreml-simulator-evidence + path: build/input/ios + - name: Validate and assemble release evidence run: | python scripts/build_release_evidence.py \ @@ -181,6 +261,7 @@ jobs: --fused build/input/native/fused.json \ --apk build/input/android/EdgeGenBench-0.1.7-device-evidence-debug.apk \ --alignment-report build/input/android/16kb-alignment.txt \ + --ios-simulator-evidence build/input/ios \ --output-dir build/release-evidence \ --git-revision "$GITHUB_SHA" \ --version 0.1.7 diff --git a/.gitignore b/.gitignore index 17e3fa5..204f590 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,12 @@ __pycache__/ # Generated data, models, and reports artifacts/ +!artifacts/ +artifacts/* +!artifacts/neural_surrogate/ +artifacts/neural_surrogate/* +!artifacts/neural_surrogate/model.pt +!artifacts/neural_surrogate/preprocessing.npz /models/ data/raw/* !data/raw/.gitkeep @@ -29,12 +35,17 @@ reports/* *.joblib *.pt *.pth +!artifacts/neural_surrogate/model.pt # Native and Android build outputs build/ android/.gradle/ android/local.properties android/**/build/ +ios/EdgeGenBenchDemo/Resources/ +ios/EdgeGenBenchDemo/.generated-coreml/ +ios/*.xcodeproj/ +ios/DerivedData/ .idea/ # Local backup files diff --git a/README.md b/README.md index c808c87..ad26751 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ The project combines: - mixed-precision INT8/FP32 static-QDQ deployment; - quantization calibration and drift analysis; - CPU and CoreML execution-provider benchmarking; -- native iOS 17 SwiftUI inference with an exported Core ML model contract; +- native iOS 17 SwiftUI/Core ML integration with CI simulator acceptance and a + physical-device evidence contract; - installable iPhone browser inference with ONNX Runtime Web; - repeated latency benchmarking; - reproducible testing, type checking, and continuous integration. @@ -42,8 +43,10 @@ proprietary aircraft-manufacturer data, software, or design information. The [browser demo](web/README.md) provides the usable iPhone path without Xcode: GitHub Pages serves an installable web app and inference runs locally in Safari. The separate [native app](ios/README.md) preserves the Core ML route, -but physical-device latency and energy remain unclaimed until a signed build is -measured on an iPhone. +automates current-model export, builds and tests on an unsigned iOS simulator +in CI, and exports validation-ready physical-iPhone evidence. Device latency, +ANE placement, and energy remain unclaimed until their respective evidence is +captured and validated. ## Native C++ and Android runtime diff --git a/artifacts/neural_surrogate/model.pt b/artifacts/neural_surrogate/model.pt new file mode 100644 index 0000000..082f5b6 Binary files /dev/null and b/artifacts/neural_surrogate/model.pt differ diff --git a/artifacts/neural_surrogate/preprocessing.npz b/artifacts/neural_surrogate/preprocessing.npz new file mode 100644 index 0000000..42991d1 Binary files /dev/null and b/artifacts/neural_surrogate/preprocessing.npz differ diff --git a/docs/ios_device_validation.md b/docs/ios_device_validation.md new file mode 100644 index 0000000..361ce18 --- /dev/null +++ b/docs/ios_device_validation.md @@ -0,0 +1,18 @@ +# iPhone evidence acceptance + +EdgeGenBench separates three iOS proof levels: + +| Level | What it proves | What it does not prove | +|---|---|---| +| CI simulator build + XCTest | Core ML resources compile, the Swift app links, and evidence contracts pass tests | Physical-device performance, ANE placement, power | +| Validated physical-iPhone JSON | Current-model Core ML app latency, device/OS identity, thermal boundary, deterministic repeated output | ANE placement or calibrated power | +| Retained Instruments capture | Only the metrics and placement visible in the named Instruments templates | Claims outside that measured boundary | + +Use [`ios/README.md`](../ios/README.md) for the physical run. The evidence +validator compares the hashes embedded by the Core ML exporter with the tracked +checkpoint and preprocessing artifacts. A screenshot alone is supporting +visual evidence and cannot replace the JSON export. + +Do not publish an Apple Neural Engine or energy-saving claim merely because the +app requests `MLComputeUnits.all`. Core ML remains free to choose an available +compute unit, and device power requires an appropriate named measurement tool. diff --git a/ios/EdgeGenBenchDemo/BenchmarkEvidence.swift b/ios/EdgeGenBenchDemo/BenchmarkEvidence.swift new file mode 100644 index 0000000..54d0120 --- /dev/null +++ b/ios/EdgeGenBenchDemo/BenchmarkEvidence.swift @@ -0,0 +1,139 @@ +import Foundation +import UIKit + +struct LatencySummary: Codable { + let coldMs: Double + let warmMeanMs: Double + let warmP95Ms: Double + let warmRuns: Int +} + +struct IOSDeviceIdentity: Codable { + let model: String + let systemName: String + let systemVersion: String + let simulator: Bool +} + +struct IOSBenchmarkEvidence: Codable { + let schemaVersion: String + let capturedAtUTC: String + let appVersion: String + let backend: String + let requestedComputeUnits: String + let neuralEnginePlacement: String + let powerMeasurement: String + let thermalStateBefore: String + let thermalStateAfter: String + let lowPowerMode: Bool + let sourceModelSha256: String + let preprocessingSha256: String + let contractSha256: String + let device: IOSDeviceIdentity + let latency: LatencySummary + let outputMaxAbsDrift: Double + let outputs: [PredictionValue] +} + +struct PredictionValue: Codable { + let name: String + let value: Double +} + +enum BenchmarkStatistics { + static func mean(_ values: [Double]) -> Double { + values.reduce(0, +) / Double(values.count) + } + + static func percentile95(_ values: [Double]) -> Double { + let sorted = values.sorted() + let index = min(sorted.count - 1, Int(ceil(Double(sorted.count) * 0.95)) - 1) + return sorted[index] + } +} + +enum IOSBenchmarkRunner { + static func run(numericValues: [Double], category: String, warmRuns: Int = 100) throws -> IOSBenchmarkEvidence { + precondition(warmRuns > 0) + let thermalBefore = ProcessInfo.processInfo.thermalState.label + let coldStart = DispatchTime.now().uptimeNanoseconds + let predictor = try SurrogatePredictor() + let coldOutput = try predictor.predict(numericValues: numericValues, category: category) + let coldEnd = DispatchTime.now().uptimeNanoseconds + + var latencies = [Double]() + var maxDrift = 0.0 + for _ in 0.. String { + if let simulatedModel = ProcessInfo.processInfo.environment["SIMULATOR_MODEL_IDENTIFIER"] { + return simulatedModel + } + var systemInfo = utsname() + uname(&systemInfo) + return withUnsafePointer(to: &systemInfo.machine) { + $0.withMemoryRebound(to: CChar.self, capacity: 1) { String(cString: $0) } + } + } +} + +extension ProcessInfo.ThermalState { + var label: String { + switch self { + case .nominal: return "nominal" + case .fair: return "fair" + case .serious: return "serious" + case .critical: return "critical" + @unknown default: return "unknown" + } + } +} + +extension IOSBenchmarkEvidence { + func writeTemporaryJSON() throws -> URL { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let url = FileManager.default.temporaryDirectory.appendingPathComponent("EdgeGenBench-iOS-evidence.json") + try encoder.encode(self).write(to: url, options: .atomic) + return url + } +} diff --git a/ios/EdgeGenBenchDemo/ContentView.swift b/ios/EdgeGenBenchDemo/ContentView.swift index 777e917..321e12a 100644 --- a/ios/EdgeGenBenchDemo/ContentView.swift +++ b/ios/EdgeGenBenchDemo/ContentView.swift @@ -9,7 +9,10 @@ struct ContentView: View { @State private var values = [4.0, 250.0, 180.0, 300.0, 0.65, 0.5] @State private var category = "battery_electric" @State private var predictions: [Prediction] = [] - @State private var message = "Export and add the model resources, then run a design point." + @State private var message = "Run the bundled Core ML model and capture cold + warm evidence." + @State private var evidence: IOSBenchmarkEvidence? + @State private var evidenceURL: URL? + @State private var isRunning = false var body: some View { NavigationStack { @@ -21,27 +24,49 @@ struct ContentView: View { } TextField("propulsion_architecture", text: $category) } - Section { Button("Run on device", action: runPrediction) } + Section { + Button(isRunning ? "Benchmarking…" : "Run cold + warm benchmark", action: runBenchmark) + .disabled(isRunning) + if let evidenceURL { + ShareLink(item: evidenceURL) { + Label("Export evidence JSON", systemImage: "square.and.arrow.up") + } + } + } Section("Result") { Text(message) ForEach(predictions) { prediction in LabeledContent(prediction.name, value: prediction.value.formatted(.number.precision(.fractionLength(3)))) } + if let evidence { + LabeledContent("Backend", value: evidence.backend) + LabeledContent("Cold", value: "\(evidence.latency.coldMs.formatted(.number.precision(.fractionLength(3)))) ms") + LabeledContent("Warm mean", value: "\(evidence.latency.warmMeanMs.formatted(.number.precision(.fractionLength(3)))) ms") + LabeledContent("Warm p95", value: "\(evidence.latency.warmP95Ms.formatted(.number.precision(.fractionLength(3)))) ms") + Text("ANE placement and power are not inferred; use Instruments for those claims.") + .font(.footnote) + .foregroundStyle(.secondary) + } } } .navigationTitle("EdgeGenBench") } } - private func runPrediction() { + private func runBenchmark() { + isRunning = true do { - let predictor = try SurrogatePredictor() - predictions = try predictor.predict(numericValues: values, category: category) - message = "Inference completed with Core ML." + let result = try IOSBenchmarkRunner.run(numericValues: values, category: category) + evidence = result + predictions = result.outputs.map { Prediction(name: $0.name, value: $0.value) } + evidenceURL = try result.writeTemporaryJSON() + message = "Core ML benchmark completed (1 cold + \(result.latency.warmRuns) warm runs)." } catch { predictions = [] + evidence = nil + evidenceURL = nil message = error.localizedDescription } + isRunning = false } } - diff --git a/ios/EdgeGenBenchDemo/SurrogatePredictor.swift b/ios/EdgeGenBenchDemo/SurrogatePredictor.swift index 9a72a31..5aee0ec 100644 --- a/ios/EdgeGenBenchDemo/SurrogatePredictor.swift +++ b/ios/EdgeGenBenchDemo/SurrogatePredictor.swift @@ -1,8 +1,11 @@ import CoreML +import CryptoKit import Foundation struct ModelContract: Decodable { let schemaVersion: String + let sourceModelSha256: String + let preprocessingSha256: String let inputName: String let outputName: String let numericFeatures: [String] @@ -39,13 +42,16 @@ enum SurrogateError: LocalizedError { final class SurrogatePredictor { let contract: ModelContract + let contractSHA256: String private let model: MLModel init(bundle: Bundle = .main) throws { guard let contractURL = bundle.url(forResource: "ModelContract", withExtension: "json") else { throw SurrogateError.missingResource("ModelContract.json") } - contract = try JSONDecoder().decode(ModelContract.self, from: Data(contentsOf: contractURL)) + let contractData = try Data(contentsOf: contractURL) + contract = try JSONDecoder().decode(ModelContract.self, from: contractData) + contractSHA256 = SHA256.hash(data: contractData).map { String(format: "%02x", $0) }.joined() guard contract.featureMean.count + contract.categories.count == contract.inputDimension, contract.featureScale.count == contract.featureMean.count, contract.targets.count == contract.outputDimension, @@ -56,7 +62,9 @@ final class SurrogatePredictor { guard let modelURL = bundle.url(forResource: "NeuralSurrogate", withExtension: "mlmodelc") else { throw SurrogateError.missingResource("NeuralSurrogate.mlpackage") } - model = try MLModel(contentsOf: modelURL) + let configuration = MLModelConfiguration() + configuration.computeUnits = .all + model = try MLModel(contentsOf: modelURL, configuration: configuration) guard model.modelDescription.inputDescriptionsByName[contract.inputName] != nil, model.modelDescription.outputDescriptionsByName[contract.outputName] != nil else { throw SurrogateError.invalidContract("Core ML feature names do not agree") @@ -86,4 +94,3 @@ final class SurrogatePredictor { } } } - diff --git a/ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift b/ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift new file mode 100644 index 0000000..e383181 --- /dev/null +++ b/ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift @@ -0,0 +1,36 @@ +import XCTest +@testable import EdgeGenBenchDemo + +final class BenchmarkStatisticsTests: XCTestCase { + func testMeanAndNearestRankP95() { + let values = Array(1...100).map(Double.init) + XCTAssertEqual(BenchmarkStatistics.mean(values), 50.5) + XCTAssertEqual(BenchmarkStatistics.percentile95(values), 95.0) + } + + func testEvidenceRoundTripsAsJSON() throws { + let evidence = IOSBenchmarkEvidence( + schemaVersion: "1.0", + capturedAtUTC: "2026-08-27T00:00:00Z", + appVersion: "0.1.0", + backend: "CoreML", + requestedComputeUnits: "all", + neuralEnginePlacement: "not_measured", + powerMeasurement: "not_measured", + thermalStateBefore: "nominal", + thermalStateAfter: "nominal", + lowPowerMode: false, + sourceModelSha256: String(repeating: "a", count: 64), + preprocessingSha256: String(repeating: "b", count: 64), + contractSha256: String(repeating: "c", count: 64), + device: IOSDeviceIdentity(model: "iPhone", systemName: "iOS", systemVersion: "17.0", simulator: true), + latency: LatencySummary(coldMs: 2, warmMeanMs: 1, warmP95Ms: 1.2, warmRuns: 100), + outputMaxAbsDrift: 0, + outputs: [PredictionValue(name: "mass", value: 1)] + ) + let data = try JSONEncoder().encode(evidence) + let decoded = try JSONDecoder().decode(IOSBenchmarkEvidence.self, from: data) + XCTAssertEqual(decoded.latency.warmRuns, 100) + XCTAssertEqual(decoded.backend, "CoreML") + } +} diff --git a/ios/README.md b/ios/README.md index 35680db..e86bc81 100644 --- a/ios/README.md +++ b/ios/README.md @@ -1,48 +1,68 @@ -# Native iOS/Core ML demo +# Native iOS/Core ML runtime -This is the optional native route. The supported no-Xcode deliverable is the -installable [browser app](../web/README.md); it is the quickest way to use the -project on an iPhone without an Apple Developer membership or App Store review. +This SwiftUI application exports the repository's current neural checkpoint to +a Core ML FP16 ML Program, applies the same preprocessing contract as Python, +and records one cold plus 100 warm application-level inference runs. The app +exports evidence JSON containing model provenance, device and OS identity, +thermal state, latency, and repeated-output drift. -This SwiftUI app runs the compact neural surrogate through Core ML on an iOS -17 device. Python owns the trained model and preprocessing statistics; the -exporter writes both the FP16 ML Program and a JSON contract so Swift applies -the same feature normalization, category encoding, and target inverse scaling. +The unsigned simulator build and XCTest suite run in CI. Simulator results +prove integration and compatibility only; they are not physical-iPhone +performance, Apple Neural Engine placement, energy, or power evidence. -On macOS, export the resources: +## Generate the app -```bash -python -m pip install -e '.[neural,coreml]' -python scripts/export_coreml.py \ - --model artifacts/neural/neural_surrogate.pt \ - --preprocessing artifacts/neural/preprocessing.npz -cp -R artifacts/coreml/NeuralSurrogate.mlpackage ios/EdgeGenBenchDemo/ -cp artifacts/coreml/ModelContract.json ios/EdgeGenBenchDemo/ -``` - -Generate and open the Xcode project: +From the repository root on macOS: ```bash +python -m pip install -e '.[neural,coreml]' +python scripts/prepare_ios_resources.py brew install xcodegen cd ios xcodegen generate open EdgeGenBenchDemo.xcodeproj ``` -Select the two generated resources in Xcode and confirm that -`EdgeGenBenchDemo` appears under Target Membership. Choose a development team, -run on a physical iPhone, and use Instruments or MetricKit for device latency -and energy evidence. The repository supplies the native integration, but does -not claim a physical-device result without a signed build and captured run. - -## Delivery choices - -- **Installable browser app:** publish the repository's `web/` app with GitHub - Pages and add it to the iPhone Home Screen. This route is implemented and - needs neither Xcode nor Apple signing. -- **Cloud-built native app:** use Expo EAS or a hosted macOS CI runner to build - and sign a native package. This avoids local Xcode but still needs Apple - credentials and does not remove TestFlight/App Store requirements. -- **Local native app:** use this SwiftUI/Core ML target with Xcode. This is the - right route when the goal is measured Core ML or Apple Neural Engine evidence - on a physical iPhone. +The resource script uses the tracked current-model checkpoint and preprocessing +state. It embeds their SHA-256 values in `ModelContract.json`; the exported +device evidence is rejected if those values do not match the repository. + +## Run on a physical iPhone + +1. Install the full Xcode application from Apple and open + `ios/EdgeGenBenchDemo.xcodeproj`. +2. Connect the iPhone by USB, unlock it, tap **Trust** if prompted, and enable + Developer Mode if Xcode requests it. +3. Select the `EdgeGenBenchDemo` target. Under **Signing & Capabilities**, choose + your personal development team. EdgeGenBench does not collect or store your + signing identity. +4. Select the connected iPhone as the run destination and press **Run**. +5. Keep Low Power Mode off, close other foreground apps, and let the device + reach a stable temperature. +6. In EdgeGenBench, tap **Run cold + warm benchmark**. Confirm it reports one + cold and 100 warm runs. +7. Tap **Export evidence JSON**, AirDrop or save the file to the Mac, and retain + a screenshot of the result screen. +8. Validate and render the report from the repository root: + +```bash +python scripts/validate_ios_evidence.py \ + "$HOME/Downloads/EdgeGenBench-iOS-evidence.json" \ + --output-json reports/ios_device_summary.json \ + --output-markdown reports/ios_device_report.md +``` + +Commit the raw evidence JSON, generated summary, Markdown report, and screenshot +in a follow-up evidence PR. The validator rejects simulator evidence, fewer than +100 warm runs, excessive output drift, mismatched model/preprocessing hashes, +and unsupported ANE or power claims. + +## Optional Instruments evidence + +The app requests Core ML `MLComputeUnits.all`, which does **not** prove that the +Apple Neural Engine executed the graph. To claim ANE placement or energy, run a +Release build under Xcode Instruments with the Core ML and Energy Log templates, +retain the `.trace` bundle or exported summary, name the Xcode/iOS/device +versions, and report the measured boundary. Without that evidence the project +correctly reports `neuralEnginePlacement=not_measured` and +`powerMeasurement=not_measured`. diff --git a/ios/project.yml b/ios/project.yml index ad07083..268575c 100644 --- a/ios/project.yml +++ b/ios/project.yml @@ -11,8 +11,30 @@ targets: settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.triasha72.EdgeGenBenchDemo + MARKETING_VERSION: 0.1.0 + CURRENT_PROJECT_VERSION: 1 SWIFT_VERSION: 5.0 GENERATE_INFOPLIST_FILE: YES INFOPLIST_KEY_UILaunchScreen_Generation: YES CODE_SIGN_STYLE: Automatic - + EdgeGenBenchDemoTests: + type: bundle.unit-test + platform: iOS + deploymentTarget: "17.0" + sources: + - EdgeGenBenchDemoTests + dependencies: + - target: EdgeGenBenchDemo + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.triasha72.EdgeGenBenchDemoTests + SWIFT_VERSION: 5.0 +schemes: + EdgeGenBenchDemo: + build: + targets: + EdgeGenBenchDemo: all + EdgeGenBenchDemoTests: [test] + test: + targets: + - EdgeGenBenchDemoTests diff --git a/reports/portfolio_acceptance.json b/reports/portfolio_acceptance.json index d3f3033..fc82f53 100644 --- a/reports/portfolio_acceptance.json +++ b/reports/portfolio_acceptance.json @@ -83,6 +83,14 @@ "report": "reports/android_16kb_emulator_reference_v0_1_7.md", "claim": "APK/JNI reference path executed on PAGE_SIZE=16384; not physical-device performance." }, + "ios_coreml": { + "status": "ci_build_and_simulator_test_configured", + "backend": "CoreML", + "source_model_sha256": "55d6db9f19f2e361c6066b639920cfd1aad54ea5544b463b205857ebb7ceb657", + "preprocessing_sha256": "c53831dd106a26b668d586b4eb83f73a1e43483c52ec8ee36c7ba35c95cdb08e", + "physical_device_status": "evidence_pending", + "claim": "Core ML export, SwiftUI app, XCTest, and unsigned simulator CI are configured; physical-iPhone latency remains pending a validated device export." + }, "power": { "status": "not_measured", "claim": "No power-savings claim is made without a named calibrated tool." diff --git a/reports/portfolio_acceptance.md b/reports/portfolio_acceptance.md index 48d110a..fd8ec8b 100644 --- a/reports/portfolio_acceptance.md +++ b/reports/portfolio_acceptance.md @@ -7,6 +7,7 @@ | `qualcomm_ai_hub_qnn` | `validated_ai_hub_physical_qnn` | Physical AI Hub model profiling; not Android APK end-to-end latency. | | `android_qnn_apk` | `implementation_complete_evidence_pending` | Build/JNI/capture paths exist; requires a supported Snapdragon APK run. | | `android_16kb_runtime` | `validated_16kb_emulator_runtime` | APK/JNI reference path executed on PAGE_SIZE=16384; not physical-device performance. | +| `ios_coreml` | `ci_build_and_simulator_test_configured` | Core ML export, SwiftUI app, XCTest, and unsigned simulator CI are configured; physical-iPhone latency remains pending a validated device export. | | `power` | `not_measured` | No power-savings claim is made without a named calibrated tool. | ## Validated Qualcomm QNN results @@ -22,4 +23,4 @@ Tracked QNN context provenance match: **True** (`artifacts/qualcomm_ai_hub/curre | 256 | 0.047000 | 5446808.511 | 122896384 | NPU × 9 | 0.002865936 | AI Hub measurements are physical-device model profiles, not Android application end-to-end timings. Current-model acceptance requires source-model provenance to match the repository, as reported above. -Power remains unmeasured. The remaining hardware proof item is a supported-device QNN APK run; the 16 KB reference APK/JNI runtime is validated on an API 35 emulator. +Power remains unmeasured. Remaining physical proof items are a supported-device QNN APK run and a validated iPhone Core ML export; the iOS simulator build/test lane does not establish device latency, ANE placement, or energy use. diff --git a/scripts/build_portfolio_acceptance.py b/scripts/build_portfolio_acceptance.py index ea1d134..56a7cf1 100644 --- a/scripts/build_portfolio_acceptance.py +++ b/scripts/build_portfolio_acceptance.py @@ -189,6 +189,39 @@ def validate_android_16kb_runtime(evidence_dir: Path, report_path: Path) -> dict } +def validate_ios_coreml_implementation(repository_root: Path) -> dict[str, Any]: + required = [ + repository_root / "ios/project.yml", + repository_root / "ios/EdgeGenBenchDemo/SurrogatePredictor.swift", + repository_root / "ios/EdgeGenBenchDemo/BenchmarkEvidence.swift", + repository_root / "ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift", + repository_root / "scripts/prepare_ios_resources.py", + repository_root / "scripts/validate_ios_evidence.py", + repository_root / "artifacts/neural_surrogate/model.pt", + repository_root / "artifacts/neural_surrogate/preprocessing.npz", + ] + missing = [ + path.relative_to(repository_root).as_posix() for path in required if not path.is_file() + ] + if missing: + raise ValueError(f"iOS Core ML implementation is incomplete: {', '.join(missing)}") + workflow = (repository_root / ".github/workflows/ci.yml").read_text(encoding="utf-8") + for marker in ("iOS Core ML simulator", "prepare_ios_resources.py", "xcodebuild", "test"): + if marker not in workflow: + raise ValueError(f"iOS CI is missing required marker: {marker}") + return { + "status": "ci_build_and_simulator_test_configured", + "backend": "CoreML", + "source_model_sha256": _sha256(required[-2]), + "preprocessing_sha256": _sha256(required[-1]), + "physical_device_status": "evidence_pending", + "claim": ( + "Core ML export, SwiftUI app, XCTest, and unsigned simulator CI are configured; " + "physical-iPhone latency remains pending a validated device export." + ), + } + + def build_portfolio_acceptance( *, repository_root: Path, qnn_report: Path, output_json: Path, output_markdown: Path ) -> dict[str, Any]: @@ -201,6 +234,7 @@ def build_portfolio_acceptance( repository_root / "reports/device/android-16kb-api35-reference-10-runs", android_16kb_report, ) + ios_coreml = validate_ios_coreml_implementation(repository_root) matrix = { "schema_version": 1, "project": "EdgeGenBench", @@ -221,6 +255,7 @@ def build_portfolio_acceptance( "claim": "Build/JNI/capture paths exist; requires a supported Snapdragon APK run.", }, "android_16kb_runtime": android_16kb, + "ios_coreml": ios_coreml, "power": { "status": "not_measured", "claim": "No power-savings claim is made without a named calibrated tool.", @@ -266,8 +301,9 @@ def build_portfolio_acceptance( "AI Hub measurements are physical-device model profiles, not Android " "application end-to-end timings. Current-model acceptance requires source-model " "provenance to match the repository, as reported above.", - "Power remains unmeasured. The remaining hardware proof item is a supported-device " - "QNN APK run; the 16 KB reference APK/JNI runtime is validated on an API 35 emulator.", + "Power remains unmeasured. Remaining physical proof items are a supported-device " + "QNN APK run and a validated iPhone Core ML export; the iOS simulator build/test " + "lane does not establish device latency, ANE placement, or energy use.", ] ) output_markdown.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/scripts/build_release_evidence.py b/scripts/build_release_evidence.py index 0f0d544..2623fbf 100644 --- a/scripts/build_release_evidence.py +++ b/scripts/build_release_evidence.py @@ -311,6 +311,7 @@ def build_release_evidence( version: str, device_evidence: Path | None = None, qnn_evidence: Path | None = None, + ios_simulator_evidence: Path | None = None, ) -> Path: baseline = _load_json(baseline_path) fused = _load_json(fused_path) @@ -393,6 +394,35 @@ def build_release_evidence( "claim": "Exclusive QNN placement validated with CPU fallback disabled.", } + ios_status: dict[str, Any] = { + "status": "not_supplied", + "claim": "No iOS build or physical-iPhone claim is made by this bundle.", + } + if ios_simulator_evidence is not None: + if not ios_simulator_evidence.is_dir(): + raise ValueError("iOS simulator evidence must be a directory") + required_ios = ( + "EdgeGenBench-ios-simulator-app.zip", + "ios-tests.xcresult.zip", + "xcode-version.txt", + "checksums.txt", + ) + missing_ios = [ + name for name in required_ios if not (ios_simulator_evidence / name).is_file() + ] + if missing_ios: + raise ValueError(f"iOS simulator evidence is incomplete: {', '.join(missing_ios)}") + destination = output_dir / "ios-simulator" + shutil.copytree(ios_simulator_evidence, destination, dirs_exist_ok=True) + ios_status = { + "status": "validated_in_ci", + "path": "ios-simulator", + "claim": ( + "Unsigned Core ML simulator app built and XCTest result retained; " + "not physical-iPhone latency, ANE placement, or power evidence." + ), + } + files = [] for path in sorted( p for p in output_dir.rglob("*") if p.is_file() and p.name != "manifest.json" @@ -415,12 +445,14 @@ def build_release_evidence( "native_fused_passed": True, "baseline_fused_max_abs_drift": drift, "android_16kb_compatible": True, + "ios_coreml_simulator_build_and_tests": ios_status["status"] == "validated_in_ci", "cpu_fallback_claim": "not applicable to deterministic reference backend", "qnn_npu_placement": "not tested in CI", "power": "not measured", }, "device_evidence": device_status, "qnn_evidence": qnn_status, + "ios_simulator_evidence": ios_status, "files": files, } manifest_path = output_dir / "manifest.json" @@ -439,6 +471,7 @@ def main() -> None: parser.add_argument("--version", required=True) parser.add_argument("--device-evidence", type=Path) parser.add_argument("--qnn-evidence", type=Path) + parser.add_argument("--ios-simulator-evidence", type=Path) args = parser.parse_args() manifest = build_release_evidence( args.baseline, @@ -450,6 +483,7 @@ def main() -> None: version=args.version, device_evidence=args.device_evidence, qnn_evidence=args.qnn_evidence, + ios_simulator_evidence=args.ios_simulator_evidence, ) print(f"Release evidence validated: {manifest}") diff --git a/scripts/prepare_ios_resources.py b/scripts/prepare_ios_resources.py new file mode 100644 index 0000000..6b14e21 --- /dev/null +++ b/scripts/prepare_ios_resources.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Export and stage the current Core ML model for the native iOS target.""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +from edgegenbench.deployment.coreml_export import export_neural_surrogate_coreml + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, default=Path("artifacts/neural_surrogate/model.pt")) + parser.add_argument( + "--preprocessing", + type=Path, + default=Path("artifacts/neural_surrogate/preprocessing.npz"), + ) + parser.add_argument("--target", type=Path, default=Path("ios/EdgeGenBenchDemo/Resources")) + args = parser.parse_args() + + build_dir = args.target.parent / ".generated-coreml" + shutil.rmtree(build_dir, ignore_errors=True) + shutil.rmtree(args.target, ignore_errors=True) + artifacts = export_neural_surrogate_coreml( + model_path=args.model, + preprocessing_path=args.preprocessing, + output_dir=build_dir, + ) + args.target.mkdir(parents=True, exist_ok=True) + shutil.copytree(artifacts.model_path, args.target / artifacts.model_path.name) + shutil.copy2(artifacts.contract_path, args.target / artifacts.contract_path.name) + shutil.rmtree(build_dir) + print(f"Staged Core ML resources in {args.target}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_ios_evidence.py b/scripts/validate_ios_evidence.py new file mode 100644 index 0000000..8674fff --- /dev/null +++ b/scripts/validate_ios_evidence.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Validate an EdgeGenBench iPhone Core ML evidence export and write a report.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def validate_ios_evidence( + evidence_path: Path, + *, + model_path: Path, + preprocessing_path: Path, + allow_simulator: bool = False, +) -> dict[str, Any]: + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + if not isinstance(evidence, dict) or evidence.get("schemaVersion") != "1.0": + raise ValueError("unsupported iOS evidence schema") + if evidence.get("backend") != "CoreML" or evidence.get("requestedComputeUnits") != "all": + raise ValueError("evidence must identify Core ML with requested compute units") + if evidence.get("neuralEnginePlacement") != "not_measured": + raise ValueError("ANE placement cannot be inferred without retained placement evidence") + if evidence.get("powerMeasurement") != "not_measured": + raise ValueError("power claims require a named calibrated measurement tool") + + device = evidence.get("device") + latency = evidence.get("latency") + if not isinstance(device, dict) or not isinstance(latency, dict): + raise ValueError("device identity and latency summary are required") + if bool(device.get("simulator")) and not allow_simulator: + raise ValueError("physical-iPhone evidence cannot come from a simulator") + if device.get("systemName") != "iOS" and not allow_simulator: + raise ValueError("physical evidence must identify iOS") + if int(latency.get("warmRuns", 0)) < 100: + raise ValueError("at least 100 warm inference runs are required") + for name in ("coldMs", "warmMeanMs", "warmP95Ms"): + if float(latency.get(name, 0)) <= 0: + raise ValueError(f"{name} must be positive") + if float(evidence.get("outputMaxAbsDrift", 1.0)) > 1e-6: + raise ValueError("iOS repeated-output drift exceeds tolerance") + if evidence.get("sourceModelSha256") != _sha256(model_path): + raise ValueError("iOS source-model provenance does not match the repository") + if evidence.get("preprocessingSha256") != _sha256(preprocessing_path): + raise ValueError("iOS preprocessing provenance does not match the repository") + + return { + "status": "validated_physical_iphone_coreml" + if not device["simulator"] + else "validated_simulator_coreml", + "captured_at_utc": evidence["capturedAtUTC"], + "app_version": evidence["appVersion"], + "device": device, + "backend": "CoreML", + "requested_compute_units": "all", + "latency": latency, + "output_max_abs_drift": evidence["outputMaxAbsDrift"], + "thermal_state_before": evidence["thermalStateBefore"], + "thermal_state_after": evidence["thermalStateAfter"], + "power_measurement": "not_measured", + "neural_engine_placement": "not_measured", + "claim_boundary": ( + "Physical iPhone Core ML application latency; not proof of Apple Neural Engine " + "placement and not a power measurement." + ), + } + + +def write_report(summary: dict[str, Any], output_json: Path, output_markdown: Path) -> None: + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + latency = summary["latency"] + device = summary["device"] + lines = [ + "# EdgeGenBench physical iPhone Core ML report", + "", + f"- Status: `{summary['status']}`", + f"- Device: `{device['model']}`", + f"- OS: `{device['systemName']} {device['systemVersion']}`", + f"- Backend: `{summary['backend']}` (requested compute units: `all`)", + f"- Cold latency: `{latency['coldMs']:.6f} ms`", + f"- Warm mean latency: `{latency['warmMeanMs']:.6f} ms`", + f"- Warm p95 latency: `{latency['warmP95Ms']:.6f} ms`", + f"- Warm runs: `{latency['warmRuns']}`", + f"- Output max absolute drift: `{summary['output_max_abs_drift']}`", + "- Thermal state: " + f"`{summary['thermal_state_before']}` → `{summary['thermal_state_after']}`", + "- Power: `not measured`", + "- Apple Neural Engine placement: `not measured`", + "", + f"> {summary['claim_boundary']}", + ] + output_markdown.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("evidence", type=Path) + parser.add_argument("--model", type=Path, default=Path("artifacts/neural_surrogate/model.pt")) + parser.add_argument( + "--preprocessing", type=Path, default=Path("artifacts/neural_surrogate/preprocessing.npz") + ) + parser.add_argument("--output-json", type=Path, default=Path("reports/ios_device_summary.json")) + parser.add_argument( + "--output-markdown", type=Path, default=Path("reports/ios_device_report.md") + ) + parser.add_argument("--allow-simulator", action="store_true") + args = parser.parse_args() + summary = validate_ios_evidence( + args.evidence, + model_path=args.model, + preprocessing_path=args.preprocessing, + allow_simulator=args.allow_simulator, + ) + write_report(summary, args.output_json, args.output_markdown) + print(f"Validated iOS evidence: {args.output_markdown}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/edgegenbench/deployment/coreml_export.py b/src/edgegenbench/deployment/coreml_export.py index 7f1b272..64fd5e9 100644 --- a/src/edgegenbench/deployment/coreml_export.py +++ b/src/edgegenbench/deployment/coreml_export.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from dataclasses import dataclass from pathlib import Path @@ -27,10 +28,25 @@ class CoreMLExportArtifacts: output_dim: int -def build_ios_contract(preprocessor: NeuralPreprocessor) -> dict[str, Any]: +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def build_ios_contract( + preprocessor: NeuralPreprocessor, + *, + source_model_sha256: str | None = None, + preprocessing_sha256: str | None = None, +) -> dict[str, Any]: """Serialize preprocessing and inverse-scaling rules for Swift.""" return { - "schemaVersion": "1.0", + "schemaVersion": "1.1", + "sourceModelSha256": source_model_sha256, + "preprocessingSha256": preprocessing_sha256, "inputName": COREML_INPUT_NAME, "outputName": COREML_OUTPUT_NAME, "numericFeatures": list(NUMERIC_FEATURES), @@ -82,7 +98,15 @@ def export_neural_surrogate_coreml( contract_destination = output_dir / "ModelContract.json" converted.save(str(model_destination)) contract_destination.write_text( - json.dumps(build_ios_contract(preprocessor), indent=2) + "\n", + json.dumps( + build_ios_contract( + preprocessor, + source_model_sha256=_sha256(model_path), + preprocessing_sha256=_sha256(preprocessing_path), + ), + indent=2, + ) + + "\n", encoding="utf-8", ) return CoreMLExportArtifacts( diff --git a/tests/neural/test_coreml_export.py b/tests/neural/test_coreml_export.py index 1c43db4..4242521 100644 --- a/tests/neural/test_coreml_export.py +++ b/tests/neural/test_coreml_export.py @@ -16,9 +16,11 @@ def test_ios_contract_preserves_preprocessing_and_output_scaling() -> None: targets=("a", "b", "c", "d", "e", "f"), ) contract = build_ios_contract(preprocessor) + assert contract["schemaVersion"] == "1.1" assert contract["inputName"] == "features" assert contract["outputName"] == "predictions" assert contract["inputDimension"] == 9 assert contract["outputDimension"] == 6 assert contract["categories"] == ["battery_electric", "hybrid", "hydrogen"] assert contract["targetScale"] == [2.0] * 6 + assert contract["sourceModelSha256"] is None diff --git a/tests/test_ios_evidence.py b/tests/test_ios_evidence.py new file mode 100644 index 0000000..d033a13 --- /dev/null +++ b/tests/test_ios_evidence.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import hashlib +import json +import runpy +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import pytest + +SCRIPT = Path(__file__).parents[1] / "scripts/validate_ios_evidence.py" +ValidateIOS = Callable[..., dict[str, object]] +validate_ios_evidence = cast(ValidateIOS, runpy.run_path(SCRIPT)["validate_ios_evidence"]) + + +def _hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _evidence(model: Path, preprocessing: Path) -> dict[str, object]: + return { + "schemaVersion": "1.0", + "capturedAtUTC": "2026-08-27T00:00:00Z", + "appVersion": "0.1.0", + "backend": "CoreML", + "requestedComputeUnits": "all", + "neuralEnginePlacement": "not_measured", + "powerMeasurement": "not_measured", + "thermalStateBefore": "nominal", + "thermalStateAfter": "fair", + "lowPowerMode": False, + "sourceModelSha256": _hash(model), + "preprocessingSha256": _hash(preprocessing), + "contractSha256": "c" * 64, + "device": { + "model": "iPhone15,4", + "systemName": "iOS", + "systemVersion": "17.6", + "simulator": False, + }, + "latency": {"coldMs": 3.0, "warmMeanMs": 1.0, "warmP95Ms": 1.2, "warmRuns": 100}, + "outputMaxAbsDrift": 0.0, + "outputs": [{"name": "mass", "value": 1.0}], + } + + +def test_validates_physical_iphone_coreml_evidence(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + preprocessing = tmp_path / "preprocessing.npz" + model.write_bytes(b"model") + preprocessing.write_bytes(b"preprocessing") + evidence = tmp_path / "evidence.json" + evidence.write_text(json.dumps(_evidence(model, preprocessing))) + result = validate_ios_evidence(evidence, model_path=model, preprocessing_path=preprocessing) + assert result["status"] == "validated_physical_iphone_coreml" + assert result["neural_engine_placement"] == "not_measured" + + +def test_rejects_simulator_as_physical_evidence(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + preprocessing = tmp_path / "preprocessing.npz" + model.write_bytes(b"model") + preprocessing.write_bytes(b"preprocessing") + payload = _evidence(model, preprocessing) + cast(dict[str, object], payload["device"])["simulator"] = True + evidence = tmp_path / "evidence.json" + evidence.write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="physical-iPhone"): + validate_ios_evidence(evidence, model_path=model, preprocessing_path=preprocessing) + + +def test_rejects_unproven_ane_claim(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + preprocessing = tmp_path / "preprocessing.npz" + model.write_bytes(b"model") + preprocessing.write_bytes(b"preprocessing") + payload = _evidence(model, preprocessing) + payload["neuralEnginePlacement"] = "ANE" + evidence = tmp_path / "evidence.json" + evidence.write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="ANE placement"): + validate_ios_evidence(evidence, model_path=model, preprocessing_path=preprocessing) diff --git a/tests/test_portfolio_acceptance.py b/tests/test_portfolio_acceptance.py index f0099f1..30b340b 100644 --- a/tests/test_portfolio_acceptance.py +++ b/tests/test_portfolio_acceptance.py @@ -13,6 +13,10 @@ validate_ai_hub_qnn = cast(ValidateQnn, FUNCTIONS["validate_ai_hub_qnn"]) Validate16Kb = Callable[[Path, Path], dict[str, object]] validate_android_16kb_runtime = cast(Validate16Kb, FUNCTIONS["validate_android_16kb_runtime"]) +ValidateIOS = Callable[[Path], dict[str, object]] +validate_ios_coreml_implementation = cast( + ValidateIOS, FUNCTIONS["validate_ios_coreml_implementation"] +) def test_validates_tracked_ai_hub_qnn_evidence() -> None: @@ -75,3 +79,11 @@ def test_rejects_non_16kb_android_runtime_evidence(tmp_path: Path) -> None: evidence, root / "reports/android_16kb_emulator_reference_v0_1_7.md", ) + + +def test_validates_ios_coreml_implementation_contract() -> None: + root = Path(__file__).parents[1] + result = validate_ios_coreml_implementation(root) + assert result["status"] == "ci_build_and_simulator_test_configured" + assert result["backend"] == "CoreML" + assert result["physical_device_status"] == "evidence_pending" diff --git a/tests/test_release_evidence.py b/tests/test_release_evidence.py index 018d62d..8b11382 100644 --- a/tests/test_release_evidence.py +++ b/tests/test_release_evidence.py @@ -249,3 +249,26 @@ def test_release_bundle_retains_validated_qnn_artifacts(tmp_path: Path) -> None: assert manifest["qnn_evidence"]["status"] == "validated_qnn_npu" assert (tmp_path / "release/qnn/summary.json").is_file() assert (tmp_path / "release/qnn/artifacts/qnn_context.bin").is_file() + + +def test_release_bundle_retains_ios_simulator_acceptance(tmp_path: Path) -> None: + ios = tmp_path / "ios-evidence" + ios.mkdir() + for name in ( + "EdgeGenBench-ios-simulator-app.zip", + "ios-tests.xcresult.zip", + "xcode-version.txt", + "checksums.txt", + ): + (ios / name).write_bytes(name.encode()) + manifest_path = build_release_evidence( + *_inputs(tmp_path), + tmp_path / "release", + git_revision="abc123", + version="0.1.8", + ios_simulator_evidence=ios, + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["acceptance"]["ios_coreml_simulator_build_and_tests"] is True + assert manifest["ios_simulator_evidence"]["status"] == "validated_in_ci" + assert (tmp_path / "release/ios-simulator/ios-tests.xcresult.zip").is_file()