From b9fb1537a96b8015b65e14a3b98d7ffee3c5a715 Mon Sep 17 00:00:00 2001 From: triasha72 Date: Thu, 27 Aug 2026 17:55:26 -0400 Subject: [PATCH] Build end-to-end iOS Core ML acceptance --- .github/workflows/ci.yml | 83 ++++++++++- .gitignore | 11 ++ README.md | 9 +- artifacts/neural_surrogate/model.pt | Bin 0 -> 16881 bytes artifacts/neural_surrogate/preprocessing.npz | Bin 0 -> 1551 bytes docs/ios_device_validation.md | 18 +++ ios/EdgeGenBenchDemo/BenchmarkEvidence.swift | 139 ++++++++++++++++++ ios/EdgeGenBenchDemo/ContentView.swift | 39 ++++- ios/EdgeGenBenchDemo/SurrogatePredictor.swift | 13 +- .../BenchmarkStatisticsTests.swift | 36 +++++ ios/README.md | 92 +++++++----- ios/project.yml | 24 ++- reports/portfolio_acceptance.json | 8 + reports/portfolio_acceptance.md | 3 +- scripts/build_portfolio_acceptance.py | 40 ++++- scripts/build_release_evidence.py | 34 +++++ scripts/prepare_ios_resources.py | 41 ++++++ scripts/validate_ios_evidence.py | 132 +++++++++++++++++ src/edgegenbench/deployment/coreml_export.py | 30 +++- tests/neural/test_coreml_export.py | 2 + tests/test_ios_evidence.py | 83 +++++++++++ tests/test_portfolio_acceptance.py | 12 ++ tests/test_release_evidence.py | 23 +++ 23 files changed, 815 insertions(+), 57 deletions(-) create mode 100644 artifacts/neural_surrogate/model.pt create mode 100644 artifacts/neural_surrogate/preprocessing.npz create mode 100644 docs/ios_device_validation.md create mode 100644 ios/EdgeGenBenchDemo/BenchmarkEvidence.swift create mode 100644 ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift create mode 100644 scripts/prepare_ios_resources.py create mode 100644 scripts/validate_ios_evidence.py create mode 100644 tests/test_ios_evidence.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 744cbe3..cba96d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,80 @@ jobs: path: build/evidence/android if-no-files-found: error + ios: + name: iOS Core ML simulator + runs-on: macos-15 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install Core ML export dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -e ".[neural,coreml]" + + - name: Export current model and stage iOS resources + run: python scripts/prepare_ios_resources.py + + - name: Install XcodeGen + run: brew install xcodegen + + - name: Generate Xcode project + working-directory: ios + run: xcodegen generate + + - name: Build unsigned iOS simulator app + working-directory: ios + run: | + xcodebuild \ + -project EdgeGenBenchDemo.xcodeproj \ + -scheme EdgeGenBenchDemo \ + -sdk iphonesimulator \ + -destination 'generic/platform=iOS Simulator' \ + -derivedDataPath DerivedData \ + CODE_SIGNING_ALLOWED=NO \ + build + + - name: Run XCTest on an available iPhone simulator + working-directory: ios + run: | + DEVICE_ID="$(xcrun simctl list devices available -j | python -c \ + 'import json,sys; d=json.load(sys.stdin)["devices"]; print(next(x["udid"] for values in d.values() for x in values if x["name"].startswith("iPhone")))')" + xcodebuild \ + -project EdgeGenBenchDemo.xcodeproj \ + -scheme EdgeGenBenchDemo \ + -destination "platform=iOS Simulator,id=$DEVICE_ID" \ + -derivedDataPath DerivedData \ + -resultBundlePath ../build/ios-tests.xcresult \ + CODE_SIGNING_ALLOWED=NO \ + test + + - name: Package simulator acceptance evidence + run: | + mkdir -p build/ios-evidence + ditto -c -k --sequesterRsrc --keepParent \ + ios/DerivedData/Build/Products/Debug-iphonesimulator/EdgeGenBenchDemo.app \ + build/ios-evidence/EdgeGenBench-ios-simulator-app.zip + ditto -c -k --sequesterRsrc --keepParent \ + build/ios-tests.xcresult \ + build/ios-evidence/ios-tests.xcresult.zip + xcodebuild -version > build/ios-evidence/xcode-version.txt + shasum -a 256 build/ios-evidence/* > build/ios-evidence/checksums.txt + + - name: Upload iOS simulator evidence + uses: actions/upload-artifact@v4 + with: + name: ios-coreml-simulator-evidence + path: build/ios-evidence + if-no-files-found: error + test: name: Python 3.12 checks runs-on: ubuntu-latest @@ -150,7 +224,7 @@ jobs: release-acceptance: name: End-to-end release acceptance runs-on: ubuntu-latest - needs: [native, android, test] + needs: [native, android, ios, test] steps: - name: Check out repository @@ -174,6 +248,12 @@ jobs: name: android-verification-evidence path: build/input/android + - name: Download iOS simulator evidence + uses: actions/download-artifact@v4 + with: + name: ios-coreml-simulator-evidence + path: build/input/ios + - name: Validate and assemble release evidence run: | python scripts/build_release_evidence.py \ @@ -181,6 +261,7 @@ jobs: --fused build/input/native/fused.json \ --apk build/input/android/EdgeGenBench-0.1.7-device-evidence-debug.apk \ --alignment-report build/input/android/16kb-alignment.txt \ + --ios-simulator-evidence build/input/ios \ --output-dir build/release-evidence \ --git-revision "$GITHUB_SHA" \ --version 0.1.7 diff --git a/.gitignore b/.gitignore index 17e3fa5..204f590 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,12 @@ __pycache__/ # Generated data, models, and reports artifacts/ +!artifacts/ +artifacts/* +!artifacts/neural_surrogate/ +artifacts/neural_surrogate/* +!artifacts/neural_surrogate/model.pt +!artifacts/neural_surrogate/preprocessing.npz /models/ data/raw/* !data/raw/.gitkeep @@ -29,12 +35,17 @@ reports/* *.joblib *.pt *.pth +!artifacts/neural_surrogate/model.pt # Native and Android build outputs build/ android/.gradle/ android/local.properties android/**/build/ +ios/EdgeGenBenchDemo/Resources/ +ios/EdgeGenBenchDemo/.generated-coreml/ +ios/*.xcodeproj/ +ios/DerivedData/ .idea/ # Local backup files diff --git a/README.md b/README.md index c808c87..ad26751 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ The project combines: - mixed-precision INT8/FP32 static-QDQ deployment; - quantization calibration and drift analysis; - CPU and CoreML execution-provider benchmarking; -- native iOS 17 SwiftUI inference with an exported Core ML model contract; +- native iOS 17 SwiftUI/Core ML integration with CI simulator acceptance and a + physical-device evidence contract; - installable iPhone browser inference with ONNX Runtime Web; - repeated latency benchmarking; - reproducible testing, type checking, and continuous integration. @@ -42,8 +43,10 @@ proprietary aircraft-manufacturer data, software, or design information. The [browser demo](web/README.md) provides the usable iPhone path without Xcode: GitHub Pages serves an installable web app and inference runs locally in Safari. The separate [native app](ios/README.md) preserves the Core ML route, -but physical-device latency and energy remain unclaimed until a signed build is -measured on an iPhone. +automates current-model export, builds and tests on an unsigned iOS simulator +in CI, and exports validation-ready physical-iPhone evidence. Device latency, +ANE placement, and energy remain unclaimed until their respective evidence is +captured and validated. ## Native C++ and Android runtime diff --git a/artifacts/neural_surrogate/model.pt b/artifacts/neural_surrogate/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..082f5b680176cc4f0bf5713adc22f8a83407ca41 GIT binary patch literal 16881 zcmbun2Ut|gwlzwYAQ_RI1VlwL5_-)^5D`J5l2t&0polggA}FE+K}0f$BvCOVpqOA) zVOA7#&KLnPV;1E#+jH*S?)&b0=llP~S6?@a8e@*GT2;)dnzOy!1cmtcL`C`j$0fz5 zz_)NwL}aXSL|9^&(c%TM&Wd$s zajuYoLBNQ=2p7eN&5Ps;|AvS=$1Vy>oWXQg1bMIFi03;-2vlk@1lcPCT9B+>zsaxRQxn zDIczMfYcvy=f;F3aAl%-TB3!cf7>$U%0`R2*(Y-4{xXrpm3QOM;wt1u53`I)KMQWg>Ty57+7s zNt6C9$@&kTNn9HqPiP|7c9=)RO(vVm;R)j@Vt#W$A{TwQc7G_E^jnerA52zU2Og71 zBG>URrY!E{VMR{AS*9d%oqf1dmjsCMx<4jv@sdQ|BfOC7;wJvrL%nE8;=gZAa}x=W z;8lu_iHL}d8?KeWb)C(1GvIZdmYYoEU(ak}Sp2-m#02j2*<5#n0A*hF$b`h0g}jl$ zd&I*QL@tVo3RxJIkPxz99@is4j`#P4F$oF02QwrxE;4@JvXBK!qq&{|s>8tFHx@_6 zhb-nHBID-q-Yr=8R}HTKEnbb-n5f9`W#O@rA(8)LNeEdSzbKiNPJ zzs?e)_{RQgmN1H16wmvdAxV+(!=K05am@diEdDWf4F766Z0gY4`x(Q7dAR)Vrj+<> z{%tDZ_aqP!78^4!Zee6xqO+sre`4)_3YcUuEWmClw@-j)e7OAY0;KsI{v}}Uvc$+m z!}ExjTu zvUVrb2X13-28`xd?{q;cMvbbkN^Jjj^#iudt)-TBCFne~18sd((8*dF-aaUWo%6Iv z?Trp}b4{np8)p!{P$m2#Ie~gNE`{d4B5HCb2z3&tk)F@H;bW8{eR%RBxo(+9)7*k+ zcD)3K@n560v5I)L)evmGrKp!s9K76Kf*V86pu@YPFn-(%$XC-q3#Bz+uqOnyFK@tN zNm~qfzmg^=d`G{iJi4W2JpP!YPjjAXbVMocA_^**WJ9|i_O`{52RpT(dwvQ=JRJcB zzH7;>oL_A6Xk}oB0M_WeI6R$^O&oOe?c@gKsP|__TC<9fllzb0z0^?D z{alIBr6yGPZVr2MZz8#2c%L|&l82q=>+tKGA82*=IDJ8m(Mu~taO?dotboQc5MDYT zQlDM|*Qe)5V5uW@6Mh3bLOQ```Dbv_-U_A#m89#_OyW{uPPeF!vg^|rpcf~9U(NL+HO*&-cNm@@f0+dM~rnq|I$ zc^4PaO}eM>zDFA!_r8SPCgHBa7gSL}Ib2_-@#X-^#E6;8zTuw3$Os1;KPt&<7dX&E=s~E;Lr8F6la~$CrG({?G_#r!JrqZI@xqn;o3HFBg)RMdR$ghLzi%>@FvL z%a*Xt5yv3ZTMlPFZDv0zmeF+$Ux>ivb=1PH7nk@rqVM2DoIY(1@jd&Lxvg9S14iN< zFD`jd1^w$Z?ST{S*pW&d%X?tHyED0VNeregX`-P!L#f5mJJ_HqgG$f$;DgUj^ha1B zomw40)+wd2XIJZUsxyP=%Vz|om+YeH={5M&P809kRma0mP77Tr61Wtt98S{u{o^3`k|K`gy9Ab^s&sW*6aBhHp1hftj*GP- zS&`Z*Ml<{(F~~i`x+>+shJ!}Lw8xdoSf8eEL&n*Szp@Zzc8|jcmaFmmsZJWm8*743 z=TMRPcVK<-WU~BxK9v(#gWOfk@a(c0>TWco$-&FX(rvNSaMW&Uo#sR*-z>u32cM|B zXdULn-XZm!~uEg-lEQE=vYI?FJX zbrZjXCS$)aHqIe5GVL3&?#m$Z&ugf#T_w%@`hy%_W)1BAkr380lB%lDCl})@VW+q* zl{(}{TeU^(KDYf~4)oIuVrGZ{wujv!)=}>8(f$lRbL^y|9;F-`cm!d=`{=~BL8_sdL*=jb zK>Dpl#)`iko+LF8iI1<@CmF7^LQI#g4&H|vXHH>Ez)7NLcaM?YWlMIhu44HY|9}6xX=tr%+p!rS!ZD*Gdx43c~y3vO-x24kBlKr@>WGh@)p-x@S zFf?#wI!2aOVCddG%(e@9RML4r;q%j^dAD6rza^CFZaqzI-DGiW|1r3)KZ1VcFXTub zJxm2Ua;a%uJKf-xM{GH*bnLt1v>?BN&}L%{n(j&bn{*(jx{hQB&7u7Zk0KpBO|1ha z(E-zCIKyxhyJ6oADkIs=7OhCdz+*O;u~Ufd^;`(&1W$1y71xtkqd>f6P>L%H4lxgH zW!tYFFd(tlU9hJJJIH<$+yZ)r&dPxe3B_j@6;I zD|aTL|0e|ynSGVQ#;3MfB1_PWXKy#7-jVlwFNU@lVaA*Sknc{%Q%-B4?4~2tG}ULi zUmQjUp9&f<#=j%LClXEFRoJ#lK|Gc@1iy=o^0&?AE5T8%i0QU-`c)Et2@-q%Sz3)4@FSv?spEZJRo_&Rg zbOTne?gq80SlnOTLJNnKF;`^^{d)2ajF@+TSuCl~ay7={kpEYr@M9zi)}BvCS4rDl z@00E*Yb(OErO!!>4+AHRTXD~ly^OAr67Jn-gMksX)IT*JgBS0lS7hVhy*rY(S#q#X z%#)O*$U~gxCUCZNfa;F2c0+%i4$ULyQAu|@?K<6wIvo$GrA!r^A^|X>P?RtRo-rsYkL&lBt%iiwF8UbE*&!}Ya6ZFX~p!Z}Q>Gi!Os3bW^=Y6{jkEC452Cf7-rY{QvRsNVW zx`hM{UMD3XE|jdzAYJ|2QKG4eEZtzt?D%C)mKs&lp@&+SxO`$qOI^M51k4@s0 z)Fi6{bFOZo2ltC~7#$PF?4%>?-Rmb9pXUti-)CBd zQyf9e-yL_aV#z5-DZ42_wipt?hxK{Ou`Mv&mRTW3J5(94=o!GH3QaV+_y$muJx#9FDB`%6cj0)W4feX8B;H=aqq*W}E{S_&OWqp#!?SU> z7!}6p^$17(PesucCXg7JI?xm?P10E3d-b2K-CIecE^|9mU8R9H0f`(qF!Is~X zqzpOY$*OuXHfRC7i)bO`(fRFb3Rn_!`ZP#h&xfGy$?$V)3Yv{AZC@6e2dk|4(0s~S zCdHzXiHPIJwcRc#c^>iSJ$0;g;$W1J89sdVoDKixj)&rR66Xd2UPF&yw5T^8^qR+e z?}6thvSh7r6)dg_#QjbD_@+({Q-r3&Qk`seq0=vT!pP&t!?|qP0}-k_LjqMc8vy^C zIT&0u6{mRe;e!n-DBco|Keh_s1rcK^;rR*Lx|YDPJ9ik%p@~>tTLC*~TcgzWK=kX( zhhKSaFnONS~4vY*WQnGTo4Rj8-> z7)&Z~W16E%;Nh>wU=hMcBls>b9|l?KBcIE9HBO@9oE@;VPLC69s{squ7SJ(M*I*hm z046;kK=xjZ-om{@>T$r)jQw` z<+^8T?N{@kAm8S)-DXM#Afr59Lv;{(umd=8zh zGM>4TlFdp!ib0hJ>qzs5*)UHuom`0ZfK{=1j3B3rJbpZz9US+A33{)|NiNU@`^)l- zgZ~qfG8oQIc)1ElY6a1cy~&hT>?23Cv~a(NHNLdIP2NwH!bHnf3@b7~;hlc)TsfPK zcd{hu)25)8-B=C?E5ZWkfw`aB$SzSG5P$1MUUXGLZRS>Xt@35g>;w@!xI=?fe$K=v zM-PLSa%{WH*ABR+A&s7e$;{;Ezu<@BRR~CWPqZ(q)6($uJYV!1d=#&MG-H8VC znH>9WPd)wKkg_$ip!mp6cKn7GXdXQQ=KU1JeLFHR@6{%VHS;99SBle8{$%Km8iTQ) zC`oRe1%q;>xLWuKlrCHk3h^zl_*V;gcDV;$>l}v-RjN1*vY6?jld!ug0hU)3GWTDu zWIk^H0ks1KoZENAahACjO^C(a2oEMyq+p##sOAiOj3PA5` zPrFUQY9f+*jS;Wrl3RCYVt&_h;&4zL!z~^_R(U><1I8r1DWv^Wo)%s=K1IeXzQUW& zqfuPI9!<;Y+O8Urw!;VHY2}4Pc6whZbNQ7GbZlqXTxm_poIS(V*4<_!GM|H2%qyap zAq32uZkR8XMS7m+K=AS#jJ}vUK1}|^Z16t^KThvM#wQ6L@r8rc`a%rz9R;RGvhk^s z0}358LN(`^Y(`2HOHOWMxO$^-`nfM~BWe-2YK*32C=t6Z7c(l27f5)804nNtftip3 zX0;7Lw)uN_ZsEZuXo!<&jx(ci3#j;$MD}cj8OlY+Fp2i|czmY-MroWO`)7ZLyQ6NA z=#dLB!J!hbJDg#FkCgYTQqoSS&rv>%>zj#^vTqsg%vwBHB!TX7 zjYQ4Tl=MH9!h`YK@I{3TAszQgeeEaasF*u`Nh^UFRuLqFKM^j71>)R9N2bv1J<~d< z4h155}_(yH*3S$+FGu?VRkGVwq}g?7PL$p*%4*?ID-<0%+Di-VYt zcgec|OA@eEgmtb9rEA;x!0G5+(rDxerCmQ5wVcI_+|P{==69{lQPl&>%jH02@h|q7 z;ds1~`IuZ4;760ikD2O?6=cDf`%J-$cQ9DJ5ZYW@;p;ge=INI@X5RK^&}&)^0~;0K zbGZk4A2Xl>xi86jV>gr!mBQNs>zTJr3*f-X_w1u!E+l5fklRlyz$Io9S?ePO9!jp% zy0ae6sT=`%a}K^}*B}Syj;4nzo5-zbAsVo~7)CZMrwcMO=t$Cn;mT#?sn`({`wCv07ENZp zdqNB~GC{|AB)XldA%{e6z`BpRING+YO{yXg>#H`i7bZlanyfL5t{G1j2IZ3$%SVu@ z$q%BNN0YK3KXj7V!rbc;!Lz~_QIRVQ%O>t-IK|m;x?v`YIoaW?*Y`m;c{Yx|tqE5S zHZz(10r>X57@M%W8guoMnGvOzK+`D;v=gGR%RihxxGsP`AJ($js@hmodIes3n(}yJ9O4ik=qJg0?UYI`;CQh12o6aioeE)}V?EG4=ed-E3_?02BtAVNXeMg3l zsuSO?qwsR}LsB|lhNF;Z1?R+_U}ozST^#F)puwk2^>wA-{1tCJD)1Nv?->F&UIJEEgriKO4%LcTO)PB}K+^3fs%ku) zssGi|e(Sp$s4n~lN8i1GeRGZA`E4sGztRL&OU1G3iGO=pqbyF>h-jbVnhHC7GeLBM z2+^OhgShA%VrOQ1;u?u@(3O`<2KJ1g6W8eAqW)UO@z@a4DyNE{O}!v1;S1Bh{s$BN z;!vC4wDVxy+Q)Dkwh*VABar{pa-y|lJdyXWBPD$`WY&2LOlvn_yU!A)c})b0r~G0r zl=1ePD~iBhR|maw0eriqq4;_XXr?%!yQnS(SQWu3mmc^LDuEf*?U0=M6xMO?!eEm; zQ6Kci2j92Co^P+vDUTIT=CHaqv#4)^2AJP+!GjqG z;g)4CQT3CCCWf~k{X~#FlQTi-TtTW^Spu3a=U{@rGP_{@dnoTKArIaw;)j3(@X+rC zm`-$uQR9?x;XDz1KUIS$&bfy9l0o=-eg@%OlEw3-K2Rd7t|< zwvX=|`MGrvUMl=z*G$+!rJM`kvgH~Y61NlB$Q6V>i(hA&Qm%$vN z!?1e=CGVec;mpPeD3;g4dYL*Bw7ZzhjV!^OK1=MM+{*DADkXab%87L6NRpJUO~&N3 zfX?1LAg&#x*={mkKRm?V`1S)XH3(BD_e4Q)aRp;UK*&Uj3mdW zOvcjO3QXA8h>s>HK;GnaY_GHoCQW%yCVrMCpCx$yT{Uk^_bC#KPh&~+!~mSWMw2{B z{!VUOvTT==u|P3!!hM3;w0r({rec8v&Uwg>yZfGi_B|EMeymHn4sua;)_5i@SRZYs zwnB7c7rXe7Fe*iiK;KojVEoHYsQa-4R$tl-T9-e9Z@vVsH&TfvJ=>B%kt4p!| z(h7V%cOqF-^pdw`n+fjVPo9_P(OWLR7~AA4?An&AAW)vcPB^bkv-PrwebGZkF4YhN zq+2+T-`c~p)%|R(I}74dn_*It9$d^)*M(b$92l=Jh~=9H8MS2_QGR_7C`bGvi8nQ2vPK|^g(kpB zmvj)Gc?#UMreV+_NmNaiM3Yx*IFsM0!k~u)y|$TyW~Ro-EmVhd>z*+Co?K&<=jfr- zNq_9ab>LGr5?NHjjhmG4-BSm8!o(Jj9BhG(a7CI{qer@Dt;23x0rXtO!7ZmE&^Bun z@#%NP9p19^yUTgTQ8W^N?Nh-|{x@KmyCe=OII>e3#uMYw@+cb+iIT@RLPqBdd{wK7 z^1hWscvUQhta;3^wv|lww>sR`T#UZ2IC%^|Iz-^cIbe!QiK_NkY}ZO*#@8vJpzZ}CJD^O>zI%iHQWNaf^~a9` zUzwY!9Z(fA0=g^i64OvEPQi~RW>@NTJnXuGoNJpwrAK#y$Lb=uqf`qggrZQ-_6i|! zPNZ4+IK#aAz^Y~Xl7LJ_>ZNstJdU|cm)|}NFKm0jJve|Aubqug&1Kn7IT9U@T^+s7)d6G?|g$g~>5;Btu04<7~!Zz<@1%G*XrJ zy069V6((S|Xg!$rzl1MC8gRZ|93!@WY70Dk5TqvSQl3Kia$SjXOtz5IYbW4wD}9Wc zKZ7g|U59V-H{oJqKC;8f1+;xbNycP%*kW0TAE)x6XSW&6d+raOuV>>fZ6Uhy;2eC} zuZ$UI4CzZDAF3{6N3^WQLQi-)bG>H|IQj*W(aZ=8<8bhC&`o$}oJDKIw}NrqV_*xC zs9a4XSPG?Lms$oB)LPGJQ4qj)?W5?PHeEboa~$yF_4YE)*-ZDC<>>ze(Q~&uwRN9J zG+rXq&0UF!D=&bt(Jp#^U@YZRd&eFOpG;K>C{;NL#5r~k={T{0+!CAm7FeQ#Nd( zTc50ju#kGv1oNo(f_jJu+(6|jF2Kr(<#1i@CiojVQ^MQR{x!>nw9QV0H=dV?F#k2Q zp5FmQR>ke7E`MOlQY7)+NJsb_q(^0=YuR^4D`~C&X^?+x$yn{)z$kCgM~yGDiT4R% z`e|AQS#wMfR~9~p&+a<-e(gl8mY)o66*4sNWHd0QKba5xA3){RMmXEHxvgVIK3AUp>NQaKm`M~xLt)*` z3j8@@u-(+j77X~tGLZEW2ICjtQ1c~7ud$(OpoRjUbwDz*lNlF$kfcAJgxc?BLrwl( zR^p%x1fQHmJ{=XrP~AtYuv#{%%S}QTmn_yz)d?rK%q1J=UIO_O>tK1d0`4m1`R(pm z9G~_ASoc_+&f1yGeE-zIzMH5)G)6vz9y@}WuwerlVZC6=?FTqUJfe;8f z^adK1%qCmTnA23#kMQxB4=}eXnM72?uz>_ppsd-}ty9Gsn6JYHJL-vcS{zPjos5@_ zl_5v29L;oQz;ll%ret(G2-q0n_gVJr62n%q^2!v*;gcdEibo-%ARLWnjKk*nu9*44 zoOuyG13Yf1!Zh$@PJ~F{;EWNZuI?H%f3d>p+h>wAzHG>K(7}xPvehWxOTl{`cO1|2 zM}o(Pp#6so67oTx{aU04;cxeHX5NxTLEl|q(_xG7zLFDmRe@37Qw2WvGO>_Zi0P;H zGeOH=u)MQkGN5Zp9KJ-;o9Aud{`V#_X@Vs_)RMvt=1CAMdjm9jwD9AYOkyB!hli)1 zB=QHuFu3^=qp2!P1jN^p!yXmDK3j+Z9RkG5c@1=U{a{}lj|N#+H=L?n!)(k7#HR=S z@o5vP<0vUQ`g;rU(b&zTl`KU$$Gecx-~}mx(zMRL2lgoKhaYnk zNR8Ywu)Q@JvqPkbzk?*pxw9EMmry3=iwE4ik^+v;ud#bdC&IqNb>xPg0-k;<*4{Ys zCk#&QhUJ8S?lo1Mx}cZ2da;Lz>9fWK-%f#w$sw>kTR}ct5yslZ3hYt8P4IRShm1Wj zk-Xk)froE~;j8mDIBI+<=U2QLgq++;tF}xfU-#=Fzf~@Cwr&P-S;^ZwDcMIZ@!Mlq z0fU=8%&1DY95!ru$#kqKC1=kRqxB&bbnXg-il;Tq(aa1iA8}4N<7H%WWeOgc}QiF{z%VE@%y`bG9 zf!ld|lqKI+Vc67aP`Tg6xCaSw-j>N@WZ^Tia#6=oq340={LobGxmY{-dF9rtwmAV z)(Lj@?FVtW2jtn2t#CB`E;vTa$H6iAXzLk<;X;=8%oB2V=Ck%-%|Xb z^ocorW-HXCjYr3>6VNkZKE_;kM$v1|cxk2`aavYN-uWspfmOkn{d_E2pka)Ag!bUo z59Q>Y#RV`KJO>We!R(E|OC-@+k1(%9Ny)NH?bB6y$tx{+=IGm*BfR<|@As45CLdz(B^M0VkA;>1E}7T97p{agldW}6$@e?$ zOtnEisVREF{77nt9a{0^;+B0xuuvB)R`rm@L6@NLJeQ1!p9h-;dwFLS&E)8YFeX@0 z0k$;n2Q`(OU|-bVZZ+*Rdwv?1Nvl>Sp>cVfRL0aq`w-Hy>5WZ-86h7pnv3$ zKSGtY{DNB+RQ*&8@ru%KDKaXJTrZ+3mT1OSpO3v@NO>)zGYIlO-U7( zdUQj9T{@gu(FQY)-Qw5_s$g}935s>{!^fN{IN`MhoID!Kv^O7R*S)r9l!9-A_qb71 zfPD-@+shc~yM!s-Ckhd9v(f4^%e?=Y&Nwx67J8dgnwWcWCoj~#n?PjntuSI zTKZX$Pq$g|hRiGlOrcDIQMS%0d{!NCLR)qok=w?;&j@f6tGiF!hO-|MED5X_yo6!*OEfa9?f(e_mdSj_T9wS6*-d}}dv2TShk^qgpIz zgH>qs^EoT+exI3{S%~*lZh%r;7b#VL4GIoYRQpB*Q*+=YNS7myck^bAlqlR1%^{1M zw=;9jsX=A`XgIX&1)0BJ7W8(h(r@(|`U1^(yt*c%@q=z>$;%~_5`(U zs!X$hVjH_I98Ap)gWc8^Hh0}4$XU7pdQPR2XYrx5=*LLpP3)1L2j5@RnP|^d}h*S`s_d%TJ=c6PBGTZc38S9)1 zI2&`Hw@1@RhRz6K-{ETLs&-)Avz_V76J2ER^+kxPJO}2}f5M&bA~-|;2x&4@V(-Wi zR?2i0EPXGJ>$mu=zbGUG*L@blh0mu`!AVDGq~0}RS|EYm zAGR_Y9rqb#Z!B{|)=&dPGWwu0!Z5MY;zL!ZNjTxWJ1?EKQci(@K4CQ8S78q2z=go|I9$ohkS-?mqig zQ;JSJ=!+Ex6sfYwP8_px1ZQz)64ke>BndT3*^I2sc;Jp4MAk~9P;Ng2>aJn)3MY}u z$KJs1=A8j|`@rJZ`KaR;MB@}yiQ|bIWY(&CoI~pmampH3V$}VK==pXMb-!RtPq>+| z)k+mmsIN^&zp+9G*(_2uxsLP>m}A)bAdHie#sj&F@x(k=meZt#<z9EY2@Z_ibLj^SO z-WY0XWP?MBW6;N;82XlIVhS&&GrBnkme@|m@B8z}<*VCiuV5W~79WG>@C4*uh(VE? z)7xv#@?*s*9a<*s4aPS{lXqcJWKHmTyrZ=eehFm}R#b|5|J0=Ao4wKhmlc)#2c+OT1C1PHa3kx0rmLG(MY(BcGyAhN|vP5_PEfS>@ zz{weCVSAe$$?}D-N#@AYFkyuS+HtkW_BR7$eZVMo#T0eAcu0<3wBVBE$xmU)UUB+C zxrhDmtDV^G)u-V@I>gKOFwvUf54WnaP)B(K%xuWV`8(EvSDOq=w4?FLx_&bI(;a*! zl#V|;t?5$ENPO!y5k8!jpdO;q48LC#UfF3!W+YF*W`}V^s-c~CrdtPrlB?;JP8FPx ztB34hEjCHWK!Lj&S^rZET5mlh!G*rCc6SIGR^*ZQZ$+sTRiSsvo`Lo;38uqhJf<#w z!Z_G#(n}_?Aaqj=yV@?mom>m>lmAJaM&BV$sn6m14t}cdCm?Y-rIL3aRI%H89yvbiE*TZtN~F{>SmCv{ z#Ji*fCJT2H-E*tRMj87NTz z_IOZb??iNp@I?8D2;BNehrSZN4nN;7XN>pmg^4P2*<*Vre^I6FUclr4u|3Q$I~z* zL6RoUU4j-O;`mcFiN0LHaIVdF!U3^1wEZehM{MB7T~Wb!F>ej3eV>5*o6YI|%gNX* znvI#$wzT`tzeVDrn&Il1Xgs!KFEl9^L2AVpa%Tn5^|hXzABEJ zgvIPmn&ptHEt7GN-Do;pOaTvGc*t@eI#T_mXULb5`y^eJpNdCSg36dmA{*QT75vuJ z(q|*<`mK-2nYjTMZM{VNTsNUr*d6H0RmV#bcfjFJEe!Yvk?TEMVRRfHuKQ$*+D6ad z$-$1l{GS&?nvQ(K8K}JSANvoMzx{`fo1S2mpoBZ*DfsR;W*;8?#PA}YsMxU-m{QzB zTJHiia4q84Y4g*AZ<655!*guHhNGalW<1W?B=}eSrYn}Lvger}F8{IpNx$tkYuBUl zmK(6rtCo3_;z??oBCyxe7ivRakOeZ&$b3CpyZ%G&INEn19ne1k_mV~FAtr@7pBPWO z46o3J^m05^x_}1oZNoW3rF8H>FS*`vh1{&#O*DjSq4k9YaXKH)%gH%IX4;Ryk->tL z?VU>;`mC_HW&>oNy+~Ge#^L;jQnc(v2>r24m-b4(A@ULTv0(ow_|X1?CEjx6#e)gd z;Ick$RXGPqLlfxc$tAd|WEy=KZ-=kt{#BC2@C1nr7!WgP{I=(R!}4#=88 z`utdS|BwbgY|ue{5aC$v-a=(P?m_aYX8O|E8w~p6~yE{-GM5#lI{W4X0!qCq%}_@G>A%c)6Xt{Lz>QXGfF&-0;5= z5KS$uEUZm!Ow26IEKIE?nNG5@vND-uV{U3~ZewO)VQN0f+|0sclGU)v)YZ#t40vrZ zT)f=)hXwyv=J?;UMTZlq|GfUpBmXD%@9CeT{s8Lpa?^if|B+q#Pl~@Ma1M9Ze_uua zP4PdnM*qq3_Y_R&|EA!*lN0{8kpD{k{3rPDQS>AK0GIz4_^)m=`6u%4@#B9YWq2v{ z|J`=RykyxwTmA1V$3Kx?vVS1W{)znin(a?yq3It;^M4}$K3o5Z^fLPc$;+($Q|;g9 z**}qmwtpZk|B3wj2>%l)!}$X_>7U5IkKsR&mzh71yeIrmwSOOzl7IB40sI^JuMz)G z=-&s5+8@y3|APL%4`DAi5fQ_GAv6Rh{ulP&L+anJ{=?sSz`LyR_bU%K{EZ2G1rEP| X=gow?Au2qaOUx(5`x&nPyYBx3_juNw literal 0 HcmV?d00001 diff --git a/artifacts/neural_surrogate/preprocessing.npz b/artifacts/neural_surrogate/preprocessing.npz new file mode 100644 index 0000000000000000000000000000000000000000..42991d12a28ca3951d3c33630b7154049359c7c7 GIT binary patch literal 1551 zcmWIWW@gc4U|`??VnqfsZi6fTp@5G;gdsVxBsD$1C^NNKFR!4IkwJjr0V7Z$5G;pK z40HQ7avd@dXghy(`3bRBpQE00?oQE@o#W-RZi~RI30Imr{2y{xA7Wi(@wJ`*-TvKw zc3$6Ud3CzznGX}hCU+L;KUug^Yh}xunmLQ)qUCk#*xcU=1=)YKQuuk+FL!E1*V8Cz zh7U(P&af2O3OXBJZc&b!celeMrc!;w`O@Ro5qpeV&+mOR@wxDGo;UTPKc+?8dGg@* z{CPV*=USfrcT#Jrh4Opl@6(>8?qv)>@uq;Vr!88yc)`4B@gYG|W{A&^pE8L{sNDOE zrXOpUc)E(%5~iP0X;!nWxVCbMFszgQE;aYt`7@=J`x~!5Jt>tYet*ByH`D9dZx~RW zm%aSf1(@>$k(^hYoS2hJJNN0WK4x-U_sr+Hf8)1UtXyGWxm#~ej&`t2GCP|4v?B_h z!`#OYbYDqgQF>|#jYCf_esXx;oXO31ySOgB*`gvDdwXj{Z2FZoMNDX}>(8E{0dpNF za-puHap0|c^kDkB?f3P{{!Uq!b$juyvrDggo`34wx1Jfzd5_F?e}XxW8}2-K{%T-` zq_;P){1tQ>l)rky_bYAJ__E}IqfK@2o-YqHub8cM{jKn^^M~la4bn3bM0$?4F1&j^ zCvEa^qqsHKY|E0&Cu&a4GJKjO*6*(LNGbn|qx#mIl4Utb^WO9>+Hq5Dtx~-~XPe;u z&;Qr1mq>0aP?*xvHQ~VaojnQW_9X2hyYFF6oBLLrcIs}w`l5W-oO}6qrWIdq+`iftm`NF#beU00 zBTzX5Ds%Xt;Q^u=z-16>*@Uhal*K_=7nF?!K$@VSf$=$zg`VNjb%SysOt&CRE0Atr zYy;X2(+y8j=-NRU1g4!IMY|dk11v$p3LA9Ip!5sVjOy|u2+cUH1|=_0+5x2_ZWODN bfg+H+08VTH-mGjOc{U)-1$wH71;hgYM6fUV literal 0 HcmV?d00001 diff --git a/docs/ios_device_validation.md b/docs/ios_device_validation.md new file mode 100644 index 0000000..361ce18 --- /dev/null +++ b/docs/ios_device_validation.md @@ -0,0 +1,18 @@ +# iPhone evidence acceptance + +EdgeGenBench separates three iOS proof levels: + +| Level | What it proves | What it does not prove | +|---|---|---| +| CI simulator build + XCTest | Core ML resources compile, the Swift app links, and evidence contracts pass tests | Physical-device performance, ANE placement, power | +| Validated physical-iPhone JSON | Current-model Core ML app latency, device/OS identity, thermal boundary, deterministic repeated output | ANE placement or calibrated power | +| Retained Instruments capture | Only the metrics and placement visible in the named Instruments templates | Claims outside that measured boundary | + +Use [`ios/README.md`](../ios/README.md) for the physical run. The evidence +validator compares the hashes embedded by the Core ML exporter with the tracked +checkpoint and preprocessing artifacts. A screenshot alone is supporting +visual evidence and cannot replace the JSON export. + +Do not publish an Apple Neural Engine or energy-saving claim merely because the +app requests `MLComputeUnits.all`. Core ML remains free to choose an available +compute unit, and device power requires an appropriate named measurement tool. diff --git a/ios/EdgeGenBenchDemo/BenchmarkEvidence.swift b/ios/EdgeGenBenchDemo/BenchmarkEvidence.swift new file mode 100644 index 0000000..54d0120 --- /dev/null +++ b/ios/EdgeGenBenchDemo/BenchmarkEvidence.swift @@ -0,0 +1,139 @@ +import Foundation +import UIKit + +struct LatencySummary: Codable { + let coldMs: Double + let warmMeanMs: Double + let warmP95Ms: Double + let warmRuns: Int +} + +struct IOSDeviceIdentity: Codable { + let model: String + let systemName: String + let systemVersion: String + let simulator: Bool +} + +struct IOSBenchmarkEvidence: Codable { + let schemaVersion: String + let capturedAtUTC: String + let appVersion: String + let backend: String + let requestedComputeUnits: String + let neuralEnginePlacement: String + let powerMeasurement: String + let thermalStateBefore: String + let thermalStateAfter: String + let lowPowerMode: Bool + let sourceModelSha256: String + let preprocessingSha256: String + let contractSha256: String + let device: IOSDeviceIdentity + let latency: LatencySummary + let outputMaxAbsDrift: Double + let outputs: [PredictionValue] +} + +struct PredictionValue: Codable { + let name: String + let value: Double +} + +enum BenchmarkStatistics { + static func mean(_ values: [Double]) -> Double { + values.reduce(0, +) / Double(values.count) + } + + static func percentile95(_ values: [Double]) -> Double { + let sorted = values.sorted() + let index = min(sorted.count - 1, Int(ceil(Double(sorted.count) * 0.95)) - 1) + return sorted[index] + } +} + +enum IOSBenchmarkRunner { + static func run(numericValues: [Double], category: String, warmRuns: Int = 100) throws -> IOSBenchmarkEvidence { + precondition(warmRuns > 0) + let thermalBefore = ProcessInfo.processInfo.thermalState.label + let coldStart = DispatchTime.now().uptimeNanoseconds + let predictor = try SurrogatePredictor() + let coldOutput = try predictor.predict(numericValues: numericValues, category: category) + let coldEnd = DispatchTime.now().uptimeNanoseconds + + var latencies = [Double]() + var maxDrift = 0.0 + for _ in 0.. String { + if let simulatedModel = ProcessInfo.processInfo.environment["SIMULATOR_MODEL_IDENTIFIER"] { + return simulatedModel + } + var systemInfo = utsname() + uname(&systemInfo) + return withUnsafePointer(to: &systemInfo.machine) { + $0.withMemoryRebound(to: CChar.self, capacity: 1) { String(cString: $0) } + } + } +} + +extension ProcessInfo.ThermalState { + var label: String { + switch self { + case .nominal: return "nominal" + case .fair: return "fair" + case .serious: return "serious" + case .critical: return "critical" + @unknown default: return "unknown" + } + } +} + +extension IOSBenchmarkEvidence { + func writeTemporaryJSON() throws -> URL { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let url = FileManager.default.temporaryDirectory.appendingPathComponent("EdgeGenBench-iOS-evidence.json") + try encoder.encode(self).write(to: url, options: .atomic) + return url + } +} diff --git a/ios/EdgeGenBenchDemo/ContentView.swift b/ios/EdgeGenBenchDemo/ContentView.swift index 777e917..321e12a 100644 --- a/ios/EdgeGenBenchDemo/ContentView.swift +++ b/ios/EdgeGenBenchDemo/ContentView.swift @@ -9,7 +9,10 @@ struct ContentView: View { @State private var values = [4.0, 250.0, 180.0, 300.0, 0.65, 0.5] @State private var category = "battery_electric" @State private var predictions: [Prediction] = [] - @State private var message = "Export and add the model resources, then run a design point." + @State private var message = "Run the bundled Core ML model and capture cold + warm evidence." + @State private var evidence: IOSBenchmarkEvidence? + @State private var evidenceURL: URL? + @State private var isRunning = false var body: some View { NavigationStack { @@ -21,27 +24,49 @@ struct ContentView: View { } TextField("propulsion_architecture", text: $category) } - Section { Button("Run on device", action: runPrediction) } + Section { + Button(isRunning ? "Benchmarking…" : "Run cold + warm benchmark", action: runBenchmark) + .disabled(isRunning) + if let evidenceURL { + ShareLink(item: evidenceURL) { + Label("Export evidence JSON", systemImage: "square.and.arrow.up") + } + } + } Section("Result") { Text(message) ForEach(predictions) { prediction in LabeledContent(prediction.name, value: prediction.value.formatted(.number.precision(.fractionLength(3)))) } + if let evidence { + LabeledContent("Backend", value: evidence.backend) + LabeledContent("Cold", value: "\(evidence.latency.coldMs.formatted(.number.precision(.fractionLength(3)))) ms") + LabeledContent("Warm mean", value: "\(evidence.latency.warmMeanMs.formatted(.number.precision(.fractionLength(3)))) ms") + LabeledContent("Warm p95", value: "\(evidence.latency.warmP95Ms.formatted(.number.precision(.fractionLength(3)))) ms") + Text("ANE placement and power are not inferred; use Instruments for those claims.") + .font(.footnote) + .foregroundStyle(.secondary) + } } } .navigationTitle("EdgeGenBench") } } - private func runPrediction() { + private func runBenchmark() { + isRunning = true do { - let predictor = try SurrogatePredictor() - predictions = try predictor.predict(numericValues: values, category: category) - message = "Inference completed with Core ML." + let result = try IOSBenchmarkRunner.run(numericValues: values, category: category) + evidence = result + predictions = result.outputs.map { Prediction(name: $0.name, value: $0.value) } + evidenceURL = try result.writeTemporaryJSON() + message = "Core ML benchmark completed (1 cold + \(result.latency.warmRuns) warm runs)." } catch { predictions = [] + evidence = nil + evidenceURL = nil message = error.localizedDescription } + isRunning = false } } - diff --git a/ios/EdgeGenBenchDemo/SurrogatePredictor.swift b/ios/EdgeGenBenchDemo/SurrogatePredictor.swift index 9a72a31..5aee0ec 100644 --- a/ios/EdgeGenBenchDemo/SurrogatePredictor.swift +++ b/ios/EdgeGenBenchDemo/SurrogatePredictor.swift @@ -1,8 +1,11 @@ import CoreML +import CryptoKit import Foundation struct ModelContract: Decodable { let schemaVersion: String + let sourceModelSha256: String + let preprocessingSha256: String let inputName: String let outputName: String let numericFeatures: [String] @@ -39,13 +42,16 @@ enum SurrogateError: LocalizedError { final class SurrogatePredictor { let contract: ModelContract + let contractSHA256: String private let model: MLModel init(bundle: Bundle = .main) throws { guard let contractURL = bundle.url(forResource: "ModelContract", withExtension: "json") else { throw SurrogateError.missingResource("ModelContract.json") } - contract = try JSONDecoder().decode(ModelContract.self, from: Data(contentsOf: contractURL)) + let contractData = try Data(contentsOf: contractURL) + contract = try JSONDecoder().decode(ModelContract.self, from: contractData) + contractSHA256 = SHA256.hash(data: contractData).map { String(format: "%02x", $0) }.joined() guard contract.featureMean.count + contract.categories.count == contract.inputDimension, contract.featureScale.count == contract.featureMean.count, contract.targets.count == contract.outputDimension, @@ -56,7 +62,9 @@ final class SurrogatePredictor { guard let modelURL = bundle.url(forResource: "NeuralSurrogate", withExtension: "mlmodelc") else { throw SurrogateError.missingResource("NeuralSurrogate.mlpackage") } - model = try MLModel(contentsOf: modelURL) + let configuration = MLModelConfiguration() + configuration.computeUnits = .all + model = try MLModel(contentsOf: modelURL, configuration: configuration) guard model.modelDescription.inputDescriptionsByName[contract.inputName] != nil, model.modelDescription.outputDescriptionsByName[contract.outputName] != nil else { throw SurrogateError.invalidContract("Core ML feature names do not agree") @@ -86,4 +94,3 @@ final class SurrogatePredictor { } } } - diff --git a/ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift b/ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift new file mode 100644 index 0000000..e383181 --- /dev/null +++ b/ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift @@ -0,0 +1,36 @@ +import XCTest +@testable import EdgeGenBenchDemo + +final class BenchmarkStatisticsTests: XCTestCase { + func testMeanAndNearestRankP95() { + let values = Array(1...100).map(Double.init) + XCTAssertEqual(BenchmarkStatistics.mean(values), 50.5) + XCTAssertEqual(BenchmarkStatistics.percentile95(values), 95.0) + } + + func testEvidenceRoundTripsAsJSON() throws { + let evidence = IOSBenchmarkEvidence( + schemaVersion: "1.0", + capturedAtUTC: "2026-08-27T00:00:00Z", + appVersion: "0.1.0", + backend: "CoreML", + requestedComputeUnits: "all", + neuralEnginePlacement: "not_measured", + powerMeasurement: "not_measured", + thermalStateBefore: "nominal", + thermalStateAfter: "nominal", + lowPowerMode: false, + sourceModelSha256: String(repeating: "a", count: 64), + preprocessingSha256: String(repeating: "b", count: 64), + contractSha256: String(repeating: "c", count: 64), + device: IOSDeviceIdentity(model: "iPhone", systemName: "iOS", systemVersion: "17.0", simulator: true), + latency: LatencySummary(coldMs: 2, warmMeanMs: 1, warmP95Ms: 1.2, warmRuns: 100), + outputMaxAbsDrift: 0, + outputs: [PredictionValue(name: "mass", value: 1)] + ) + let data = try JSONEncoder().encode(evidence) + let decoded = try JSONDecoder().decode(IOSBenchmarkEvidence.self, from: data) + XCTAssertEqual(decoded.latency.warmRuns, 100) + XCTAssertEqual(decoded.backend, "CoreML") + } +} diff --git a/ios/README.md b/ios/README.md index 35680db..e86bc81 100644 --- a/ios/README.md +++ b/ios/README.md @@ -1,48 +1,68 @@ -# Native iOS/Core ML demo +# Native iOS/Core ML runtime -This is the optional native route. The supported no-Xcode deliverable is the -installable [browser app](../web/README.md); it is the quickest way to use the -project on an iPhone without an Apple Developer membership or App Store review. +This SwiftUI application exports the repository's current neural checkpoint to +a Core ML FP16 ML Program, applies the same preprocessing contract as Python, +and records one cold plus 100 warm application-level inference runs. The app +exports evidence JSON containing model provenance, device and OS identity, +thermal state, latency, and repeated-output drift. -This SwiftUI app runs the compact neural surrogate through Core ML on an iOS -17 device. Python owns the trained model and preprocessing statistics; the -exporter writes both the FP16 ML Program and a JSON contract so Swift applies -the same feature normalization, category encoding, and target inverse scaling. +The unsigned simulator build and XCTest suite run in CI. Simulator results +prove integration and compatibility only; they are not physical-iPhone +performance, Apple Neural Engine placement, energy, or power evidence. -On macOS, export the resources: +## Generate the app -```bash -python -m pip install -e '.[neural,coreml]' -python scripts/export_coreml.py \ - --model artifacts/neural/neural_surrogate.pt \ - --preprocessing artifacts/neural/preprocessing.npz -cp -R artifacts/coreml/NeuralSurrogate.mlpackage ios/EdgeGenBenchDemo/ -cp artifacts/coreml/ModelContract.json ios/EdgeGenBenchDemo/ -``` - -Generate and open the Xcode project: +From the repository root on macOS: ```bash +python -m pip install -e '.[neural,coreml]' +python scripts/prepare_ios_resources.py brew install xcodegen cd ios xcodegen generate open EdgeGenBenchDemo.xcodeproj ``` -Select the two generated resources in Xcode and confirm that -`EdgeGenBenchDemo` appears under Target Membership. Choose a development team, -run on a physical iPhone, and use Instruments or MetricKit for device latency -and energy evidence. The repository supplies the native integration, but does -not claim a physical-device result without a signed build and captured run. - -## Delivery choices - -- **Installable browser app:** publish the repository's `web/` app with GitHub - Pages and add it to the iPhone Home Screen. This route is implemented and - needs neither Xcode nor Apple signing. -- **Cloud-built native app:** use Expo EAS or a hosted macOS CI runner to build - and sign a native package. This avoids local Xcode but still needs Apple - credentials and does not remove TestFlight/App Store requirements. -- **Local native app:** use this SwiftUI/Core ML target with Xcode. This is the - right route when the goal is measured Core ML or Apple Neural Engine evidence - on a physical iPhone. +The resource script uses the tracked current-model checkpoint and preprocessing +state. It embeds their SHA-256 values in `ModelContract.json`; the exported +device evidence is rejected if those values do not match the repository. + +## Run on a physical iPhone + +1. Install the full Xcode application from Apple and open + `ios/EdgeGenBenchDemo.xcodeproj`. +2. Connect the iPhone by USB, unlock it, tap **Trust** if prompted, and enable + Developer Mode if Xcode requests it. +3. Select the `EdgeGenBenchDemo` target. Under **Signing & Capabilities**, choose + your personal development team. EdgeGenBench does not collect or store your + signing identity. +4. Select the connected iPhone as the run destination and press **Run**. +5. Keep Low Power Mode off, close other foreground apps, and let the device + reach a stable temperature. +6. In EdgeGenBench, tap **Run cold + warm benchmark**. Confirm it reports one + cold and 100 warm runs. +7. Tap **Export evidence JSON**, AirDrop or save the file to the Mac, and retain + a screenshot of the result screen. +8. Validate and render the report from the repository root: + +```bash +python scripts/validate_ios_evidence.py \ + "$HOME/Downloads/EdgeGenBench-iOS-evidence.json" \ + --output-json reports/ios_device_summary.json \ + --output-markdown reports/ios_device_report.md +``` + +Commit the raw evidence JSON, generated summary, Markdown report, and screenshot +in a follow-up evidence PR. The validator rejects simulator evidence, fewer than +100 warm runs, excessive output drift, mismatched model/preprocessing hashes, +and unsupported ANE or power claims. + +## Optional Instruments evidence + +The app requests Core ML `MLComputeUnits.all`, which does **not** prove that the +Apple Neural Engine executed the graph. To claim ANE placement or energy, run a +Release build under Xcode Instruments with the Core ML and Energy Log templates, +retain the `.trace` bundle or exported summary, name the Xcode/iOS/device +versions, and report the measured boundary. Without that evidence the project +correctly reports `neuralEnginePlacement=not_measured` and +`powerMeasurement=not_measured`. diff --git a/ios/project.yml b/ios/project.yml index ad07083..268575c 100644 --- a/ios/project.yml +++ b/ios/project.yml @@ -11,8 +11,30 @@ targets: settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.triasha72.EdgeGenBenchDemo + MARKETING_VERSION: 0.1.0 + CURRENT_PROJECT_VERSION: 1 SWIFT_VERSION: 5.0 GENERATE_INFOPLIST_FILE: YES INFOPLIST_KEY_UILaunchScreen_Generation: YES CODE_SIGN_STYLE: Automatic - + EdgeGenBenchDemoTests: + type: bundle.unit-test + platform: iOS + deploymentTarget: "17.0" + sources: + - EdgeGenBenchDemoTests + dependencies: + - target: EdgeGenBenchDemo + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.triasha72.EdgeGenBenchDemoTests + SWIFT_VERSION: 5.0 +schemes: + EdgeGenBenchDemo: + build: + targets: + EdgeGenBenchDemo: all + EdgeGenBenchDemoTests: [test] + test: + targets: + - EdgeGenBenchDemoTests diff --git a/reports/portfolio_acceptance.json b/reports/portfolio_acceptance.json index d3f3033..fc82f53 100644 --- a/reports/portfolio_acceptance.json +++ b/reports/portfolio_acceptance.json @@ -83,6 +83,14 @@ "report": "reports/android_16kb_emulator_reference_v0_1_7.md", "claim": "APK/JNI reference path executed on PAGE_SIZE=16384; not physical-device performance." }, + "ios_coreml": { + "status": "ci_build_and_simulator_test_configured", + "backend": "CoreML", + "source_model_sha256": "55d6db9f19f2e361c6066b639920cfd1aad54ea5544b463b205857ebb7ceb657", + "preprocessing_sha256": "c53831dd106a26b668d586b4eb83f73a1e43483c52ec8ee36c7ba35c95cdb08e", + "physical_device_status": "evidence_pending", + "claim": "Core ML export, SwiftUI app, XCTest, and unsigned simulator CI are configured; physical-iPhone latency remains pending a validated device export." + }, "power": { "status": "not_measured", "claim": "No power-savings claim is made without a named calibrated tool." diff --git a/reports/portfolio_acceptance.md b/reports/portfolio_acceptance.md index 48d110a..fd8ec8b 100644 --- a/reports/portfolio_acceptance.md +++ b/reports/portfolio_acceptance.md @@ -7,6 +7,7 @@ | `qualcomm_ai_hub_qnn` | `validated_ai_hub_physical_qnn` | Physical AI Hub model profiling; not Android APK end-to-end latency. | | `android_qnn_apk` | `implementation_complete_evidence_pending` | Build/JNI/capture paths exist; requires a supported Snapdragon APK run. | | `android_16kb_runtime` | `validated_16kb_emulator_runtime` | APK/JNI reference path executed on PAGE_SIZE=16384; not physical-device performance. | +| `ios_coreml` | `ci_build_and_simulator_test_configured` | Core ML export, SwiftUI app, XCTest, and unsigned simulator CI are configured; physical-iPhone latency remains pending a validated device export. | | `power` | `not_measured` | No power-savings claim is made without a named calibrated tool. | ## Validated Qualcomm QNN results @@ -22,4 +23,4 @@ Tracked QNN context provenance match: **True** (`artifacts/qualcomm_ai_hub/curre | 256 | 0.047000 | 5446808.511 | 122896384 | NPU × 9 | 0.002865936 | AI Hub measurements are physical-device model profiles, not Android application end-to-end timings. Current-model acceptance requires source-model provenance to match the repository, as reported above. -Power remains unmeasured. The remaining hardware proof item is a supported-device QNN APK run; the 16 KB reference APK/JNI runtime is validated on an API 35 emulator. +Power remains unmeasured. Remaining physical proof items are a supported-device QNN APK run and a validated iPhone Core ML export; the iOS simulator build/test lane does not establish device latency, ANE placement, or energy use. diff --git a/scripts/build_portfolio_acceptance.py b/scripts/build_portfolio_acceptance.py index ea1d134..56a7cf1 100644 --- a/scripts/build_portfolio_acceptance.py +++ b/scripts/build_portfolio_acceptance.py @@ -189,6 +189,39 @@ def validate_android_16kb_runtime(evidence_dir: Path, report_path: Path) -> dict } +def validate_ios_coreml_implementation(repository_root: Path) -> dict[str, Any]: + required = [ + repository_root / "ios/project.yml", + repository_root / "ios/EdgeGenBenchDemo/SurrogatePredictor.swift", + repository_root / "ios/EdgeGenBenchDemo/BenchmarkEvidence.swift", + repository_root / "ios/EdgeGenBenchDemoTests/BenchmarkStatisticsTests.swift", + repository_root / "scripts/prepare_ios_resources.py", + repository_root / "scripts/validate_ios_evidence.py", + repository_root / "artifacts/neural_surrogate/model.pt", + repository_root / "artifacts/neural_surrogate/preprocessing.npz", + ] + missing = [ + path.relative_to(repository_root).as_posix() for path in required if not path.is_file() + ] + if missing: + raise ValueError(f"iOS Core ML implementation is incomplete: {', '.join(missing)}") + workflow = (repository_root / ".github/workflows/ci.yml").read_text(encoding="utf-8") + for marker in ("iOS Core ML simulator", "prepare_ios_resources.py", "xcodebuild", "test"): + if marker not in workflow: + raise ValueError(f"iOS CI is missing required marker: {marker}") + return { + "status": "ci_build_and_simulator_test_configured", + "backend": "CoreML", + "source_model_sha256": _sha256(required[-2]), + "preprocessing_sha256": _sha256(required[-1]), + "physical_device_status": "evidence_pending", + "claim": ( + "Core ML export, SwiftUI app, XCTest, and unsigned simulator CI are configured; " + "physical-iPhone latency remains pending a validated device export." + ), + } + + def build_portfolio_acceptance( *, repository_root: Path, qnn_report: Path, output_json: Path, output_markdown: Path ) -> dict[str, Any]: @@ -201,6 +234,7 @@ def build_portfolio_acceptance( repository_root / "reports/device/android-16kb-api35-reference-10-runs", android_16kb_report, ) + ios_coreml = validate_ios_coreml_implementation(repository_root) matrix = { "schema_version": 1, "project": "EdgeGenBench", @@ -221,6 +255,7 @@ def build_portfolio_acceptance( "claim": "Build/JNI/capture paths exist; requires a supported Snapdragon APK run.", }, "android_16kb_runtime": android_16kb, + "ios_coreml": ios_coreml, "power": { "status": "not_measured", "claim": "No power-savings claim is made without a named calibrated tool.", @@ -266,8 +301,9 @@ def build_portfolio_acceptance( "AI Hub measurements are physical-device model profiles, not Android " "application end-to-end timings. Current-model acceptance requires source-model " "provenance to match the repository, as reported above.", - "Power remains unmeasured. The remaining hardware proof item is a supported-device " - "QNN APK run; the 16 KB reference APK/JNI runtime is validated on an API 35 emulator.", + "Power remains unmeasured. Remaining physical proof items are a supported-device " + "QNN APK run and a validated iPhone Core ML export; the iOS simulator build/test " + "lane does not establish device latency, ANE placement, or energy use.", ] ) output_markdown.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/scripts/build_release_evidence.py b/scripts/build_release_evidence.py index 0f0d544..2623fbf 100644 --- a/scripts/build_release_evidence.py +++ b/scripts/build_release_evidence.py @@ -311,6 +311,7 @@ def build_release_evidence( version: str, device_evidence: Path | None = None, qnn_evidence: Path | None = None, + ios_simulator_evidence: Path | None = None, ) -> Path: baseline = _load_json(baseline_path) fused = _load_json(fused_path) @@ -393,6 +394,35 @@ def build_release_evidence( "claim": "Exclusive QNN placement validated with CPU fallback disabled.", } + ios_status: dict[str, Any] = { + "status": "not_supplied", + "claim": "No iOS build or physical-iPhone claim is made by this bundle.", + } + if ios_simulator_evidence is not None: + if not ios_simulator_evidence.is_dir(): + raise ValueError("iOS simulator evidence must be a directory") + required_ios = ( + "EdgeGenBench-ios-simulator-app.zip", + "ios-tests.xcresult.zip", + "xcode-version.txt", + "checksums.txt", + ) + missing_ios = [ + name for name in required_ios if not (ios_simulator_evidence / name).is_file() + ] + if missing_ios: + raise ValueError(f"iOS simulator evidence is incomplete: {', '.join(missing_ios)}") + destination = output_dir / "ios-simulator" + shutil.copytree(ios_simulator_evidence, destination, dirs_exist_ok=True) + ios_status = { + "status": "validated_in_ci", + "path": "ios-simulator", + "claim": ( + "Unsigned Core ML simulator app built and XCTest result retained; " + "not physical-iPhone latency, ANE placement, or power evidence." + ), + } + files = [] for path in sorted( p for p in output_dir.rglob("*") if p.is_file() and p.name != "manifest.json" @@ -415,12 +445,14 @@ def build_release_evidence( "native_fused_passed": True, "baseline_fused_max_abs_drift": drift, "android_16kb_compatible": True, + "ios_coreml_simulator_build_and_tests": ios_status["status"] == "validated_in_ci", "cpu_fallback_claim": "not applicable to deterministic reference backend", "qnn_npu_placement": "not tested in CI", "power": "not measured", }, "device_evidence": device_status, "qnn_evidence": qnn_status, + "ios_simulator_evidence": ios_status, "files": files, } manifest_path = output_dir / "manifest.json" @@ -439,6 +471,7 @@ def main() -> None: parser.add_argument("--version", required=True) parser.add_argument("--device-evidence", type=Path) parser.add_argument("--qnn-evidence", type=Path) + parser.add_argument("--ios-simulator-evidence", type=Path) args = parser.parse_args() manifest = build_release_evidence( args.baseline, @@ -450,6 +483,7 @@ def main() -> None: version=args.version, device_evidence=args.device_evidence, qnn_evidence=args.qnn_evidence, + ios_simulator_evidence=args.ios_simulator_evidence, ) print(f"Release evidence validated: {manifest}") diff --git a/scripts/prepare_ios_resources.py b/scripts/prepare_ios_resources.py new file mode 100644 index 0000000..6b14e21 --- /dev/null +++ b/scripts/prepare_ios_resources.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Export and stage the current Core ML model for the native iOS target.""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +from edgegenbench.deployment.coreml_export import export_neural_surrogate_coreml + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, default=Path("artifacts/neural_surrogate/model.pt")) + parser.add_argument( + "--preprocessing", + type=Path, + default=Path("artifacts/neural_surrogate/preprocessing.npz"), + ) + parser.add_argument("--target", type=Path, default=Path("ios/EdgeGenBenchDemo/Resources")) + args = parser.parse_args() + + build_dir = args.target.parent / ".generated-coreml" + shutil.rmtree(build_dir, ignore_errors=True) + shutil.rmtree(args.target, ignore_errors=True) + artifacts = export_neural_surrogate_coreml( + model_path=args.model, + preprocessing_path=args.preprocessing, + output_dir=build_dir, + ) + args.target.mkdir(parents=True, exist_ok=True) + shutil.copytree(artifacts.model_path, args.target / artifacts.model_path.name) + shutil.copy2(artifacts.contract_path, args.target / artifacts.contract_path.name) + shutil.rmtree(build_dir) + print(f"Staged Core ML resources in {args.target}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_ios_evidence.py b/scripts/validate_ios_evidence.py new file mode 100644 index 0000000..8674fff --- /dev/null +++ b/scripts/validate_ios_evidence.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Validate an EdgeGenBench iPhone Core ML evidence export and write a report.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def validate_ios_evidence( + evidence_path: Path, + *, + model_path: Path, + preprocessing_path: Path, + allow_simulator: bool = False, +) -> dict[str, Any]: + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + if not isinstance(evidence, dict) or evidence.get("schemaVersion") != "1.0": + raise ValueError("unsupported iOS evidence schema") + if evidence.get("backend") != "CoreML" or evidence.get("requestedComputeUnits") != "all": + raise ValueError("evidence must identify Core ML with requested compute units") + if evidence.get("neuralEnginePlacement") != "not_measured": + raise ValueError("ANE placement cannot be inferred without retained placement evidence") + if evidence.get("powerMeasurement") != "not_measured": + raise ValueError("power claims require a named calibrated measurement tool") + + device = evidence.get("device") + latency = evidence.get("latency") + if not isinstance(device, dict) or not isinstance(latency, dict): + raise ValueError("device identity and latency summary are required") + if bool(device.get("simulator")) and not allow_simulator: + raise ValueError("physical-iPhone evidence cannot come from a simulator") + if device.get("systemName") != "iOS" and not allow_simulator: + raise ValueError("physical evidence must identify iOS") + if int(latency.get("warmRuns", 0)) < 100: + raise ValueError("at least 100 warm inference runs are required") + for name in ("coldMs", "warmMeanMs", "warmP95Ms"): + if float(latency.get(name, 0)) <= 0: + raise ValueError(f"{name} must be positive") + if float(evidence.get("outputMaxAbsDrift", 1.0)) > 1e-6: + raise ValueError("iOS repeated-output drift exceeds tolerance") + if evidence.get("sourceModelSha256") != _sha256(model_path): + raise ValueError("iOS source-model provenance does not match the repository") + if evidence.get("preprocessingSha256") != _sha256(preprocessing_path): + raise ValueError("iOS preprocessing provenance does not match the repository") + + return { + "status": "validated_physical_iphone_coreml" + if not device["simulator"] + else "validated_simulator_coreml", + "captured_at_utc": evidence["capturedAtUTC"], + "app_version": evidence["appVersion"], + "device": device, + "backend": "CoreML", + "requested_compute_units": "all", + "latency": latency, + "output_max_abs_drift": evidence["outputMaxAbsDrift"], + "thermal_state_before": evidence["thermalStateBefore"], + "thermal_state_after": evidence["thermalStateAfter"], + "power_measurement": "not_measured", + "neural_engine_placement": "not_measured", + "claim_boundary": ( + "Physical iPhone Core ML application latency; not proof of Apple Neural Engine " + "placement and not a power measurement." + ), + } + + +def write_report(summary: dict[str, Any], output_json: Path, output_markdown: Path) -> None: + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + latency = summary["latency"] + device = summary["device"] + lines = [ + "# EdgeGenBench physical iPhone Core ML report", + "", + f"- Status: `{summary['status']}`", + f"- Device: `{device['model']}`", + f"- OS: `{device['systemName']} {device['systemVersion']}`", + f"- Backend: `{summary['backend']}` (requested compute units: `all`)", + f"- Cold latency: `{latency['coldMs']:.6f} ms`", + f"- Warm mean latency: `{latency['warmMeanMs']:.6f} ms`", + f"- Warm p95 latency: `{latency['warmP95Ms']:.6f} ms`", + f"- Warm runs: `{latency['warmRuns']}`", + f"- Output max absolute drift: `{summary['output_max_abs_drift']}`", + "- Thermal state: " + f"`{summary['thermal_state_before']}` → `{summary['thermal_state_after']}`", + "- Power: `not measured`", + "- Apple Neural Engine placement: `not measured`", + "", + f"> {summary['claim_boundary']}", + ] + output_markdown.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("evidence", type=Path) + parser.add_argument("--model", type=Path, default=Path("artifacts/neural_surrogate/model.pt")) + parser.add_argument( + "--preprocessing", type=Path, default=Path("artifacts/neural_surrogate/preprocessing.npz") + ) + parser.add_argument("--output-json", type=Path, default=Path("reports/ios_device_summary.json")) + parser.add_argument( + "--output-markdown", type=Path, default=Path("reports/ios_device_report.md") + ) + parser.add_argument("--allow-simulator", action="store_true") + args = parser.parse_args() + summary = validate_ios_evidence( + args.evidence, + model_path=args.model, + preprocessing_path=args.preprocessing, + allow_simulator=args.allow_simulator, + ) + write_report(summary, args.output_json, args.output_markdown) + print(f"Validated iOS evidence: {args.output_markdown}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/edgegenbench/deployment/coreml_export.py b/src/edgegenbench/deployment/coreml_export.py index 7f1b272..64fd5e9 100644 --- a/src/edgegenbench/deployment/coreml_export.py +++ b/src/edgegenbench/deployment/coreml_export.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from dataclasses import dataclass from pathlib import Path @@ -27,10 +28,25 @@ class CoreMLExportArtifacts: output_dim: int -def build_ios_contract(preprocessor: NeuralPreprocessor) -> dict[str, Any]: +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def build_ios_contract( + preprocessor: NeuralPreprocessor, + *, + source_model_sha256: str | None = None, + preprocessing_sha256: str | None = None, +) -> dict[str, Any]: """Serialize preprocessing and inverse-scaling rules for Swift.""" return { - "schemaVersion": "1.0", + "schemaVersion": "1.1", + "sourceModelSha256": source_model_sha256, + "preprocessingSha256": preprocessing_sha256, "inputName": COREML_INPUT_NAME, "outputName": COREML_OUTPUT_NAME, "numericFeatures": list(NUMERIC_FEATURES), @@ -82,7 +98,15 @@ def export_neural_surrogate_coreml( contract_destination = output_dir / "ModelContract.json" converted.save(str(model_destination)) contract_destination.write_text( - json.dumps(build_ios_contract(preprocessor), indent=2) + "\n", + json.dumps( + build_ios_contract( + preprocessor, + source_model_sha256=_sha256(model_path), + preprocessing_sha256=_sha256(preprocessing_path), + ), + indent=2, + ) + + "\n", encoding="utf-8", ) return CoreMLExportArtifacts( diff --git a/tests/neural/test_coreml_export.py b/tests/neural/test_coreml_export.py index 1c43db4..4242521 100644 --- a/tests/neural/test_coreml_export.py +++ b/tests/neural/test_coreml_export.py @@ -16,9 +16,11 @@ def test_ios_contract_preserves_preprocessing_and_output_scaling() -> None: targets=("a", "b", "c", "d", "e", "f"), ) contract = build_ios_contract(preprocessor) + assert contract["schemaVersion"] == "1.1" assert contract["inputName"] == "features" assert contract["outputName"] == "predictions" assert contract["inputDimension"] == 9 assert contract["outputDimension"] == 6 assert contract["categories"] == ["battery_electric", "hybrid", "hydrogen"] assert contract["targetScale"] == [2.0] * 6 + assert contract["sourceModelSha256"] is None diff --git a/tests/test_ios_evidence.py b/tests/test_ios_evidence.py new file mode 100644 index 0000000..d033a13 --- /dev/null +++ b/tests/test_ios_evidence.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import hashlib +import json +import runpy +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import pytest + +SCRIPT = Path(__file__).parents[1] / "scripts/validate_ios_evidence.py" +ValidateIOS = Callable[..., dict[str, object]] +validate_ios_evidence = cast(ValidateIOS, runpy.run_path(SCRIPT)["validate_ios_evidence"]) + + +def _hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _evidence(model: Path, preprocessing: Path) -> dict[str, object]: + return { + "schemaVersion": "1.0", + "capturedAtUTC": "2026-08-27T00:00:00Z", + "appVersion": "0.1.0", + "backend": "CoreML", + "requestedComputeUnits": "all", + "neuralEnginePlacement": "not_measured", + "powerMeasurement": "not_measured", + "thermalStateBefore": "nominal", + "thermalStateAfter": "fair", + "lowPowerMode": False, + "sourceModelSha256": _hash(model), + "preprocessingSha256": _hash(preprocessing), + "contractSha256": "c" * 64, + "device": { + "model": "iPhone15,4", + "systemName": "iOS", + "systemVersion": "17.6", + "simulator": False, + }, + "latency": {"coldMs": 3.0, "warmMeanMs": 1.0, "warmP95Ms": 1.2, "warmRuns": 100}, + "outputMaxAbsDrift": 0.0, + "outputs": [{"name": "mass", "value": 1.0}], + } + + +def test_validates_physical_iphone_coreml_evidence(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + preprocessing = tmp_path / "preprocessing.npz" + model.write_bytes(b"model") + preprocessing.write_bytes(b"preprocessing") + evidence = tmp_path / "evidence.json" + evidence.write_text(json.dumps(_evidence(model, preprocessing))) + result = validate_ios_evidence(evidence, model_path=model, preprocessing_path=preprocessing) + assert result["status"] == "validated_physical_iphone_coreml" + assert result["neural_engine_placement"] == "not_measured" + + +def test_rejects_simulator_as_physical_evidence(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + preprocessing = tmp_path / "preprocessing.npz" + model.write_bytes(b"model") + preprocessing.write_bytes(b"preprocessing") + payload = _evidence(model, preprocessing) + cast(dict[str, object], payload["device"])["simulator"] = True + evidence = tmp_path / "evidence.json" + evidence.write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="physical-iPhone"): + validate_ios_evidence(evidence, model_path=model, preprocessing_path=preprocessing) + + +def test_rejects_unproven_ane_claim(tmp_path: Path) -> None: + model = tmp_path / "model.pt" + preprocessing = tmp_path / "preprocessing.npz" + model.write_bytes(b"model") + preprocessing.write_bytes(b"preprocessing") + payload = _evidence(model, preprocessing) + payload["neuralEnginePlacement"] = "ANE" + evidence = tmp_path / "evidence.json" + evidence.write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="ANE placement"): + validate_ios_evidence(evidence, model_path=model, preprocessing_path=preprocessing) diff --git a/tests/test_portfolio_acceptance.py b/tests/test_portfolio_acceptance.py index f0099f1..30b340b 100644 --- a/tests/test_portfolio_acceptance.py +++ b/tests/test_portfolio_acceptance.py @@ -13,6 +13,10 @@ validate_ai_hub_qnn = cast(ValidateQnn, FUNCTIONS["validate_ai_hub_qnn"]) Validate16Kb = Callable[[Path, Path], dict[str, object]] validate_android_16kb_runtime = cast(Validate16Kb, FUNCTIONS["validate_android_16kb_runtime"]) +ValidateIOS = Callable[[Path], dict[str, object]] +validate_ios_coreml_implementation = cast( + ValidateIOS, FUNCTIONS["validate_ios_coreml_implementation"] +) def test_validates_tracked_ai_hub_qnn_evidence() -> None: @@ -75,3 +79,11 @@ def test_rejects_non_16kb_android_runtime_evidence(tmp_path: Path) -> None: evidence, root / "reports/android_16kb_emulator_reference_v0_1_7.md", ) + + +def test_validates_ios_coreml_implementation_contract() -> None: + root = Path(__file__).parents[1] + result = validate_ios_coreml_implementation(root) + assert result["status"] == "ci_build_and_simulator_test_configured" + assert result["backend"] == "CoreML" + assert result["physical_device_status"] == "evidence_pending" diff --git a/tests/test_release_evidence.py b/tests/test_release_evidence.py index 018d62d..8b11382 100644 --- a/tests/test_release_evidence.py +++ b/tests/test_release_evidence.py @@ -249,3 +249,26 @@ def test_release_bundle_retains_validated_qnn_artifacts(tmp_path: Path) -> None: assert manifest["qnn_evidence"]["status"] == "validated_qnn_npu" assert (tmp_path / "release/qnn/summary.json").is_file() assert (tmp_path / "release/qnn/artifacts/qnn_context.bin").is_file() + + +def test_release_bundle_retains_ios_simulator_acceptance(tmp_path: Path) -> None: + ios = tmp_path / "ios-evidence" + ios.mkdir() + for name in ( + "EdgeGenBench-ios-simulator-app.zip", + "ios-tests.xcresult.zip", + "xcode-version.txt", + "checksums.txt", + ): + (ios / name).write_bytes(name.encode()) + manifest_path = build_release_evidence( + *_inputs(tmp_path), + tmp_path / "release", + git_revision="abc123", + version="0.1.8", + ios_simulator_evidence=ios, + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["acceptance"]["ios_coreml_simulator_build_and_tests"] is True + assert manifest["ios_simulator_evidence"]["status"] == "validated_in_ci" + assert (tmp_path / "release/ios-simulator/ios-tests.xcresult.zip").is_file()