diff --git a/JetlinkKit/Sources/COrt/include/jl_ort.h b/JetlinkKit/Sources/COrt/include/jl_ort.h index 2ab08937..21075924 100644 --- a/JetlinkKit/Sources/COrt/include/jl_ort.h +++ b/JetlinkKit/Sources/COrt/include/jl_ort.h @@ -36,6 +36,7 @@ enum { typedef struct jl_env jl_env; typedef struct jl_session jl_session; typedef struct jl_binding jl_binding; +typedef struct jl_shared_allocator jl_shared_allocator; typedef struct { const char *key; @@ -77,10 +78,19 @@ char *jl_session_io_info(const jl_session *session, int output, size_t index, char *name, size_t name_cap, int *elem_type, int64_t *dims, size_t dims_cap, size_t *rank); +// QNN's host-accessible RPC memory. The session must outlive this allocator, +// and every allocation and binding must be freed before releasing it. +char *jl_shared_allocator_create(jl_session *session, jl_shared_allocator **out); +char *jl_shared_allocator_alloc(jl_shared_allocator *allocator, size_t bytes, void **out); +void jl_shared_allocator_free(jl_shared_allocator *allocator, void *data); +void jl_shared_allocator_release(jl_shared_allocator *allocator); +size_t jl_shared_allocator_total_bytes(const jl_shared_allocator *allocator); + // Binds caller-owned buffers to a session's inputs and outputs once, for // jl_binding_run to run with. The buffers must outlive the binding. char *jl_binding_create(jl_session *session, const jl_tensor *inputs, size_t n_inputs, - const jl_tensor *outputs, size_t n_outputs, jl_binding **out); + const jl_tensor *outputs, size_t n_outputs, + const jl_shared_allocator *shared, jl_binding **out); char *jl_binding_run(jl_binding *binding); void jl_binding_release(jl_binding *binding); diff --git a/JetlinkKit/Sources/COrt/jl_ort.c b/JetlinkKit/Sources/COrt/jl_ort.c index 77422489..c48b6e65 100644 --- a/JetlinkKit/Sources/COrt/jl_ort.c +++ b/JetlinkKit/Sources/COrt/jl_ort.c @@ -2,6 +2,7 @@ #include #include +#include #include @@ -24,6 +25,13 @@ struct jl_binding { size_t n_values; }; +struct jl_shared_allocator { + OrtSession *session; + OrtAllocator *allocator; + OrtMemoryInfo *memory; + size_t total_bytes; +}; + #ifdef JL_ORT_DLOPEN // NULL when the library or its entry point is missing; jl_env_create says why. static const OrtApiBase *base(void) { @@ -274,14 +282,76 @@ void jl_binding_release(jl_binding *binding) { free(binding); } +void jl_shared_allocator_release(jl_shared_allocator *allocator) { + if (allocator == NULL) return; + if (allocator->allocator != NULL) api()->ReleaseAllocator(allocator->allocator); + if (allocator->memory != NULL) api()->ReleaseMemoryInfo(allocator->memory); + free(allocator); +} + +char *jl_shared_allocator_create(jl_session *session, jl_shared_allocator **out) { + if (session == NULL || out == NULL) return copy("shared allocator needs a session and output"); + *out = NULL; + jl_shared_allocator *allocator = calloc(1, sizeof(*allocator)); + if (allocator == NULL) return copy("out of memory"); + allocator->session = session->session; + char *error = take(api()->CreateMemoryInfo("QnnHtpShared", OrtDeviceAllocator, 0, + OrtMemTypeDefault, &allocator->memory)); + if (error == NULL) error = take(api()->CreateAllocator(session->session, allocator->memory, &allocator->allocator)); + const OrtMemoryInfo *actual = NULL; + if (error == NULL) error = take(api()->AllocatorGetInfo(allocator->allocator, &actual)); + int different = 1; + if (error == NULL) error = take(api()->CompareMemoryInfo(allocator->memory, actual, &different)); + if (error == NULL && different != 0) error = copy("session did not return QNN shared memory"); + if (error != NULL) { + jl_shared_allocator_release(allocator); + return error; + } + *out = allocator; + return NULL; +} + +char *jl_shared_allocator_alloc(jl_shared_allocator *allocator, size_t bytes, void **out) { + if (allocator == NULL || out == NULL) return copy("invalid shared allocation"); + *out = NULL; + if (bytes > SIZE_MAX - allocator->total_bytes) return copy("shared allocation byte count overflow"); + char *error = take(api()->AllocatorAlloc(allocator->allocator, bytes, out)); + if (error == NULL && *out == NULL) error = copy("QNN shared allocator returned no memory"); + if (error == NULL) allocator->total_bytes += bytes; + return error; +} + +void jl_shared_allocator_free(jl_shared_allocator *allocator, void *data) { + if (allocator == NULL || data == NULL) return; + char *error = take(api()->AllocatorFree(allocator->allocator, data)); + if (error != NULL) { + fprintf(stderr, "Jetlink QNN shared memory free: %s\n", error); + free(error); + } +} + +size_t jl_shared_allocator_total_bytes(const jl_shared_allocator *allocator) { + return allocator == NULL ? 0 : allocator->total_bytes; +} + static char *wrap(const OrtMemoryInfo *memory, const jl_tensor *t, OrtValue **out) { return take(api()->CreateTensorWithDataAsOrtValue(memory, t->data, t->nbytes, t->dims, t->rank, (ONNXTensorElementDataType)t->elem_type, out)); } char *jl_binding_create(jl_session *session, const jl_tensor *inputs, size_t n_inputs, - const jl_tensor *outputs, size_t n_outputs, jl_binding **out) { + const jl_tensor *outputs, size_t n_outputs, + const jl_shared_allocator *shared, jl_binding **out) { *out = NULL; + if (shared != NULL && shared->session != session->session) { + // ORT's QNN allocator tracks RPC allocations globally. A second HTP + // context can register the same host-accessible allocation. Verify that + // this target session really exposes QnnHtpShared; CPU/GPU may not claim it. + jl_shared_allocator *compatible = NULL; + char *error = jl_shared_allocator_create(session, &compatible); + if (error != NULL) return error; + jl_shared_allocator_release(compatible); + } jl_binding *binding = calloc(1, sizeof(jl_binding)); if (binding == NULL) { return copy("out of memory"); @@ -292,8 +362,13 @@ char *jl_binding_create(jl_session *session, const jl_tensor *inputs, size_t n_i free(binding); return copy("out of memory"); } - OrtMemoryInfo *memory = NULL; - char *error = take(api()->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &memory)); + OrtMemoryInfo *cpu_memory = NULL; + const OrtMemoryInfo *memory = shared != NULL ? shared->memory : NULL; + char *error = NULL; + if (memory == NULL) { + error = take(api()->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &cpu_memory)); + memory = cpu_memory; + } if (error == NULL) { error = take(api()->CreateIoBinding(session->session, &binding->binding)); } @@ -311,8 +386,8 @@ char *jl_binding_create(jl_session *session, const jl_tensor *inputs, size_t n_i error = take(api()->BindOutput(binding->binding, outputs[i].name, *value)); } } - if (memory != NULL) { - api()->ReleaseMemoryInfo(memory); + if (cpu_memory != NULL) { + api()->ReleaseMemoryInfo(cpu_memory); } if (error != NULL) { jl_binding_release(binding); diff --git a/JetlinkKit/Sources/JetlinkAndroid/Bridge.swift b/JetlinkKit/Sources/JetlinkAndroid/Bridge.swift index 29e537bb..191aa476 100644 --- a/JetlinkKit/Sources/JetlinkAndroid/Bridge.swift +++ b/JetlinkKit/Sources/JetlinkAndroid/Bridge.swift @@ -232,7 +232,8 @@ } return OrtBackend( profile: profile, preparer: ONNXPreparer(), keepAlive: (config["keep_alive"] as? Bool) ?? true, - keepCPUWarm: (config["keep_cpu_warm"] as? Bool) ?? false, chip: chip) + keepCPUWarm: (config["keep_cpu_warm"] as? Bool) ?? false, chip: chip, + qnn: try QnnPolicy(configuration: config)) case let other: throw HostFailure("unknown backend \(other)") } diff --git a/JetlinkKit/Sources/JetlinkORT/Ort.swift b/JetlinkKit/Sources/JetlinkORT/Ort.swift index 3a6830ad..1021f6d5 100644 --- a/JetlinkKit/Sources/JetlinkORT/Ort.swift +++ b/JetlinkKit/Sources/JetlinkORT/Ort.swift @@ -113,11 +113,45 @@ final class OrtSession: @unchecked Sendable { } } +/// Host-accessible allocations tracked by QNN for RPC memory registration. +/// Retaining the session keeps its allocator valid until every buffer is freed. +final class OrtSharedAllocator: @unchecked Sendable { + let pointer: OpaquePointer + private let session: OrtSession + + init(session: OrtSession) throws { + self.session = session + var allocator: OpaquePointer? + try OrtError.check(jl_shared_allocator_create(session.pointer, &allocator)) + guard let allocator else { throw OrtError("QNN returned no shared allocator") } + pointer = allocator + } + + deinit { jl_shared_allocator_release(pointer) } + + var totalBytes: Int { jl_shared_allocator_total_bytes(pointer) } + + var hostAllocator: HostAllocator { + HostAllocator( + allocate: { [self] bytes in + var memory: UnsafeMutableRawPointer? + try OrtError.check(jl_shared_allocator_alloc(pointer, bytes, &memory)) + guard let memory else { throw OrtError("QNN shared allocation returned no pointer") } + return memory + }, + free: { [self] memory in + jl_shared_allocator_free(pointer, memory) + }) + } +} + /// Caller-owned buffers bound to one session's inputs and outputs. final class OrtBinding { private let pointer: OpaquePointer + private let sharedAllocator: OrtSharedAllocator? - init(session: OrtSession, inputs: [(TensorSpec, UnsafeMutableRawPointer)], outputs: [(TensorSpec, UnsafeMutableRawPointer)]) throws { + init(session: OrtSession, inputs: [(TensorSpec, UnsafeMutableRawPointer)], outputs: [(TensorSpec, UnsafeMutableRawPointer)], sharedAllocator: OrtSharedAllocator? = nil) throws { + self.sharedAllocator = sharedAllocator var names: [UnsafeMutablePointer?] = [] var dims: [UnsafeMutablePointer] = [] defer { @@ -135,7 +169,7 @@ final class OrtBinding { let ins = inputs.map { tensor($0.0, $0.1) } let outs = outputs.map { tensor($0.0, $0.1) } var binding: OpaquePointer? - try OrtError.check(jl_binding_create(session.pointer, ins, ins.count, outs, outs.count, &binding)) + try OrtError.check(jl_binding_create(session.pointer, ins, ins.count, outs, outs.count, sharedAllocator?.pointer, &binding)) guard let binding else { throw OrtError("onnxruntime returned no binding") } pointer = binding } diff --git a/JetlinkKit/Sources/JetlinkORT/OrtBackend.swift b/JetlinkKit/Sources/JetlinkORT/OrtBackend.swift index a789aba3..08964b94 100644 --- a/JetlinkKit/Sources/JetlinkORT/OrtBackend.swift +++ b/JetlinkKit/Sources/JetlinkORT/OrtBackend.swift @@ -138,6 +138,7 @@ public final class OrtBackend: EngineBackend { public let keepAlive: Bool /// Keep a CPU core warm while the Neural Engine or the NPU runs a frame. public let keepCPUWarm: Bool + public let qnn: QnnPolicy private let preparer: any ModelPreparer private let chip: String private let log = ServerLog(category: "ort") @@ -145,18 +146,22 @@ public final class OrtBackend: EngineBackend { /// `chip` names what the artifacts are valid for: on Android the SoC's /// model, "SM8650" (Build.SOC_MODEL), which a QNN context is compiled for. /// Nil is the Apple chip on Apple platforms and "cpu" elsewhere. - public init(profile: OrtProfile, preparer: any ModelPreparer, keepAlive: Bool = true, keepCPUWarm: Bool = true, chip: String? = nil) { + public init( + profile: OrtProfile, preparer: any ModelPreparer, keepAlive: Bool = true, keepCPUWarm: Bool = true, + chip: String? = nil, qnn: QnnPolicy = .init() + ) { self.profile = profile self.preparer = preparer self.keepAlive = keepAlive self.keepCPUWarm = keepCPUWarm self.chip = HostChip.resolve(chip) + self.qnn = qnn } public var runtimeVersion: String { OrtRuntime.version } public func deviceTag() -> String { - sanitize("\(profile.rawValue)-\(chip)") + sanitize("\(profile.rawValue)-\(chip)\(profile.sessions.contains(where: { $0.unit == .htp }) ? qnn.cacheSuffix : "")") } public func deriveSpec(model: URL, sha256: String, nbytes: Int64, frameSkip: Int) throws -> ModelSpec { @@ -167,8 +172,7 @@ public final class OrtBackend: EngineBackend { func providerOptions(_ unit: OrtUnit) -> [String: String] { switch unit { case .htp: - [ - "backend_type": "htp", + var options = qnn.htpBackend.merging([ // burst holds the clocks up between frames, the counterpart of the // Mac's GPU keep-alive; sustained trades speed for less heat. "htp_performance_mode": keepAlive ? "burst" : "sustained_high_performance", @@ -176,9 +180,11 @@ public final class OrtBackend: EngineBackend { "htp_graph_finalization_optimization_mode": "3", "enable_htp_fp16_precision": "1", "qnn_context_priority": "high", - ] - case .qnnGPU: ["backend_type": "gpu"] - case .coreML, .cpu: [:] + ]) { _, value in value } + if qnn.sharedIO { options["enable_htp_shared_memory_allocator"] = "1" } + return options + case .qnnGPU: return ["backend_type": "gpu"] + case .coreML, .cpu: return [:] } } @@ -187,7 +193,9 @@ public final class OrtBackend: EngineBackend { func plan(_ unit: OrtUnit, model: URL, cache: URL?) -> SessionPlan { switch unit { case .coreML(let units): SessionPlan(model: model, computeUnits: units, cacheDirectory: cache) - case .htp: SessionPlan(model: model, provider: "QNN", options: providerOptions(unit), label: "QNN(htp)", usesNeuralEngine: true) + case .htp: SessionPlan( + model: model, provider: "QNN", options: providerOptions(unit), config: qnn.sessionConfig, + label: "QNN(htp)", usesNeuralEngine: true) case .qnnGPU: SessionPlan(model: model, provider: "QNN", options: providerOptions(unit), label: "QNN(gpu)", usesGPU: true) case .cpu: SessionPlan(model: model, provider: nil, threads: HostChip.cpuThreads, label: "CPU") } @@ -204,6 +212,7 @@ public final class OrtBackend: EngineBackend { // MARK: build public func build(model: URL, artifact: URL, report: @escaping ProgressFn, metaExtra: [String: Any]) throws { + try qnn.validate() let started = Date() let expect = Artifact.sidecar(artifact) try Artifact.build(artifact, metaExtra: metaExtra, report: report) { staged in @@ -267,12 +276,7 @@ public final class OrtBackend: EngineBackend { let context = staged.appending(path: "\(session)_ctx.onnx") report("compile", 0, "compiling \(session) for the NPU") try Ticker.during(interval: 2, Ticker.paced("compile", "compiling \(session) for the NPU", took: took, report: report)) { - let config = [ - "ep.context_enable": "1", - "ep.context_file_path": context.path, - // the context binary in a file of its own beside the model, not base64 inside it - "ep.context_embed_mode": "0", - ] + let config = contextConfiguration(context: context) _ = try OrtSession(model: source, provider: "QNN", options: providerOptions(.htp), config: config) } guard FileManager.default.fileExists(atPath: context.path) else { @@ -283,6 +287,16 @@ public final class OrtBackend: EngineBackend { return context.lastPathComponent } + /// The same strict provider contract applies during compilation and reload. + func contextConfiguration(context: URL) -> [String: String] { + qnn.sessionConfig.merging([ + "ep.context_enable": "1", + "ep.context_file_path": context.path, + // the context binary in a file of its own beside the model, not base64 inside it + "ep.context_embed_mode": "0", + ]) { _, value in value } + } + /// A CoreML build's first session creation is the conversion and CoreML's /// compile, so its progress is read off the cache directories as they fill. private func checkCoreML( @@ -326,6 +340,7 @@ public final class OrtBackend: EngineBackend { // MARK: load public func load(artifact: URL, report: @escaping ProgressFn) throws -> any Engine { + try qnn.validate() let (manifest, meta) = try OrtArtifact.open(artifact, version: profile.prepareVersion) { entry in guard let unit = OrtUnit(entry: entry) else { return "a session names no unit" } // Without its compile, onnxruntime would recompile under a "loading" diff --git a/JetlinkKit/Sources/JetlinkORT/OrtEngine.swift b/JetlinkKit/Sources/JetlinkORT/OrtEngine.swift index f12f9e18..bb272438 100644 --- a/JetlinkKit/Sources/JetlinkORT/OrtEngine.swift +++ b/JetlinkKit/Sources/JetlinkORT/OrtEngine.swift @@ -9,6 +9,8 @@ public struct SessionPlan: Sendable, Equatable { /// provider alone. public let provider: String? public let options: [String: String] + /// Per-session configuration, including an explicit fallback contract. + public let config: [String: String] /// The CPU provider's intra-op pool: 1 where an accelerator does the work. public let threads: Int /// The session as the log and the hello name it: "CoreML(CPUAndGPU)", @@ -20,12 +22,13 @@ public struct SessionPlan: Sendable, Equatable { public let usesNeuralEngine: Bool public init( - model: URL, provider: String?, options: [String: String] = [:], threads: Int = 1, label: String, + model: URL, provider: String?, options: [String: String] = [:], config: [String: String] = [:], threads: Int = 1, label: String, usesGPU: Bool = false, usesNeuralEngine: Bool = false ) { self.model = model self.provider = provider self.options = options + self.config = config self.threads = threads self.label = label self.usesGPU = usesGPU @@ -70,10 +73,14 @@ public final class OrtEngine: EngineCore, @unchecked Sendable { /// Whether the CPU is kept warm beside this engine: a busy thread, or /// Android's performance hints. public var keepsCPUWarm: Bool { keepWarm != nil || hint != nil } - public override var notes: String { "CPU keep-warm \(keepsCPUWarm ? "on" : "off")" } + public override var notes: String { + "CPU keep-warm \(keepsCPUWarm ? "on" : "off"); QNN shared I/O \(sharedAllocator == nil ? "off" : "on")" + } public override var coolsWhenIdle: Bool { usesNeuralEngine } private let chain: [OrtSession] + private let sharedAllocator: OrtSharedAllocator? + private let sharedSessions: [Bool] private var bindings: [[OrtBinding]] = [] // [parity][session] private let keepAlive: MetalKeepAlive? private let keepWarm: CPUKeepWarm? @@ -86,10 +93,19 @@ public final class OrtEngine: EngineCore, @unchecked Sendable { self.device = device var chain: [OrtSession] = [] for plan in plans { - chain.append(try OrtSession(model: plan.model, provider: plan.provider, options: plan.options, threads: plan.threads)) + chain.append(try OrtSession(model: plan.model, provider: plan.provider, options: plan.options, config: plan.config, threads: plan.threads)) } self.chain = chain self.providers = plans.map(\.label) + let sharedSessions = plans.map { $0.options["enable_htp_shared_memory_allocator"] == "1" } + for (index, enabled) in sharedSessions.enumerated() where enabled { + guard plans[index].provider == "QNN", plans[index].usesNeuralEngine, !plans[index].usesGPU else { + throw OrtError("only QNN HTP sessions may bind QNN shared memory") + } + } + let shared = try sharedSessions.firstIndex(of: true).map { try OrtSharedAllocator(session: chain[$0]) } + self.sharedAllocator = shared + self.sharedSessions = sharedSessions // What the host stages: every session's inputs no earlier session // produces. What it can read: every session's outputs no later one reads. @@ -122,7 +138,7 @@ public final class OrtEngine: EngineCore, @unchecked Sendable { let hint = warmCPU ? PerformanceHint.make() : nil self.hint = hint self.keepWarm = warmCPU && hint == nil ? CPUKeepWarm() : nil - try super.init(inputs: inputs, outputs: outputs, sizes: sizes) + try super.init(inputs: inputs, outputs: outputs, sizes: sizes, allocator: shared?.hostAllocator ?? .heap) // A throw from here on closes through deinit. try rebind([]) } @@ -145,14 +161,18 @@ public final class OrtEngine: EngineCore, @unchecked Sendable { var sets: [[OrtBinding]] = [] for p in 0..<(pairs.isEmpty ? 1 : 2) { var set: [OrtBinding] = [] - for session in chain { + for (index, session) in chain.enumerated() { let ins = session.inputs.map { ($0, buffer($0.name, parity: p)!) } let outs = session.outputs.map { spec -> (TensorSpec, UnsafeMutableRawPointer) in guard let input = outputOf[spec.name] else { return (spec, buffer(spec.name, parity: p)!) } // writes the buffer the state_ input reads next run return (spec, buffer(input, parity: p ^ 1)!) } - set.append(try OrtBinding(session: session, inputs: ins, outputs: outs)) + // These allocations remain host-accessible. Only HTP sessions label + // their bindings QnnHtpShared; CPU/GPU sessions use CPU memory metadata. + set.append(try OrtBinding( + session: session, inputs: ins, outputs: outs, + sharedAllocator: sharedSessions[index] ? sharedAllocator : nil)) } sets.append(set) } @@ -181,7 +201,8 @@ public final class OrtEngine: EngineCore, @unchecked Sendable { /// second is the steady state. public override func warm() throws -> String { _ = try super.warm() - return "onnxruntime \(OrtRuntime.version) on \(device) in process, sessions \(providers.joined(separator: " then "))" + let shared = sharedAllocator.map { "; QNN shared I/O: \($0.totalBytes) bytes allocated" } ?? "" + return "onnxruntime \(OrtRuntime.version) on \(device) in process, sessions \(providers.joined(separator: " then "))\(shared)" } public override func close() { diff --git a/JetlinkKit/Sources/JetlinkORT/QnnPolicy.swift b/JetlinkKit/Sources/JetlinkORT/QnnPolicy.swift new file mode 100644 index 00000000..5917e0f6 --- /dev/null +++ b/JetlinkKit/Sources/JetlinkORT/QnnPolicy.swift @@ -0,0 +1,66 @@ +import CoreFoundation +import Foundation +import JetlinkServer + +/// Optional QNN HTP behavior. Defaults preserve the existing provider and caches. +/// Shared I/O changes host allocation; strict mode changes fallback admission; +/// disabling activation fusion is a separate numerical-correctness workaround. +public struct QnnPolicy: Sendable, Equatable { + public let sharedIO: Bool + public let strict: Bool + public let disableAdvancedActivationFusion: Bool + public let libraryDirectory: String? + public let runtimeIdentity: String? + + public init( + sharedIO: Bool = false, strict: Bool = false, disableAdvancedActivationFusion: Bool = false, + libraryDirectory: String? = nil, runtimeIdentity: String? = nil + ) { + self.sharedIO = sharedIO + self.strict = strict + self.disableAdvancedActivationFusion = disableAdvancedActivationFusion + self.libraryDirectory = libraryDirectory + self.runtimeIdentity = runtimeIdentity + } + + /// App configuration accepts JSON Booleans, not strings or numeric lookalikes. + public init(configuration: [String: Any]) throws { + func flag(_ key: String) throws -> Bool { + guard let value = configuration[key] else { return false } + guard let number = value as? NSNumber, CFGetTypeID(number) == CFBooleanGetTypeID() else { + throw HostError.invalid("\(key) must be a Boolean") + } + return number.boolValue + } + self.init( + sharedIO: try flag("qnn_shared_io"), strict: try flag("qnn_strict"), + disableAdvancedActivationFusion: try flag("qnn_no_activation_fusion"), + libraryDirectory: configuration["native_library_dir"] as? String, + runtimeIdentity: configuration["qnn_runtime_identity"] as? String) + try validate() + } + + var sessionConfig: [String: String] { strict ? ["session.disable_cpu_ep_fallback": "1"] : [:] } + + var cacheSuffix: String { + guard sharedIO || strict || disableAdvancedActivationFusion else { return "" } + let sdk = disableAdvancedActivationFusion ? "-sdk\(runtimeIdentity ?? "missing")" : "" + return "-qnn-v1-shared\(sharedIO ? 1 : 0)-strict\(strict ? 1 : 0)-nofusion\(disableAdvancedActivationFusion ? 1 : 0)\(sdk)" + } + + func validate() throws { + guard disableAdvancedActivationFusion else { return } + guard let libraryDirectory, libraryDirectory.hasPrefix("/"), !libraryDirectory.contains("\0"), + let runtimeIdentity, runtimeIdentity.count == 64, + runtimeIdentity.utf8.allSatisfy({ (48...57).contains($0) || (97...102).contains($0) }) else { + throw HostError.invalid("the no-fusion adapter requires a matching SDK directory and runtime identity") + } + } + + var htpBackend: [String: String] { + guard disableAdvancedActivationFusion else { return ["backend_type": "htp"] } + let directory = libraryDirectory.map { URL(fileURLWithPath: $0, isDirectory: true) } + let path = directory?.appendingPathComponent("libJetlinkQnnHtpNoFusion.so").path ?? "libJetlinkQnnHtpNoFusion.so" + return ["backend_path": path] + } +} diff --git a/JetlinkKit/Tests/JetlinkServerTests/OrtSharedMemoryTests.swift b/JetlinkKit/Tests/JetlinkServerTests/OrtSharedMemoryTests.swift new file mode 100644 index 00000000..bc015571 --- /dev/null +++ b/JetlinkKit/Tests/JetlinkServerTests/OrtSharedMemoryTests.swift @@ -0,0 +1,22 @@ +import Foundation +import Testing + +@testable import JetlinkORT + +@Suite("QNN shared memory", .serialized) +struct OrtSharedMemoryTests { + @Test("A CPU session cannot silently substitute heap memory for QNN RPC memory") + func cpuDoesNotPretendToShare() throws { + let session = try OrtSession(model: Golden("tiny_stateful").model, provider: nil) + #expect(throws: OrtError.self) { _ = try OrtSharedAllocator(session: session) } + } + + @Test("CPU plans cannot request HTP shared bindings") + func wrongProvider() throws { + let plan = SessionPlan(model: try Golden("tiny_stateful").model, provider: nil, + options: ["enable_htp_shared_memory_allocator": "1"], label: "CPU") + #expect(throws: OrtError.self) { + _ = try OrtEngine(plans: [plan], device: "cpu", keepAlive: false, keepCPUWarm: false) + } + } +} diff --git a/JetlinkKit/Tests/JetlinkServerTests/QnnPolicyTests.swift b/JetlinkKit/Tests/JetlinkServerTests/QnnPolicyTests.swift new file mode 100644 index 00000000..965eb0ea --- /dev/null +++ b/JetlinkKit/Tests/JetlinkServerTests/QnnPolicyTests.swift @@ -0,0 +1,98 @@ +import Foundation +import JetlinkKit +import JetlinkServer +import Testing + +@testable import JetlinkORT + +@Suite("QNN policy", .serialized) +struct QnnPolicyTests { + private let model = URL(fileURLWithPath: "/tmp/model.onnx") + private let runtime = String(repeating: "a", count: 64) + + @Test("Default policy preserves existing providers, session configs and cache tags") + func defaults() throws { + let policy = try QnnPolicy(configuration: [:]) + #expect(policy == QnnPolicy()) + for profile in OrtProfile.allCases { + let backend = OrtBackend(profile: profile, preparer: ONNXPreparer(), chip: "example", qnn: policy) + #expect(backend.deviceTag() == "\(profile.rawValue)-example") + #expect(backend.providerOptions(.htp)["backend_type"] == "htp") + #expect(backend.providerOptions(.htp)["backend_path"] == nil) + #expect(backend.providerOptions(.htp)["enable_htp_shared_memory_allocator"] == nil) + #expect(backend.plan(.htp, model: model, cache: nil).config.isEmpty) + } + } + + @Test("Strict configuration reaches the compilation and reload session plans") + func strictBothPhases() { + let backend = OrtBackend(profile: .htpWhole, preparer: ONNXPreparer(), qnn: QnnPolicy(strict: true)) + let key = "session.disable_cpu_ep_fallback" + let context = URL(fileURLWithPath: "/tmp/model_ctx.onnx") + #expect(backend.contextConfiguration(context: context)[key] == "1") + #expect(backend.contextConfiguration(context: context)["ep.context_enable"] == "1") + #expect(backend.contextConfiguration(context: context)["ep.context_file_path"] == context.path) + #expect(backend.plan(.htp, model: model, cache: nil).config == [key: "1"]) + #expect(backend.plan(.qnnGPU, model: model, cache: nil).config.isEmpty) + #expect(backend.plan(.cpu, model: model, cache: nil).config.isEmpty) + } + + @Test("Shared I/O belongs only to HTP, independent of the numerical workaround") + func sharedOptions() { + let backend = OrtBackend(profile: .htpWhole, preparer: ONNXPreparer(), qnn: QnnPolicy(sharedIO: true)) + #expect(backend.providerOptions(.htp)["enable_htp_shared_memory_allocator"] == "1") + #expect(backend.providerOptions(.htp)["backend_type"] == "htp") + #expect(backend.providerOptions(.qnnGPU) == ["backend_type": "gpu"]) + #expect(backend.providerOptions(.cpu).isEmpty) + #expect(backend.providerOptions(.coreML("ALL")).isEmpty) + #expect(backend.plan(.htp, model: model, cache: nil).config.isEmpty) + } + + @Test("The adapter replaces backend discovery and requires matching-runtime metadata") + func adapter() throws { + let policy = QnnPolicy(disableAdvancedActivationFusion: true, libraryDirectory: "/app/native", runtimeIdentity: runtime) + try policy.validate() + let backend = OrtBackend(profile: .htpWhole, preparer: ONNXPreparer(), qnn: policy) + #expect(backend.providerOptions(.htp)["backend_type"] == nil) + #expect(backend.providerOptions(.htp)["backend_path"] == "/app/native/libJetlinkQnnHtpNoFusion.so") + #expect(throws: HostError.self) { try QnnPolicy(disableAdvancedActivationFusion: true).validate() } + #expect(throws: HostError.self) { + try QnnPolicy(disableAdvancedActivationFusion: true, libraryDirectory: "relative", runtimeIdentity: runtime).validate() + } + #expect(throws: HostError.self) { + try QnnPolicy(disableAdvancedActivationFusion: true, libraryDirectory: "/native", runtimeIdentity: "unknown").validate() + } + } + + @Test("All option combinations and SDK bytes have distinct HTP cache namespaces") + func caches() { + var tags = Set() + for shared in [false, true] { + for strict in [false, true] { + for fusion in [false, true] { + let policy = QnnPolicy(sharedIO: shared, strict: strict, disableAdvancedActivationFusion: fusion, + libraryDirectory: "/native", runtimeIdentity: runtime) + let backend = OrtBackend(profile: .htpWhole, preparer: ONNXPreparer(), chip: "example", qnn: policy) + tags.insert(backend.deviceTag()) + #expect(OrtBackend(profile: .cpu, preparer: ONNXPreparer(), chip: "example", qnn: policy).deviceTag() == "cpu-example") + } + } + } + #expect(tags.count == 8) + let first = QnnPolicy(disableAdvancedActivationFusion: true, libraryDirectory: "/native", runtimeIdentity: runtime) + let second = QnnPolicy(disableAdvancedActivationFusion: true, libraryDirectory: "/native", runtimeIdentity: String(repeating: "b", count: 64)) + #expect(first.cacheSuffix != second.cacheSuffix) + } + + @Test("JSON accepts actual Booleans and rejects strings and numeric lookalikes") + func typedConfiguration() throws { + let object = try JSONSerialization.jsonObject(with: Data("{\"qnn_shared_io\":true,\"qnn_strict\":false}".utf8)) as! [String: Any] + #expect(try QnnPolicy(configuration: object).sharedIO) + let invalid: [Any] = ["true", "false", 1, 0, NSNull()] + for key in ["qnn_shared_io", "qnn_strict", "qnn_no_activation_fusion"] { + for value in invalid { + #expect(throws: HostError.self) { _ = try QnnPolicy(configuration: [key: value]) } + } + } + } +} diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 058b1c99..d6c4eaae 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -22,6 +22,47 @@ val jetlinkVersionCode: Int = jetlinkVersion.split(".").map { it.takeWhile(Char::isDigit).toIntOrNull() ?: 0 } .let { (it.getOrElse(0) { 0 } * 10000) + (it.getOrElse(1) { 0 } * 100) + it.getOrElse(2) { 0 } } +// All options are build-time opt-ins; ordinary releases retain the Maven runtime. +fun qnnFlag(name: String): Boolean = providers.gradleProperty(name).orNull?.let { + it.toBooleanStrictOrNull() ?: error("$name must be true or false") +} ?: false +val qnnSharedIO = qnnFlag("jetlink.qnnSharedIO") +val qnnStrict = qnnFlag("jetlink.qnnStrict") +val qnnNoActivationFusion = qnnFlag("jetlink.qnnNoActivationFusion") +val qnnJniDir = providers.gradleProperty("jetlink.qnnJniDir").orNull +val qnnJniRoot = qnnJniDir?.let { file(it) } +val qnnRuntimeIdentity = if (qnnNoActivationFusion) { + check(qnnJniDir != null) { "no-fusion requires one matching SDK; see docs/android-qnn-npu.md" } + val directory = File(qnnJniRoot!!, "arm64-v8a") + val libraries = directory.listFiles()?.filter { it.isFile && it.extension == "so" }?.sortedBy { it.name }.orEmpty() + val names = libraries.map { it.name }.toSet() + check(names.containsAll(listOf("libJetlinkQnnHtpNoFusion.so", "libQnnHtp.so", "libQnnGpu.so", "libQnnSystem.so", + "libQnnHtpPrepare.so", "libQnnIr.so", "libQnnSaver.so"))) { "incomplete SDK jniLibs directory: $directory" } + val stub = names.filter { it.matches(Regex("libQnnHtpV\\d+Stub\\.so")) }.map { it.removeSuffix("Stub.so") }.toSet() + val skel = names.filter { it.matches(Regex("libQnnHtpV\\d+Skel\\.so")) }.map { it.removeSuffix("Skel.so") }.toSet() + check(stub.isNotEmpty() && stub == skel) { "matching HTP stub/skel pairs are required" } + // Actual runtime bytes form a cache namespace, not a preselected SDK/SoC pin. + val identity = MessageDigest.getInstance("SHA-256") + for (library in libraries) { + val digest = MessageDigest.getInstance("SHA-256") + library.inputStream().use { stream -> + val buffer = ByteArray(1024 * 1024) + while (true) { + val count = stream.read(buffer) + if (count < 0) break + digest.update(buffer, 0, count) + } + } + identity.update(library.name.toByteArray(Charsets.UTF_8)) + identity.update(0.toByte()) + identity.update(digest.digest()) + } + identity.digest().joinToString("") { "%02x".format(it) } +} else { + check(qnnJniDir == null) { "jetlink.qnnJniDir requires jetlink.qnnNoActivationFusion=true" } + "" +} + android { namespace = "io.zoompilot.jetlink" compileSdk = 37 @@ -36,6 +77,10 @@ android { targetSdk = 36 versionCode = jetlinkVersionCode versionName = jetlinkVersion + buildConfigField("boolean", "QNN_SHARED_IO", qnnSharedIO.toString()) + buildConfigField("boolean", "QNN_STRICT", qnnStrict.toString()) + buildConfigField("boolean", "QNN_NO_ACTIVATION_FUSION", qnnNoActivationFusion.toString()) + buildConfigField("String", "QNN_RUNTIME_IDENTITY", "\"$qnnRuntimeIdentity\"") // The QNN runtime is arm64 only, and so is every Snapdragon with an NPU // worth driving on. The emulator on an Apple silicon Mac is arm64 too. ndk { abiFilters += "arm64-v8a" } @@ -84,6 +129,8 @@ android { } } + if (qnnNoActivationFusion) sourceSets.getByName("main").jniLibs.srcDir(qnnJniRoot!!) + compileOptions { sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 @@ -132,7 +179,9 @@ dependencies { implementation(libs.kotlinx.coroutines.android) implementation(libs.kotlinx.serialization.json) // libonnxruntime.so with the QNN provider, and the QNN runtime's libraries - implementation(libs.onnxruntime.qnn) + implementation(libs.onnxruntime.qnn) { + if (qnnNoActivationFusion) exclude(group = "com.qualcomm.qti", module = "qnn-runtime") + } debugImplementation(libs.compose.ui.tooling) onnxruntimeAar(variantOf(libs.onnxruntime.qnn) { artifactType("aar") }) diff --git a/android/app/src/main/java/io/zoompilot/jetlink/server/ServerController.kt b/android/app/src/main/java/io/zoompilot/jetlink/server/ServerController.kt index c14ae1cf..b093f844 100644 --- a/android/app/src/main/java/io/zoompilot/jetlink/server/ServerController.kt +++ b/android/app/src/main/java/io/zoompilot/jetlink/server/ServerController.kt @@ -4,6 +4,7 @@ import android.content.Context import android.net.Uri import android.os.Build import android.util.Log +import io.zoompilot.jetlink.BuildConfig import io.zoompilot.jetlink.settings.Chip import io.zoompilot.jetlink.settings.SettingsValues import kotlinx.coroutines.CoroutineScope @@ -99,6 +100,10 @@ class ServerController(private val context: Context, private val scope: Coroutin // a model compiled for the NPU is good for this system build alone put("firmware", Build.FINGERPRINT) put("native_library_dir", context.applicationInfo.nativeLibraryDir) + put("qnn_shared_io", BuildConfig.QNN_SHARED_IO) + put("qnn_strict", BuildConfig.QNN_STRICT) + put("qnn_no_activation_fusion", BuildConfig.QNN_NO_ACTIVATION_FUSION) + put("qnn_runtime_identity", BuildConfig.QNN_RUNTIME_IDENTITY) } /** Follows the snapshot and the log for as long as the app runs. */ diff --git a/android/qnn/build.sh b/android/qnn/build.sh new file mode 100644 index 00000000..62aa1572 --- /dev/null +++ b/android/qnn/build.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Use one locally obtained SDK for headers AND every packaged QNN library. +# No vendor code or model is downloaded or stored in this repository. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +: "${QNN_SDK_ROOT:?set QNN_SDK_ROOT to the matching SDK}" +: "${ANDROID_NDK_HOME:?set ANDROID_NDK_HOME to the Android NDK}" +OUT=${1:?usage: android/qnn/build.sh NEW_OUTPUT_JNILIBS_DIRECTORY} +[[ ! -e "$OUT" ]] || { echo "Output must not exist: $OUT" >&2; exit 1; } +bash "$HERE/test.sh" +INCLUDE="$QNN_SDK_ROOT/include/QNN" +[[ -d "$INCLUDE" ]] || INCLUDE="$QNN_SDK_ROOT/include" +TOOLCHAIN="$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin" +mkdir -p "$OUT/arm64-v8a" +"$TOOLCHAIN/aarch64-linux-android31-clang" -std=gnu11 -Wall -Wextra -Werror \ + -I "$INCLUDE" -fPIC -shared "$HERE/htp_no_activation_fusion.c" \ + -ldl -pthread -Wl,-z,relro,-z,now -o "$OUT/arm64-v8a/libJetlinkQnnHtpNoFusion.so" +python3 - "$QNN_SDK_ROOT" "$OUT/arm64-v8a" <<'PY' +import hashlib +from pathlib import Path +import re +import shutil +import sys +sdk, out = map(Path, sys.argv[1:]) +core = {'libQnnHtp.so', 'libQnnGpu.so', 'libQnnSystem.so', 'libQnnHtpPrepare.so', 'libQnnIr.so', 'libQnnSaver.so'} +found = {} +for path in sdk.rglob('libQnn*.so'): + name = path.name + skel = re.fullmatch(r'libQnnHtpV\d+Skel\.so', name) + stub = re.fullmatch(r'libQnnHtpV\d+Stub\.so', name) + if name not in core and not skel and not stub: + continue + if skel: + if 'unsigned' not in path.parts: + continue + elif 'aarch64-android' not in path.parts: + continue + with path.open('rb') as stream: + header = stream.read(20) + machine = 164 if skel else 183 + if header[:4] != b'\x7fELF' or header[5] != 1 or int.from_bytes(header[18:20], 'little') != machine: + continue + with path.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + if name in found and found[name][0] != digest: + raise SystemExit(f'Ambiguous SDK library: {name}; choose one SDK root') + found[name] = (digest, path) +missing = core - found.keys() +stubs = {name.removesuffix('Stub.so') for name in found if name.endswith('Stub.so')} +skels = {name.removesuffix('Skel.so') for name in found if name.endswith('Skel.so')} +if missing or not stubs or stubs != skels: + raise SystemExit(f'Incomplete matching SDK: missing={sorted(missing)}, stubs={sorted(stubs)}, skels={sorted(skels)}') +for _, path in found.values(): + shutil.copyfile(path, out / path.name) +print(f'Staged adapter and {len(found)} libraries from one SDK; Gradle fingerprints their actual bytes for cache identity.') +PY diff --git a/android/qnn/htp_no_activation_fusion.c b/android/qnn/htp_no_activation_fusion.c new file mode 100644 index 00000000..fb0a5cfb --- /dev/null +++ b/android/qnn/htp_no_activation_fusion.c @@ -0,0 +1,124 @@ +// Public QNN API adapter; build against the same SDK as the packaged backend. +// This is a numerical workaround, independent of shared I/O and scheduling. +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "QnnInterface.h" +#include "HTP/QnnHtpCommon.h" +#include "HTP/QnnHtpGraph.h" + +static pthread_once_t initialized = PTHREAD_ONCE_INIT; +static void *backend; +static QnnInterface_t wrapped; +static const QnnInterface_t *providers[] = {&wrapped}; +static QnnGraph_CreateFn_t real_create; +static QnnContext_FreeFn_t real_context_free; +static Qnn_ErrorHandle_t init_status = QNN_COMMON_ERROR_GENERAL; + +// Some backends consume graph configuration during finalization. Keep both +// the list and the custom configuration alive until contextFree succeeds. +typedef struct ConfigStorage { + struct ConfigStorage *next; + Qnn_ContextHandle_t context; + QnnHtpGraph_CustomConfig_t custom; + QnnGraph_Config_t config; + const QnnGraph_Config_t *all[]; +} ConfigStorage; +static ConfigStorage *retained; +static pthread_mutex_t retained_mutex = PTHREAD_MUTEX_INITIALIZER; + +static Qnn_ErrorHandle_t context_free(Qnn_ContextHandle_t context, Qnn_ProfileHandle_t profile) { + Qnn_ErrorHandle_t status = real_context_free(context, profile); + if (status != QNN_SUCCESS) return status; + pthread_mutex_lock(&retained_mutex); + ConfigStorage **link = &retained; + while (*link != NULL) { + ConfigStorage *entry = *link; + if (entry->context == context) { + *link = entry->next; + free(entry); + } else { + link = &entry->next; + } + } + pthread_mutex_unlock(&retained_mutex); + return status; +} + +static Qnn_ErrorHandle_t graph_create(Qnn_ContextHandle_t context, const char *name, + const QnnGraph_Config_t **original, Qnn_GraphHandle_t *graph) { + size_t count = 0; + if (original != NULL) { + while (count < 128 && original[count] != NULL) ++count; + if (count == 128) return QNN_COMMON_ERROR_INVALID_ARGUMENT; + } + ConfigStorage *storage = calloc(1, sizeof(*storage) + (count + 2) * sizeof(*storage->all)); + if (storage == NULL) return QNN_COMMON_ERROR_MEM_ALLOC; + storage->context = context; + storage->custom.option = QNN_HTP_GRAPH_CONFIG_OPTION_ADVANCED_ACTIVATION_FUSION; + storage->custom.advancedActivationFusion = false; + storage->config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; + storage->config.customConfig = &storage->custom; + for (size_t i = 0; i < count; ++i) storage->all[i] = original[i]; + storage->all[count] = &storage->config; + Qnn_ErrorHandle_t status = real_create(context, name, storage->all, graph); + if (status == QNN_SUCCESS) { + pthread_mutex_lock(&retained_mutex); + storage->next = retained; + retained = storage; + pthread_mutex_unlock(&retained_mutex); + } else { + free(storage); + } + return status; +} + +static void initialize(void) { + Dl_info location; + if (dladdr((void *)&initialize, &location) == 0 || location.dli_fname == NULL) return; + const char *slash = strrchr(location.dli_fname, '/'); + if (slash == NULL) return; + char path[4096]; + if (snprintf(path, sizeof(path), "%.*s/libQnnHtp.so", (int)(slash - location.dli_fname), location.dli_fname) >= (int)sizeof(path)) return; + backend = dlopen(path, RTLD_NOW | RTLD_LOCAL); + if (backend == NULL) { + fprintf(stderr, "QNN no-fusion adapter: cannot open the sibling HTP backend: %s\n", dlerror()); + return; + } + typedef Qnn_ErrorHandle_t (*GetProviders)(const QnnInterface_t ***, uint32_t *); + GetProviders get = (GetProviders)dlsym(backend, "QnnInterface_getProviders"); + if (get == NULL) return; + const QnnInterface_t **list = NULL; + uint32_t count = 0; + if (get(&list, &count) != QNN_SUCCESS || list == NULL) return; + for (uint32_t i = 0; i < count; ++i) { + const QnnInterface_t *p = list[i]; + if (p == NULL || p->backendId != QNN_BACKEND_ID_HTP || + p->apiVersion.coreApiVersion.major != QNN_API_VERSION_MAJOR || + p->apiVersion.coreApiVersion.minor < QNN_API_VERSION_MINOR) continue; + wrapped = *p; + real_create = p->QNN_INTERFACE_VER_NAME.graphCreate; + real_context_free = p->QNN_INTERFACE_VER_NAME.contextFree; + if (real_create == NULL || real_context_free == NULL) return; + wrapped.QNN_INTERFACE_VER_NAME.graphCreate = graph_create; + wrapped.QNN_INTERFACE_VER_NAME.contextFree = context_free; + init_status = QNN_SUCCESS; + return; + } +} + +Qnn_ErrorHandle_t QnnInterface_getProviders(const QnnInterface_t ***out, uint32_t *count) { + if (out == NULL || count == NULL) return QNN_COMMON_ERROR_INVALID_ARGUMENT; + *out = NULL; + *count = 0; + pthread_once(&initialized, initialize); + if (init_status != QNN_SUCCESS) return init_status; + *out = providers; + *count = 1; + return QNN_SUCCESS; +} diff --git a/android/qnn/test.sh b/android/qnn/test.sh new file mode 100644 index 00000000..6beb3d37 --- /dev/null +++ b/android/qnn/test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Host-only public-API forwarding/lifetime test; no accelerator execution. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +: "${QNN_SDK_ROOT:?set QNN_SDK_ROOT to the matching SDK}" +INCLUDE="$QNN_SDK_ROOT/include/QNN" +[[ -d "$INCLUDE" ]] || INCLUDE="$QNN_SDK_ROOT/include" +BUILD=$(mktemp -d) +trap 'rm -rf "$BUILD"' EXIT +FLAGS=(-std=gnu11 -Wall -Wextra -Werror -I "$INCLUDE") +cc "${FLAGS[@]}" -fPIC -shared "$HERE/test_htp_no_activation_fusion.c" -DFAKE_BACKEND -o "$BUILD/libQnnHtp.so" +cc "${FLAGS[@]}" -fPIC -shared "$HERE/htp_no_activation_fusion.c" -ldl -pthread -o "$BUILD/libJetlinkQnnHtpNoFusion.so" +cc "${FLAGS[@]}" "$HERE/test_htp_no_activation_fusion.c" -ldl -o "$BUILD/test-no-fusion" +"$BUILD/test-no-fusion" "$BUILD/libJetlinkQnnHtpNoFusion.so" diff --git a/android/qnn/test_htp_no_activation_fusion.c b/android/qnn/test_htp_no_activation_fusion.c new file mode 100644 index 00000000..9a7cc672 --- /dev/null +++ b/android/qnn/test_htp_no_activation_fusion.c @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include +#include "QnnInterface.h" +#include "HTP/QnnHtpCommon.h" +#include "HTP/QnnHtpGraph.h" + +#ifdef FAKE_BACKEND +static const QnnGraph_Config_t **saved[4]; +static Qnn_ContextHandle_t contexts[4]; +static size_t saved_count; +static bool first_free = true; +static void check(const QnnGraph_Config_t **configs) { + assert(configs && configs[0] && configs[1] && !configs[2]); + assert(configs[0]->option == QNN_GRAPH_CONFIG_OPTION_PRIORITY); + assert(configs[1]->option == QNN_GRAPH_CONFIG_OPTION_CUSTOM); + const QnnHtpGraph_CustomConfig_t *c = configs[1]->customConfig; + assert(c->option == QNN_HTP_GRAPH_CONFIG_OPTION_ADVANCED_ACTIVATION_FUSION); + assert(!c->advancedActivationFusion); +} +static Qnn_ErrorHandle_t create(Qnn_ContextHandle_t context, const char *name, + const QnnGraph_Config_t **configs, Qnn_GraphHandle_t *graph) { + check(configs); + if (strcmp(name, "fail") == 0) return QNN_COMMON_ERROR_GENERAL; + assert(saved_count < 4); + saved[saved_count] = configs; + contexts[saved_count++] = context; + *graph = (void *)0x1; + return QNN_SUCCESS; +} +static Qnn_ErrorHandle_t free_context(Qnn_ContextHandle_t context, Qnn_ProfileHandle_t profile) { + (void)profile; + // Inspect configuration after graphCreate has returned, across contexts and + // after a failed contextFree. The wrapper may release only after success. + for (size_t i = 0; i < saved_count; ++i) if (saved[i]) check(saved[i]); + if (first_free) { first_free = false; return QNN_COMMON_ERROR_GENERAL; } + for (size_t i = 0; i < saved_count; ++i) if (contexts[i] == context) saved[i] = NULL; + return QNN_SUCCESS; +} +Qnn_ErrorHandle_t QnnInterface_getProviders(const QnnInterface_t ***list, uint32_t *count) { + static QnnInterface_t p = QNN_INTERFACE_INIT; + static const QnnInterface_t *ps[] = {&p}; + p.backendId = QNN_BACKEND_ID_HTP; + p.apiVersion.coreApiVersion.major = QNN_API_VERSION_MAJOR; + p.apiVersion.coreApiVersion.minor = QNN_API_VERSION_MINOR; + p.QNN_INTERFACE_VER_NAME.graphCreate = create; + p.QNN_INTERFACE_VER_NAME.contextFree = free_context; + *list = ps; *count = 1; + return QNN_SUCCESS; +} +#else +#include +int main(int argc, char **argv) { + assert(argc == 2); + void *lib = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!lib) { puts(dlerror()); return 1; } + typedef Qnn_ErrorHandle_t (*Get)(const QnnInterface_t ***, uint32_t *); + Get get = (Get)dlsym(lib, "QnnInterface_getProviders"); + assert(get && get(NULL, NULL) == QNN_COMMON_ERROR_INVALID_ARGUMENT); + const QnnInterface_t **p; uint32_t n; + assert(get(&p, &n) == QNN_SUCCESS && n == 1); + QnnGraph_Config_t priority = QNN_GRAPH_CONFIG_INIT; + priority.option = QNN_GRAPH_CONFIG_OPTION_PRIORITY; + const QnnGraph_Config_t *configs[] = {&priority, NULL}; + Qnn_GraphHandle_t graph = NULL; + Qnn_ContextHandle_t a = (void *)0x10, b = (void *)0x20; + assert(p[0]->QNN_INTERFACE_VER_NAME.graphCreate(a, "first", configs, &graph) == QNN_SUCCESS); + assert(p[0]->QNN_INTERFACE_VER_NAME.graphCreate(a, "second", configs, &graph) == QNN_SUCCESS); + assert(p[0]->QNN_INTERFACE_VER_NAME.graphCreate(b, "third", configs, &graph) == QNN_SUCCESS); + assert(p[0]->QNN_INTERFACE_VER_NAME.graphCreate(a, "fail", configs, &graph) == QNN_COMMON_ERROR_GENERAL); + assert(p[0]->QNN_INTERFACE_VER_NAME.contextFree(a, NULL) == QNN_COMMON_ERROR_GENERAL); + assert(p[0]->QNN_INTERFACE_VER_NAME.contextFree(a, NULL) == QNN_SUCCESS); + assert(p[0]->QNN_INTERFACE_VER_NAME.contextFree(b, NULL) == QNN_SUCCESS); + puts("configuration forwarding, graph failure and context lifetime passed"); + return 0; +} +#endif diff --git a/docs/android-qnn-npu.md b/docs/android-qnn-npu.md new file mode 100644 index 00000000..c7bd7b2d --- /dev/null +++ b/docs/android-qnn-npu.md @@ -0,0 +1,76 @@ +# Optional Android QNN HTP policy + +Ordinary builds retain the existing profiles, QNN runtime, provider options and +cache names. Three independent build-time options are available: + +| Gradle property | Effect | +| --- | --- | +| `jetlink.qnnSharedIO=true` | Allocate host tensors through ORT's actual QNN HTP shared allocator and bind them with the matching memory information. | +| `jetlink.qnnStrict=true` | Disable CPU execution-provider fallback in HTP sessions during both compilation and reload. Unsupported graphs fail instead of silently leaving compute on CPU. | +| `jetlink.qnnNoActivationFusion=true` | Use the optional matching-SDK adapter to disable advanced HTP activation fusion, a numerical workaround. | + +All default to false. The policy is typed; JSON configuration accepts Booleans, +not strings or numeric substitutes. GPU and CPU sessions do not register their +bindings as HTP shared memory. Original models, preparation and outputs are +unchanged by this patch. It adds no model-specific route or parallel scheduler. + +For shared I/O and strict HTP admission using the normal runtime: + +```bash +cd android +./gradlew :app:assembleRelease :app:testDebugUnitTest \ + -Pjetlink.qnnSharedIO=true -Pjetlink.qnnStrict=true +``` + +The optional adapter requires locally obtained headers and libraries from one +matching Qualcomm SDK. Vendor headers, libraries and models are not included or +downloaded. Use the project's normal Android/Swift build prerequisites, then: + +```bash +export QNN_SDK_ROOT=/path/to/one/qairt-sdk +export ANDROID_NDK_HOME=/path/to/android-ndk +bash android/qnn/build.sh /path/to/NEW-qnn-jni +cd android +./gradlew :app:assembleRelease :app:testDebugUnitTest \ + -Pjetlink.qnnSharedIO=true -Pjetlink.qnnStrict=true \ + -Pjetlink.qnnNoActivationFusion=true -Pjetlink.qnnJniDir=/path/to/NEW-qnn-jni +``` + +The helper runs a host forwarding/lifetime test, builds the adapter and stages +Android libraries plus available matched HTP stub/skel pairs from that same SDK. +The SDK must expose `QNN_HTP_GRAPH_CONFIG_OPTION_ADVANCED_ACTIVATION_FUSION`. +The adapter forwards the public backend interface and disables only this option. +It retains custom configuration through successful `contextFree`, including when +the backend consumes configuration after graph creation. API incompatibility, +missing libraries, missing allocators or unsupported graphs produce errors. + +Gradle replaces the transitive Maven QNN runtime only in this explicit adapter +build, avoiding a mixture of runtimes. It fingerprints the actual staged library +bytes for the cache namespace; these are not fixed vendor-binary or SoC pins. +The runtime directory and identity flow through `QnnPolicy`, without global +tuning variables. Each changed policy has a different HTP cache namespace. + +## Evidence and limits + +On one Xiaomi 15 / Snapdragon 8 Elite, a historical whole-NPU CTV3 test using +ORT 1.29 and QAIRT 2.47 compared the same range-safe prepared graph with stock +advanced activation fusion versus fusion disabled. The stock capture failed +**six output slices** despite finite values; the disabled-fusion capture passed +all original gates across **32 identical input frames**. This establishes a +numerical workaround for that tested stack, not a universal cause or speedup. +The preparation used in that experiment is not added or changed here. + +A separate sequential one-minute comparison of the same whole-NPU engine +reported ordinary memory **42.9 ms mean / 45.2 ms P99**, versus shared I/O +**40.5 ms mean / 42.2 ms P99**. The mean reduction was about 5.6%. Order, +temperature and clocks were not fully controlled, so this is indicative evidence, +not a general performance guarantee. Strict fallback prevention is a correctness +contract rather than a speed optimization. None of these measurements establishes +50% improvement, 100% NPU occupancy or vehicle readiness. + +Before promoting an option on another graph/runtime/device, retain original +precision and use the unchanged original-model parity gates on identical inputs +with independent recurrent state, including reset tests. Compare alternating +heap/shared runs under similar temperatures, then perform sustained and cold-load +tests. Record exact runtime/options, package/native identities and mean/P99/max. +The current patch's host tests do not qualify an HTP device or a car connection.