diff --git a/packages/serve-sim/Sources/SimNative/HIDInjector.swift b/packages/serve-sim/Sources/SimNative/HIDInjector.swift index 86a546fbd..ea38b92f3 100644 --- a/packages/serve-sim/Sources/SimNative/HIDInjector.swift +++ b/packages/serve-sim/Sources/SimNative/HIDInjector.swift @@ -182,15 +182,22 @@ actor HIDInjector { static let edgeRight: UInt32 = 4 // Right edge /// Synchronously hand an already-built Indigo message to the guest, freeing it. - /// Must run on `inputQueue`. - private func rawSend(_ msg: UnsafeMutableRawPointer) { - guard let client = hidClient, let sendSel = sendSel else { free(msg); return } + /// Must run on `inputQueue`. Returns false when there is no HID client to + /// send through (setup never completed), so callers can report the drop. + @discardableResult + private func rawSend(_ msg: UnsafeMutableRawPointer) -> Bool { + guard let client = hidClient, let sendSel = sendSel else { + print("[hid] Dropped HID message: no SimDeviceLegacyHIDClient") + free(msg) + return false + } typealias SendFunc = @convention(c) (AnyObject, Selector, UnsafeMutableRawPointer, ObjCBool, AnyObject?, AnyObject?) -> Void guard let sendIMP = class_getMethodImplementation(object_getClass(client)!, sendSel) else { free(msg) - return + return false } unsafeBitCast(sendIMP, to: SendFunc.self)(client, sendSel, msg, ObjCBool(true), nil, nil) + return true } /// Build a single-finger touch message (normalized 0..1 coords). Pure — safe @@ -209,25 +216,28 @@ actor HIDInjector { /// Synchronously build + send a single touch. For use inside gesture blocks /// already running on `inputQueue`. - private func rawSendTouch(type: String, x: Double, y: Double, edge: UInt32 = 0) { - if let msg = touchMessage(type: type, x: x, y: y, edge: edge) { rawSend(msg) } + @discardableResult + private func rawSendTouch(type: String, x: Double, y: Double, edge: UInt32 = 0) -> Bool { + guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return false } + return rawSend(msg) } - func sendTouch(type: String, x: Double, y: Double, screenWidth: Int, screenHeight: Int, edge: UInt32 = 0) { + /// Returns whether the touch was handed to the HID client. + func sendTouch(type: String, x: Double, y: Double, screenWidth: Int, screenHeight: Int, edge: UInt32 = 0) -> Bool { updateTouchTarget(width: screenWidth, height: screenHeight) - guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return } + guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return false } hidLog("[hid] Sending \(type) at (\(String(format:"%.3f",x)),\(String(format:"%.3f",y)))\(edge > 0 ? " edge=\(edge)" : "")") - rawSend(msg) + return rawSend(msg) } - func sendMultiTouch(type: String, x1: Double, y1: Double, x2: Double, y2: Double, screenWidth: Int, screenHeight: Int) { - guard let mouseFunc = mouseFunc else { return } + func sendMultiTouch(type: String, x1: Double, y1: Double, x2: Double, y2: Double, screenWidth: Int, screenHeight: Int) -> Bool { + guard let mouseFunc = mouseFunc else { return false } let eventType: Int32 switch type { case "begin", "move": eventType = 1 case "end": eventType = 2 - default: return + default: return false } updateTouchTarget(width: screenWidth, height: screenHeight) @@ -236,11 +246,11 @@ actor HIDInjector { var point2 = CGPoint(x: x2, y: y2) guard let rawMsg = mouseFunc(&point1, &point2, touchTarget, eventType, 1.0, 1.0, 0) else { print("[hid] IndigoHIDMessageForMouseNSEvent returned nil for multi-touch \(type)") - return + return false } hidLog("[hid] Multi-touch \(type) f1=(\(String(format:"%.3f",x1)),\(String(format:"%.3f",y1))) f2=(\(String(format:"%.3f",x2)),\(String(format:"%.3f",y2)))") - rawSend(rawMsg) + return rawSend(rawMsg) } // MARK: - Button events @@ -266,13 +276,14 @@ actor HIDInjector { /// Synchronously build + send a hardware-button message. Call only inside an /// `inputQueue` block (button sequences below run there). - private func sendHIDButton(eventSource: Int32, direction: Int32) { - guard let buttonFunc = buttonFunc else { return } + @discardableResult + private func sendHIDButton(eventSource: Int32, direction: Int32) -> Bool { + guard let buttonFunc = buttonFunc else { return false } guard let msg = buttonFunc(eventSource, direction, Self.buttonTargetHardware) else { print("[hid] IndigoHIDMessageForButton returned nil") - return + return false } - rawSend(msg) + return rawSend(msg) } // MARK: - Keyboard events @@ -432,43 +443,46 @@ actor HIDInjector { /// volume up/down, the action button, and the watch side button. /// - phase: "down" / "up" hold the button for natural long-presses (power /// off slider, side-button menus); "press" sends a momentary down+up. - func sendButtonHID(page: UInt32, usage: UInt32, phase: String) async { + /// Returns whether the press was handed to the guest. + @discardableResult + func sendButtonHID(page: UInt32, usage: UInt32, phase: String) async -> Bool { if let duoBridge { switch phase { - case "down": _ = duoBridge.send("key \(page) \(usage) 1") - case "up": _ = duoBridge.send("key \(page) \(usage) 0") + case "down": return duoBridge.send("key \(page) \(usage) 1") + case "up": return duoBridge.send("key \(page) \(usage) 0") case "press": - guard duoBridge.send("key \(page) \(usage) 1") else { return } + guard duoBridge.send("key \(page) \(usage) 1") else { return false } try? await Task.sleep(for: .milliseconds(250)) - _ = duoBridge.send("key \(page) \(usage) 0") - default: break + return duoBridge.send("key \(page) \(usage) 0") + default: return false } - return } guard let arb = hidArbitraryFunc else { print("[hid] Arbitrary HID injection unavailable (page=\(page) usage=\(usage))") - return + return false } let target = Self.buttonHIDTarget - func emit(_ direction: UInt32) { + func emit(_ direction: UInt32) -> Bool { guard let msg = arb(target, page, usage, direction) else { print("[hid] IndigoHIDMessageForHIDArbitrary returned nil (page=\(page) usage=\(usage) dir=\(direction))") - return + return false } - rawSend(msg) + return rawSend(msg) } hidLog("[hid] HID button page=\(page) usage=\(usage) phase=\(phase)") switch phase { - case "down": emit(1) - case "up": emit(2) + case "down": return emit(1) + case "up": return emit(2) default: - emit(1) + guard emit(1) else { return false } try? await Task.sleep(for: .seconds(0.05)) - emit(2) + return emit(2) } } - func sendButton(button: String, deviceUDID: String) async { + /// Returns whether the button was handed to the guest (false for an + /// unknown button or when the HID client / message builder is missing). + func sendButton(button: String, deviceUDID: String) async -> Bool { hidLog("[hid] Sending button: \(button)") switch button { @@ -481,43 +495,45 @@ actor HIDInjector { // unconditionally rather than depending on whether the HID symbol // resolved. await launchSpringBoard(deviceUDID: deviceUDID) + return true case "swipe_home": - sendSwipeHome() + return sendSwipeHome() case "app_switcher": if buttonFunc != nil { // Double home press with delay for app switcher - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) + guard sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) else { return false } sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) try? await Task.sleep(for: .seconds(0.15)) sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) + return sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) } else { print("[hid] App switcher not available (IndigoHIDMessageForButton not loaded)") + return false } case "lock": if duoBridge != nil { - await sendButtonHID(page: 0x0c, usage: 0x30, phase: "press") - return + return await sendButtonHID(page: 0x0c, usage: 0x30, phase: "press") } - sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonUp) + guard sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonDown) else { return false } + return sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonUp) case "siri": // Holding Siri for ~300ms matches Simulator.app's "hold side button // to invoke Siri" gesture; a tap is ignored. - sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonDown) + guard sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonDown) else { return false } try? await Task.sleep(for: .seconds(0.3)) - sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonUp) + return sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonUp) case "side_button": - sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonUp) + guard sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonDown) else { return false } + return sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonUp) default: print("[hid] Unknown button: \(button)") + return false } } @@ -603,7 +619,7 @@ actor HIDInjector { /// Synthesize a swipe-up-from-bottom gesture (Face ID "go home" gesture). /// Uses IndigoHIDEdge.bottom to flag touches as system edge gestures, /// which iOS interprets as the home indicator swipe. - private func sendSwipeHome() { + private func sendSwipeHome() -> Bool { let xPos = 0.5 let yStart = 0.95 let yEnd = 0.35 @@ -612,7 +628,7 @@ actor HIDInjector { let edge = Self.edgeBottom // Touch down at bottom edge - rawSendTouch(type: "begin", x: xPos, y: yStart, edge: edge) + guard rawSendTouch(type: "begin", x: xPos, y: yStart, edge: edge) else { return false } Thread.sleep(forTimeInterval: stepDelay) // Interpolated moves upward @@ -624,7 +640,7 @@ actor HIDInjector { } // Touch up - rawSendTouch(type: "end", x: xPos, y: yEnd, edge: edge) + return rawSendTouch(type: "end", x: xPos, y: yEnd, edge: edge) } /// Launches SpringBoard off the actor, falling back to `simctl`, and returns diff --git a/packages/serve-sim/Sources/SimNative/sim-module.swift b/packages/serve-sim/Sources/SimNative/sim-module.swift index 8d494d3aa..82ef88bb4 100644 --- a/packages/serve-sim/Sources/SimNative/sim-module.swift +++ b/packages/serve-sim/Sources/SimNative/sim-module.swift @@ -33,57 +33,75 @@ private func u32(_ v: Int) -> UInt32 { setupTask = Task { try await injector.setup(deviceUDID: udid, duoHelper: duoHelper) } } + // Every input method waits for `setup` first. The session is created on + // the first /ws connect, so a CLI `tap` lands milliseconds after this + // constructor; without the wait the frame reached the actor before setup + // and was dropped (no HID client yet) while the CLI exited 0 (#136). + // A failed setup (device gone / not booted) now throws to JS instead of + // silently swallowing every event. + @NodeMethod func touch(_ type: String, _ x: Double, _ y: Double, - _ w: Int, _ h: Int, _ edge: Int) async { - await injector.sendTouch(type: type, x: x, y: y, - screenWidth: w, screenHeight: h, edge: u32(edge)) + _ w: Int, _ h: Int, _ edge: Int) async throws -> Bool { + try await setupTask.value + return await injector.sendTouch(type: type, x: x, y: y, + screenWidth: w, screenHeight: h, edge: u32(edge)) } @NodeMethod func multiTouch(_ type: String, _ x1: Double, _ y1: Double, - _ x2: Double, _ y2: Double, _ w: Int, _ h: Int) async { - await injector.sendMultiTouch(type: type, x1: x1, y1: y1, x2: x2, y2: y2, - screenWidth: w, screenHeight: h) + _ x2: Double, _ y2: Double, _ w: Int, _ h: Int) async throws -> Bool { + try await setupTask.value + return await injector.sendMultiTouch(type: type, x1: x1, y1: y1, x2: x2, y2: y2, + screenWidth: w, screenHeight: h) } - @NodeMethod func button(_ button: String) async { - await injector.sendButton(button: button, deviceUDID: udid) + @NodeMethod func button(_ button: String) async throws -> Bool { + try await setupTask.value + return await injector.sendButton(button: button, deviceUDID: udid) } - @NodeMethod func buttonHid(_ page: Int, _ usage: Int, _ phase: String) async { - await injector.sendButtonHID(page: u32(page), usage: u32(usage), phase: phase) + @NodeMethod func buttonHid(_ page: Int, _ usage: Int, _ phase: String) async throws -> Bool { + try await setupTask.value + return await injector.sendButtonHID(page: u32(page), usage: u32(usage), phase: phase) } - @NodeMethod func key(_ type: String, _ usage: Int) async { + @NodeMethod func key(_ type: String, _ usage: Int) async throws { + try await setupTask.value await injector.sendKey(type: type, usage: u32(usage)) } /// NaN anchorX/anchorY mean "center" (the Swift API's nil). @NodeMethod func scroll(_ dx: Double, _ dy: Double, - _ anchorX: Double, _ anchorY: Double, _ w: Int, _ h: Int) async { + _ anchorX: Double, _ anchorY: Double, _ w: Int, _ h: Int) async throws { + try await setupTask.value await injector.sendScroll(dx: dx, dy: dy, anchorX: anchorX.isNaN ? nil : anchorX, anchorY: anchorY.isNaN ? nil : anchorY, screenWidth: w, screenHeight: h) } - @NodeMethod func digitalCrown(_ delta: Double) async { + @NodeMethod func digitalCrown(_ delta: Double) async throws { + try await setupTask.value await injector.sendDigitalCrown(delta: delta) } - @NodeMethod func orientation(_ orientation: Int) async -> Bool { - await injector.sendOrientation(orientation: u32(orientation)) + @NodeMethod func orientation(_ orientation: Int) async throws -> Bool { + try await setupTask.value + return await injector.sendOrientation(orientation: u32(orientation)) } - @NodeMethod func memoryWarning() async { + @NodeMethod func memoryWarning() async throws { + try await setupTask.value await injector.simulateMemoryWarning() } - @NodeMethod func softwareKeyboard() async { + @NodeMethod func softwareKeyboard() async throws { + try await setupTask.value await injector.toggleSoftwareKeyboard() } - @NodeMethod func caDebug(_ name: String, _ enabled: Bool) async -> Bool { - await injector.setCADebugOption(name: name, enabled: enabled) + @NodeMethod func caDebug(_ name: String, _ enabled: Bool) async throws -> Bool { + try await setupTask.value + return await injector.setCADebugOption(name: name, enabled: enabled) } @NodeMethod func isFoldable() async throws -> Bool { diff --git a/packages/serve-sim/src/__tests__/cli-input-ack.test.ts b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts new file mode 100644 index 000000000..ce5fe1406 --- /dev/null +++ b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts @@ -0,0 +1,146 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { spawn } from "child_process"; +import { createServer, type Server } from "http"; +import type { AddressInfo } from "net"; +import { join } from "path"; +import { WebSocketServer } from "ws"; +import { previewConfigForState } from "../middleware"; +import { inProcessServeSimState } from "../state"; + +// #136: tap/button/gesture used to fire their frame, wait 50ms and exit 0 no +// matter what the server did with it. They now ask for an ack and exit 1 when +// the server reports the injector could not deliver. Servers advertise acks +// with `inputAck` in /api; one that doesn't (an older serve-sim) gets untagged +// frames and the old fast fire-and-forget exit 0. + +const ACK_TAG = 0x84; +const cli = join(import.meta.dir, "..", "index.ts"); + +// ok/fail: advertises acks and replies. close: advertises, then drops the +// socket without replying. hung: advertises but never replies. legacy: an +// older server that neither advertises nor replies. +type Mode = "ok" | "fail" | "close" | "hung" | "legacy"; +const devices: Record = { + "ACK-OK": "ok", + "ACK-FAIL": "fail", + "ACK-CLOSE": "close", + "ACK-HUNG": "hung", + "ACK-LEGACY": "legacy", +}; + +let PORT = 0; +let server: Server; +const frames: Buffer[] = []; + +beforeAll(async () => { + const wss = new WebSocketServer({ noServer: true }); + server = createServer((req, res) => { + const url = new URL(req.url ?? "/", "http://x"); + if (url.pathname === "/api") { + const device = url.searchParams.get("device"); + const advertises = device !== null && devices[device] !== undefined && devices[device] !== "legacy"; + res.writeHead(200, { "Content-Type": "application/json" }) + .end(JSON.stringify({ device, ...(advertises ? { inputAck: true } : {}) })); + return; + } + res.writeHead(404).end(); + }); + server.on("upgrade", (req, socket, head) => { + const device = /^\/helper\/([^/]+)\/ws$/.exec(req.url ?? "")?.[1]; + const mode = device ? devices[device] : undefined; + if (!mode) { socket.destroy(); return; } + wss.handleUpgrade(req, socket, head, (ws) => ws.on("message", (data) => { + const frame = Buffer.from(data as Buffer); + frames.push(frame); + if (mode === "close") { ws.terminate(); return; } + if (mode === "hung" || mode === "legacy") return; + const body = JSON.parse(frame.subarray(1).toString()) as { ack?: number }; + if (typeof body.ack !== "number") return; + ws.send(Buffer.concat([Buffer.from([ACK_TAG]), Buffer.from(JSON.stringify({ ack: body.ack, ok: mode === "ok" }))])); + })); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + PORT = (server.address() as AddressInfo).port; +}); + +afterAll(async () => { + if (!server) return; + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); +}); + +function runCli(args: string[]): Promise<{ status: number | null; stderr: string; ms: number }> { + const env = { ...process.env }; + delete env.SERVE_SIM_URL; + delete env.SERVE_SIM_AUTH_TOKEN; + return new Promise((resolve) => { + const started = Date.now(); + const child = spawn("bun", ["run", cli, ...args, "--url", `http://127.0.0.1:${PORT}`], { env }); + let stderr = ""; + child.stderr.on("data", (d) => (stderr += d)); + const timer = setTimeout(() => child.kill("SIGKILL"), 20_000); + child.on("close", (status) => { + clearTimeout(timer); + resolve({ status, stderr, ms: Date.now() - started }); + }); + }); +} + +describe("input commands propagate HID delivery failures (#136)", () => { + for (const [name, args] of [ + ["tap", ["tap", "0.5", "0.5"]], + ["button", ["button", "lock"]], + ["gesture", ["gesture", '{"type":"begin","x":0.5,"y":0.5}']], + ] as const) { + test(`${name} exits 0 when the server acks delivery`, async () => { + const r = await runCli([...args, "-d", "ACK-OK"]); + expect(r.stderr).toBe(""); + expect(r.status).toBe(0); + }, 25_000); + + test(`${name} exits 1 with a clear error when the injector could not deliver`, async () => { + const r = await runCli([...args, "-d", "ACK-FAIL"]); + expect(r.status).toBe(1); + expect(r.stderr).toContain("could not deliver"); + expect(r.stderr).toContain("ACK-FAIL"); + }, 25_000); + + test(`${name} exits 1 when the server closes the socket before acking`, async () => { + const r = await runCli([...args, "-d", "ACK-CLOSE"]); + expect(r.status).toBe(1); + expect(r.stderr).toContain("closed the connection"); + }, 25_000); + + test(`${name} does not wait for acks from an older server`, async () => { + frames.length = 0; + const r = await runCli([...args, "-d", "ACK-LEGACY"]); + expect(r.status).toBe(0); + // The CLI process itself takes a while to start; the point is that it + // never sits out the 4s ack timeout. + expect(r.ms).toBeLessThan(3_500); + expect(frames.length).toBeGreaterThan(0); + for (const f of frames) expect(JSON.parse(f.subarray(1).toString())).not.toHaveProperty("ack"); + }, 25_000); + } + + test("an advertising server that never acks exits 0 with a warning after the timeout", async () => { + const r = await runCli(["button", "lock", "-d", "ACK-HUNG"]); + expect(r.status).toBe(0); + expect(r.stderr).toContain("serve-sim: no delivery confirmation within 4s; input may not have reached ACK-HUNG"); + expect(r.stderr.trim().split("\n")).toHaveLength(1); + }, 25_000); + + test("tap tags begin and end with distinct ack ids", async () => { + frames.length = 0; + await runCli(["tap", "0.1", "0.2", "-d", "ACK-OK"]); + const bodies = frames.map((f) => JSON.parse(f.subarray(1).toString()) as { type: string; ack: number }); + expect(bodies.map((b) => b.type)).toEqual(["begin", "end"]); + expect(new Set(bodies.map((b) => b.ack)).size).toBe(2); + }, 25_000); +}); + +test("servers advertise input acks in their state file and /api (#136)", () => { + const state = inProcessServeSimState("ADVERTISE", 3999); + expect(state.inputAck).toBe(true); + expect(previewConfigForState(state, "", "serve-sim", false).inputAck).toBe(true); +}); diff --git a/packages/serve-sim/src/__tests__/hid-input-ack.test.ts b/packages/serve-sim/src/__tests__/hid-input-ack.test.ts new file mode 100644 index 000000000..79a92a612 --- /dev/null +++ b/packages/serve-sim/src/__tests__/hid-input-ack.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { DeviceSession, type DeviceSessionDependencies } from "../device-session"; + +// #136: `serve-sim tap/button/gesture` exited 0 even when the HID injector +// could not deliver the event (injector not set up, device gone, unknown +// button). The CLI now tags its frames with an `ack` id and the session answers +// with [0x84]{ ack, ok } once the native send settles. Frames without `ack` +// (the browser viewer) must get no reply so the viewer protocol is unchanged. + +const ACK_TAG = 0x84; + +type Hid = DeviceSessionDependencies["hid"]; + +function session(hid: Partial): { + send: (tag: number, payload: unknown) => void; + replies: Array<{ tag: number; body: Record }>; + calls: unknown[][]; +} { + const calls: unknown[][] = []; + const base: Hid = { + touch: async (...args) => { calls.push(["touch", ...args]); return true; }, + multiTouch: async () => true, + button: async (...args) => { calls.push(["button", ...args]); return true; }, + buttonHid: async (...args) => { calls.push(["buttonHid", ...args]); return true; }, + key: async () => {}, + scroll: async () => {}, + digitalCrown: async () => {}, + orientation: async () => false, + memoryWarning: async () => {}, + softwareKeyboard: async () => {}, + caDebug: async () => false, + pose: async () => false, + hinge: async () => false, + }; + const s = new DeviceSession("ACK-UDID", { + capture: { + start: async () => {}, + stop: async () => {}, + subscribeMjpeg: async () => () => {}, + subscribeAvcc: async () => () => {}, + subscribeSize: async () => () => {}, + snapshotJpeg: async () => null, + setPreferredScreenSize: async () => {}, + }, + hid: { ...base, ...hid }, + }); + const replies: Array<{ tag: number; body: Record }> = []; + let receive!: (data: Buffer) => void; + s.attachHidSocket({ + send(frame: Buffer) { + // Ignore config/projection seeds; only collect input acks. + if (frame[0] === ACK_TAG) replies.push({ tag: frame[0]!, body: JSON.parse(frame.subarray(1).toString()) }); + }, + close() {}, + on(event: string, cb: (data: Buffer) => void) { if (event === "message") receive = cb; }, + } as never); + return { + send: (tag, payload) => receive(Buffer.concat([Buffer.from([tag]), Buffer.from(JSON.stringify(payload))])), + replies, + calls, + }; +} + +const settle = () => new Promise((r) => setTimeout(r, 10)); + +describe("HID input acknowledgements (#136)", () => { + test("a touch the injector delivered is acked ok", async () => { + const s = session({}); + s.send(0x03, { type: "begin", x: 0.5, y: 0.5, ack: 1 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 1, ok: true } }]); + }); + + test("a touch the injector could not deliver is acked not-ok", async () => { + const s = session({ touch: async () => false }); + s.send(0x03, { type: "begin", x: 0.5, y: 0.5, ack: 7 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 7, ok: false } }]); + }); + + test("a button the injector could not deliver is acked not-ok", async () => { + const s = session({ button: async () => false }); + s.send(0x04, { button: "lock", ack: 3 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 3, ok: false } }]); + }); + + test("an arbitrary HID button reports the native result", async () => { + const s = session({ buttonHid: async () => false }); + s.send(0x04, { button: "volume-up", page: 12, usage: 233, ack: 4 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 4, ok: false } }]); + }); + + test("a native throw is acked not-ok instead of escaping", async () => { + const s = session({ touch: async () => { throw new Error("Device not found"); } }); + s.send(0x03, { type: "end", x: 0.5, y: 0.5, ack: 9 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 9, ok: false } }]); + }); + + test("frames without an ack id (browser viewer) get no reply", async () => { + const s = session({ touch: async () => false, button: async () => false }); + s.send(0x03, { type: "begin", x: 0.5, y: 0.5 }); + s.send(0x04, { button: "home" }); + await settle(); + expect(s.replies).toEqual([]); + }); + + test("the ack id is not forwarded to the injector", async () => { + const s = session({}); + s.send(0x04, { button: "lock", ack: 2 }); + s.send(0x03, { type: "begin", x: 0.25, y: 0.75, ack: 5 }); + await settle(); + expect(s.calls[0]).toEqual(["button", "lock"]); + expect(s.calls[1]!.slice(0, 4)).toEqual(["touch", "begin", 0.25, 0.75]); + }); +}); diff --git a/packages/serve-sim/src/device-session.ts b/packages/serve-sim/src/device-session.ts index 2ee6ab7a0..4d2507569 100644 --- a/packages/serve-sim/src/device-session.ts +++ b/packages/serve-sim/src/device-session.ts @@ -65,6 +65,27 @@ const AVCC_SEED_TAG = 0x04; // WS server→client screen-config push (ClientManager.wsMsgConfig). const WS_MSG_CONFIG = 0x82; +// WS server→client input acknowledgement: [0x84]{ ack, ok }. Sent only for +// touch/button frames whose JSON carries a numeric `ack` id (the CLI's +// tap/button/gesture), so the browser viewer's protocol is unchanged (#136). +const WS_MSG_INPUT_ACK = 0x84; +const ACKABLE_INPUT_TAGS = new Set([0x03, 0x04]); + +/** The CLI's `ack` id on a touch/button frame, if it asked for one. */ +function inputAckId(frame: Buffer): number | undefined { + if (!ACKABLE_INPUT_TAGS.has(frame[0]!) || !frame.includes('"ack"', 1)) return undefined; + try { + const ack = (JSON.parse(frame.subarray(1).toString("utf8")) as { ack?: unknown }).ack; + return typeof ack === "number" && Number.isFinite(ack) ? ack : undefined; + } catch { + return undefined; + } +} + +function withoutAck(message: T): Omit { + const { ack: _ack, ...rest } = message; + return rest; +} const MJPEG_TRAILER = Buffer.from("\r\n", "ascii"); // Native capture is event-driven, so an unchanged screen can legitimately @@ -176,12 +197,12 @@ function newStreamHealth(): StreamHealth { function duoRendererErrorBody(error: DuoRendererError): { error: "duo_renderer_unavailable"; reason: string; stage: string; details: Record } { return { error: "duo_renderer_unavailable", reason: redactHome(error.reason), stage: error.stage, details: redactHome(error.details) }; } -type HidTransport = Pick< +/** Delivery-reporting sends resolve true only when the event reached the guest. */ +type ReportsDelivery = F extends (...args: infer A) => Promise ? (...args: A) => Promise : never; +type HidTransport = { + [K in "touch" | "multiTouch" | "button" | "buttonHid"]: ReportsDelivery; +} & Pick< NativeHid, - | "touch" - | "multiTouch" - | "button" - | "buttonHid" | "key" | "scroll" | "digitalCrown" @@ -854,7 +875,22 @@ export class DeviceSession { } }).catch((error) => { console.error("[hid] Pose reply failed:", error); }); } else { - void this.handleHidMessage(frame).catch((error) => { console.error("[hid] Input failed:", error); }); + const ack = inputAckId(frame); + const handled = this.handleHidMessage(frame); + if (ack === undefined) { + void handled.catch((error) => { console.error("[hid] Input failed:", error); }); + } else { + void handled + .then((result) => result === true, (error) => { + console.error("[hid] Input failed:", error); + return false; + }) + .then((ok) => { + if (this.hidSockets.has(ws)) { + ws.send(Buffer.concat([Buffer.from([WS_MSG_INPUT_ACK]), Buffer.from(JSON.stringify({ ack, ok }))])); + } + }); + } } }); ws.on("close", () => this.hidSockets.delete(ws)); @@ -877,24 +913,24 @@ export class DeviceSession { const H = this.height; switch (tag) { + // Touch and button resolve whether the injector delivered the event; the + // socket handler turns that into an ack when the frame asked for one. case 0x03: { - const m = json<{ type: string; x: number; y: number; edge?: number }>(); - if (m) { - this.recordTouchEvent(m); - this.hid.touch(m.type as "begin" | "move" | "end", m.x, m.y, W, H, m.edge ?? 0); - } - break; + const raw = json<{ type: string; x: number; y: number; edge?: number; ack?: number }>(); + if (!raw) return false; + const m = withoutAck(raw); + this.recordTouchEvent(m); + return this.hid.touch(m.type as "begin" | "move" | "end", m.x, m.y, W, H, m.edge ?? 0); } case 0x04: { - const m = json<{ button: string; page?: number; usage?: number; phase?: string }>(); - if (!m) break; + const raw = json<{ button: string; page?: number; usage?: number; phase?: string; ack?: number }>(); + if (!raw) return false; + const m = withoutAck(raw); this.recordHidEvent(tag, m); if (m.page != null && m.usage != null) { - this.hid.buttonHid(m.page, m.usage, (m.phase as "down" | "up" | "press") ?? "press"); - } else { - this.hid.button(m.button); + return this.hid.buttonHid(m.page, m.usage, (m.phase as "down" | "up" | "press") ?? "press"); } - break; + return this.hid.button(m.button); } case 0x05: { const m = json<{ type: string; x1: number; y1: number; x2: number; y2: number }>(); diff --git a/packages/serve-sim/src/index.ts b/packages/serve-sim/src/index.ts index 0d620a6ea..db30732fa 100755 --- a/packages/serve-sim/src/index.ts +++ b/packages/serve-sim/src/index.ts @@ -711,6 +711,8 @@ interface InputTarget { url: string; wsUrl: string; headers: Record; + /** The server sends input delivery acks (#136); older servers don't. */ + inputAck: boolean; } /** @@ -731,7 +733,7 @@ async function resolveInputTarget(deviceArg?: string): Promise { console.error("No serve-sim server running. Run `serve-sim` first."); process.exit(1); } - return { device: state.device, url: state.url, wsUrl: state.wsUrl, headers }; + return { device: state.device, url: state.url, wsUrl: state.wsUrl, headers, inputAck: state.inputAck === true }; } let base: URL; @@ -761,7 +763,7 @@ async function resolveInputTarget(deviceArg?: string): Promise { console.error(`serve-sim at ${base.origin} returned HTTP ${res.status} for /api`); process.exit(1); } - const config = await res.json().catch(() => null) as { device?: string } | null; + const config = await res.json().catch(() => null) as { device?: string; inputAck?: unknown } | null; const device = config?.device; if (!device) { console.error( @@ -773,7 +775,13 @@ async function resolveInputTarget(deviceArg?: string): Promise { } const ws = new URL(`helper/${encodeURIComponent(device)}/ws`, base); ws.protocol = base.protocol === "https:" ? "wss:" : "ws:"; - return { device, url: base.toString().replace(/\/$/, ""), wsUrl: ws.toString(), headers }; + return { + device, + url: base.toString().replace(/\/$/, ""), + wsUrl: ws.toString(), + headers, + inputAck: config.inputAck === true, + }; } /** @@ -957,6 +965,108 @@ async function touchInputOrientation( } } +// Server→client input ack ([0x84]{ ack, ok }), sent by servers that support +// #136 (they advertise `inputAck` in their state / `/api`) for frames carrying +// an `ack` id. Older servers never reply, so input to them stays +// fire-and-forget. INPUT_ACK_TIMEOUT_MS covers HID setup plus the new-client +// settle delay on a session's first input; a missing ack within it warns on +// stderr and exits 0. +const WS_MSG_INPUT_ACK = 0x84; +const INPUT_ACK_TIMEOUT_MS = 4_000; + +/** + * Send touch/button frames and wait for the server to say whether the HID + * injector delivered them. Each frame goes out `delayMs` after the previous + * one regardless of acks (a tap's begin/end spacing must not stretch into a + * long-press). When the server reports a frame was not delivered, or closes + * the socket before acknowledging, prints why and sets exit code 1. A server + * that doesn't advertise acks gets the frames untagged and the socket closes + * 50ms after the last one, as before #136. + */ +function sendAckedInput( + target: InputTarget, + what: string, + frames: Array<{ tag: number; payload: object; delayMs?: number }>, +): Promise { + return new Promise((resolve, reject) => { + const ws = new WebSocket(target.wsUrl, { headers: target.headers }); + ws.binaryType = "arraybuffer"; + const pending = new Set(); + let sentAll = false; + let settled = false; + let timer: ReturnType | undefined; + const finish = (failure?: string) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + ws.onclose = null; + ws.close(); + if (failure) { + console.error(failure); + process.exitCode = 1; + } + resolve(); + }; + const undelivered = () => ( + `serve-sim could not deliver the ${what} to simulator ${target.device}: the HID injector ` + + `reported it was not sent (simulator not booted, HID setup failed, or unsupported input). ` + + `See the serve-sim server log; restart with \`serve-sim --kill ${target.device}\` and start it again.` + ); + ws.onmessage = ({ data }) => { + const frame = Buffer.from(data as ArrayBuffer); + if (frame[0] !== WS_MSG_INPUT_ACK) return; + let reply: { ack?: unknown; ok?: unknown }; + try { reply = JSON.parse(frame.subarray(1).toString()); } catch { return; } + if (typeof reply.ack !== "number" || !pending.delete(reply.ack)) return; + if (reply.ok !== true) return finish(undelivered()); + if (sentAll && pending.size === 0) finish(); + }; + ws.onclose = () => { + finish(`serve-sim closed the connection before the ${what} reached simulator ${target.device}.`); + }; + ws.onerror = () => { + if (settled) return; + console.error("Failed to connect to serve-sim server at", target.wsUrl); + settled = true; + if (timer) clearTimeout(timer); + reject(new Error("WebSocket connection failed")); + }; + ws.onopen = () => { + let ack = 0; + const sendNext = (index: number) => { + if (settled) return; + const { tag, payload } = frames[index]!; + if (target.inputAck) { + ack += 1; + pending.add(ack); + sendHid(ws, tag, { ...payload, ack }); + } else { + sendHid(ws, tag, payload); + } + const next = frames[index + 1]; + if (next) { + setTimeout(() => sendNext(index + 1), next.delayMs ?? 0); + return; + } + sentAll = true; + if (!target.inputAck) { + timer = setTimeout(() => finish(), 50); + return; + } + timer = setTimeout(() => { + debugCli("no input ack from %s after %dms", target.wsUrl, INPUT_ACK_TIMEOUT_MS); + console.error( + `serve-sim: no delivery confirmation within ${INPUT_ACK_TIMEOUT_MS / 1000}s; ` + + `input may not have reached ${target.device}`, + ); + finish(); + }, INPUT_ACK_TIMEOUT_MS); + }; + sendNext(0); + }; + }); +} + async function gesture(jsonStr: string, deviceArg?: string, raw?: boolean) { const state = await resolveInputTarget(deviceArg); @@ -971,24 +1081,7 @@ async function gesture(jsonStr: string, deviceArg?: string, raw?: boolean) { const orientation = await touchInputOrientation(state, raw); const rawTouch = displayTouchToRaw(touch, orientation); - return new Promise((resolve, reject) => { - const ws = new WebSocket(state.wsUrl, { headers: state.headers }); - ws.binaryType = "arraybuffer"; - - ws.onopen = () => { - const json = new TextEncoder().encode(JSON.stringify(rawTouch)); - const msg = new Uint8Array(1 + json.length); - msg[0] = 0x03; - msg.set(json, 1); - ws.send(msg); - setTimeout(() => { ws.close(); resolve(); }, 50); - }; - - ws.onerror = () => { - console.error("Failed to connect to serve-sim server at", state.wsUrl); - reject(new Error("WebSocket connection failed")); - }; - }); + await sendAckedInput(state, "gesture", [{ tag: 0x03, payload: rawTouch }]); } async function tap(xArg: string, yArg: string, deviceArg?: string, raw?: boolean) { @@ -1002,28 +1095,10 @@ async function tap(xArg: string, yArg: string, deviceArg?: string, raw?: boolean } const state = await resolveInputTarget(deviceArg); const orientation = await touchInputOrientation(state, raw); - return new Promise((resolve, reject) => { - const ws = new WebSocket(state.wsUrl, { headers: state.headers }); - ws.binaryType = "arraybuffer"; - const send = (type: "begin" | "end") => { - const json = new TextEncoder().encode(JSON.stringify(displayTouchToRaw({ type, x, y }, orientation))); - const msg = new Uint8Array(1 + json.length); - msg[0] = 0x03; - msg.set(json, 1); - ws.send(msg); - }; - ws.onopen = () => { - send("begin"); - setTimeout(() => { - send("end"); - setTimeout(() => { ws.close(); resolve(); }, 50); - }, 40); - }; - ws.onerror = () => { - console.error("Failed to connect to serve-sim server at", state.wsUrl); - reject(new Error("WebSocket connection failed")); - }; - }); + await sendAckedInput(state, "tap", [ + { tag: 0x03, payload: displayTouchToRaw({ type: "begin", x, y }, orientation) }, + { tag: 0x03, payload: displayTouchToRaw({ type: "end", x, y }, orientation), delayMs: 40 }, + ]); } /** @@ -1318,24 +1393,7 @@ async function button(buttonName = "home", deviceArg?: string) { const hid = HID_BUTTON_CODES[buttonName]; const payload = hid ? { button: buttonName, ...hid } : { button: buttonName }; - return new Promise((resolve, reject) => { - const ws = new WebSocket(state.wsUrl, { headers: state.headers }); - ws.binaryType = "arraybuffer"; - - ws.onopen = () => { - const json = new TextEncoder().encode(JSON.stringify(payload)); - const msg = new Uint8Array(1 + json.length); - msg[0] = 0x04; - msg.set(json, 1); - ws.send(msg); - setTimeout(() => { ws.close(); resolve(); }, 50); - }; - - ws.onerror = () => { - console.error("Failed to connect to serve-sim server at", state.wsUrl); - reject(new Error("WebSocket connection failed")); - }; - }); + await sendAckedInput(state, `${buttonName} button press`, [{ tag: 0x04, payload }]); } // Send a CoreAnimation debug option toggle to the helper, which invokes diff --git a/packages/serve-sim/src/native.ts b/packages/serve-sim/src/native.ts index a6eef7717..6fec1b18c 100644 --- a/packages/serve-sim/src/native.ts +++ b/packages/serve-sim/src/native.ts @@ -25,10 +25,11 @@ const require = createRequire(import.meta.url); // handle is garbage-collected (Swift `deinit`), so there are no explicit // destroy/free calls here. interface SimHIDHandle { - touch(type: TouchType, x: number, y: number, w: number, hh: number, edge: number): Promise; - multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, hh: number): Promise; - button(button: string): Promise; - buttonHid(page: number, usage: number, phase: ButtonPhase): Promise; + /** Resolves whether the event reached the HID client (addons before #136 resolve undefined). */ + touch(type: TouchType, x: number, y: number, w: number, hh: number, edge: number): Promise; + multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, hh: number): Promise; + button(button: string): Promise; + buttonHid(page: number, usage: number, phase: ButtonPhase): Promise; key(type: KeyType, usage: number): Promise; scroll(dx: number, dy: number, anchorX: number, anchorY: number, w: number, hh: number): Promise; digitalCrown(delta: number): Promise; @@ -193,6 +194,13 @@ function load(): NativeAddon { return addon; } +// How long a new HID client is given before touch/button input is sent to it. +// A heuristic, not a guarantee: measured on iOS 27.2 from only a few runs (a tap +// 0.5s after the session was created landed 1/2 times; at 1.5s, 2/2 and every +// later tap landed). If the first tap after a fresh start is flaky on a slower +// machine or runtime, suspect this value first. +const GUEST_HID_SETTLE_MS = 1_500; + /** * In-process HID injector for one simulator. Mirrors the WebSocket HID protocol * the spawned helper used to handle, but as direct native calls. @@ -208,6 +216,16 @@ export interface NativeHidDeps { export class NativeHid { private readonly handle: SimHIDHandle; private readonly keys: KeySequencer; + /** + * Resolves once HID setup finished and the guest had time to pick up the + * new HID client. A brand-new SimDeviceLegacyHIDClient accepts messages + * (the send completion reports no error) but the guest ignores input for + * roughly the first second, so `serve-sim tap` right after a fresh start + * was silently dropped (#136). Touch/button input waits on this one shared + * promise; JS resolves its waiters in FIFO order, so a tap's begin/end can't + * be reordered. Later input passes through immediately. + */ + private readonly ready: Promise; /** Settles once a Duo's bridge points at the launchable helper. */ private readonly duoHelperReady: Promise; private duoHelperSettled = false; @@ -216,14 +234,24 @@ export class NativeHid { const bundled = (deps.bundledDuoHelper ?? bundledDuoHidHelperPath)(); this.handle = new (deps.SimHID ?? load().SimHID)(udid, bundled); this.keys = new KeySequencer(this.handle); + const setup = this.handle.isFoldable(); + this.ready = setup.then( + () => new Promise((resolve) => setTimeout(resolve, GUEST_HID_SETTLE_MS).unref?.()), + () => {}, // setup failed: let the sends run so they report the failure + ); const resolve = deps.duoHelper ?? duoHidHelperPath; - this.duoHelperReady = this.handle.isFoldable().then(async (foldable) => { + this.duoHelperReady = setup.then(async (foldable) => { if (!foldable) return; const helper = resolve(); if (helper !== bundled) await this.handle.setDuoHelper(helper); }).catch(() => {}).finally(() => { this.duoHelperSettled = true; }); } + private async settled(fn: () => PromiseLike): Promise { + await this.ready; + return fn(); + } + /** * Commands a Duo routes through its guest helper wait for it to be resolved. * Once it has been, they run immediately so they keep their order relative @@ -239,30 +267,39 @@ export class NativeHid { // so an unhandled throw here crashes the whole server — and if it lands // mid-gesture, the guest is left with a stuck finger that wedges input until // the sim reboots. The spawned helper used to absorb this in its own process; - // `guard` restores that isolation by swallowing malformed-input errors. + // `guard` restores that isolation by swallowing malformed-input errors (and + // HID setup failures, which every input method now awaits). private async guard(op: string, fn: () => PromiseLike, fallback: T): Promise { try { return await fn(); } catch (err) { - console.error(`[hid] ${op} ignored bad input:`, err instanceof Error ? err.message : err); + console.error(`[hid] ${op} failed:`, err instanceof Error ? err.message : err); return fallback; } } - touch(type: TouchType, x: number, y: number, w: number, h: number, edge = 0): Promise { - return this.guard("touch", () => this.handle.touch(type, x, y, w, h, edge), undefined); + // The four sends below resolve whether the injector handed the event to the + // guest: false when it couldn't (HID setup failed, message builder missing, + // unknown button) so `serve-sim tap/button/gesture` can exit non-zero (#136). + + touch(type: TouchType, x: number, y: number, w: number, h: number, edge = 0): Promise { + return this.delivered("touch", () => this.settled(() => this.handle.touch(type, x, y, w, h, edge))); + } + + multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, h: number): Promise { + return this.delivered("multiTouch", () => this.settled(() => this.handle.multiTouch(type, x1, y1, x2, y2, w, h))); } - multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, h: number): Promise { - return this.guard("multiTouch", () => this.handle.multiTouch(type, x1, y1, x2, y2, w, h), undefined); + button(button: string): Promise { + return this.delivered("button", () => this.settled(this.viaDuoHelper(() => this.handle.button(button)))); } - button(button: string): Promise { - return this.guard("button", this.viaDuoHelper(() => this.handle.button(button)), undefined); + buttonHid(page: number, usage: number, phase: ButtonPhase = "press"): Promise { + return this.delivered("buttonHid", () => this.settled(this.viaDuoHelper(() => this.handle.buttonHid(page, usage, phase)))); } - buttonHid(page: number, usage: number, phase: ButtonPhase = "press"): Promise { - return this.guard("buttonHid", this.viaDuoHelper(() => this.handle.buttonHid(page, usage, phase)), undefined); + private async delivered(op: string, fn: () => PromiseLike): Promise { + return (await this.guard(op, fn, false)) !== false; } key(type: KeyType, usage: number): Promise { @@ -271,11 +308,11 @@ export class NativeHid { /** anchorX/anchorY default to screen center when omitted. */ scroll(dx: number, dy: number, w: number, h: number, anchorX?: number, anchorY?: number): Promise { - return this.guard("scroll", () => this.handle.scroll(dx, dy, anchorX ?? NaN, anchorY ?? NaN, w, h), undefined); + return this.guard("scroll", () => this.settled(() => this.handle.scroll(dx, dy, anchorX ?? NaN, anchorY ?? NaN, w, h)), undefined); } digitalCrown(delta: number): Promise { - return this.guard("digitalCrown", () => this.handle.digitalCrown(delta), undefined); + return this.guard("digitalCrown", () => this.settled(() => this.handle.digitalCrown(delta)), undefined); } orientation(orientation: number): Promise { @@ -287,7 +324,7 @@ export class NativeHid { } softwareKeyboard(): Promise { - return this.guard("softwareKeyboard", () => this.handle.softwareKeyboard(), undefined); + return this.guard("softwareKeyboard", () => this.settled(() => this.handle.softwareKeyboard()), undefined); } caDebug(name: string, enabled: boolean): Promise { diff --git a/packages/serve-sim/src/state.ts b/packages/serve-sim/src/state.ts index 54d1e6f3e..20b2311bc 100644 --- a/packages/serve-sim/src/state.ts +++ b/packages/serve-sim/src/state.ts @@ -53,6 +53,12 @@ export interface ServeSimDeviceState { * each can tell only whether its own set's simulators are booted. */ deviceSet?: string; + /** + * The server replies to input frames carrying an `ack` id with a delivery + * report (#136). Absent on servers older than that, which never reply, so + * the input subcommands only wait for acks when this is set. + */ + inputAck?: boolean; } /** @@ -81,6 +87,7 @@ export function inProcessServeSimState( url: `http://${h}:${port}`, streamUrl: `http://${h}:${port}${prefix}/helper/${udid}/stream.mjpeg`, wsUrl: `ws://${h}:${port}${prefix}/helper/${udid}/ws`, + inputAck: true, }; }