From 0e97eb1e16c72377aa70c18e3f1aced3989729f9 Mon Sep 17 00:00:00 2001 From: Evan Bacon Date: Mon, 9 Nov 2026 19:11:54 -0800 Subject: [PATCH 1/3] fix(serve-sim): report undelivered tap/button/gesture input instead of exiting 0 (#136) `serve-sim tap`, `button` and `gesture` sent their WebSocket frame, waited 50ms and exited 0 whatever happened server-side. The native SimHID sends returned nothing and did not wait for HID setup, so input that never reached the guest (setup failed, unknown button, message builder missing) was dropped silently. - Native: input methods await HID setup and touch/multiTouch/button/buttonHid return whether the event was handed to the HID client. A failed setup now throws to JS instead of every event being swallowed. - Server: touch/button frames carrying an `ack` id get a [0x84]{ack, ok} reply once the native send settles. Frames without `ack` (the browser viewer) get no reply, so the viewer protocol is unchanged. - CLI: tap/button/gesture tag frames with ack ids and exit 1 with a clear error when the server reports a frame was not delivered or closes the socket first. A server that never acks (older serve-sim) keeps exit 0 after a 4s wait. - A brand-new SimDeviceLegacyHIDClient accepts messages but the guest ignores input for about the first second, so the first `tap` after a fresh start was dropped (0/4 landed on iOS 27.2). Touch/button input now waits up to 1.5s after HID setup on a new session (5/5 landed); later input is not delayed. --- .../Sources/SimNative/HIDInjector.swift | 123 +++++++++------ .../Sources/SimNative/sim-module.swift | 56 ++++--- .../src/__tests__/cli-input-ack.test.ts | 105 +++++++++++++ .../src/__tests__/hid-input-ack.test.ts | 118 ++++++++++++++ packages/serve-sim/src/device-session.ts | 72 ++++++--- packages/serve-sim/src/index.ts | 148 +++++++++++------- packages/serve-sim/src/native.ts | 68 ++++++-- 7 files changed, 527 insertions(+), 163 deletions(-) create mode 100644 packages/serve-sim/src/__tests__/cli-input-ack.test.ts create mode 100644 packages/serve-sim/src/__tests__/hid-input-ack.test.ts diff --git a/packages/serve-sim/Sources/SimNative/HIDInjector.swift b/packages/serve-sim/Sources/SimNative/HIDInjector.swift index 947c6b788..bbf16eeb9 100644 --- a/packages/serve-sim/Sources/SimNative/HIDInjector.swift +++ b/packages/serve-sim/Sources/SimNative/HIDInjector.swift @@ -175,15 +175,22 @@ actor HIDInjector { static let edgeRight: UInt32 = 4 // Right edge /// Synchronously hand an already-built Indigo message to the guest, freeing it. - /// Must run on `inputQueue`. - private func rawSend(_ msg: UnsafeMutableRawPointer) { - guard let client = hidClient, let sendSel = sendSel else { free(msg); return } + /// Must run on `inputQueue`. Returns false when there is no HID client to + /// send through (setup never completed), so callers can report the drop. + @discardableResult + private func rawSend(_ msg: UnsafeMutableRawPointer) -> Bool { + guard let client = hidClient, let sendSel = sendSel else { + print("[hid] Dropped HID message: no SimDeviceLegacyHIDClient") + free(msg) + return false + } typealias SendFunc = @convention(c) (AnyObject, Selector, UnsafeMutableRawPointer, ObjCBool, AnyObject?, AnyObject?) -> Void guard let sendIMP = class_getMethodImplementation(object_getClass(client)!, sendSel) else { free(msg) - return + return false } unsafeBitCast(sendIMP, to: SendFunc.self)(client, sendSel, msg, ObjCBool(true), nil, nil) + return true } /// Build a single-finger touch message (normalized 0..1 coords). Pure — safe @@ -202,25 +209,28 @@ actor HIDInjector { /// Synchronously build + send a single touch. For use inside gesture blocks /// already running on `inputQueue`. - private func rawSendTouch(type: String, x: Double, y: Double, edge: UInt32 = 0) { - if let msg = touchMessage(type: type, x: x, y: y, edge: edge) { rawSend(msg) } + @discardableResult + private func rawSendTouch(type: String, x: Double, y: Double, edge: UInt32 = 0) -> Bool { + guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return false } + return rawSend(msg) } - func sendTouch(type: String, x: Double, y: Double, screenWidth: Int, screenHeight: Int, edge: UInt32 = 0) { + /// Returns whether the touch was handed to the HID client. + func sendTouch(type: String, x: Double, y: Double, screenWidth: Int, screenHeight: Int, edge: UInt32 = 0) -> Bool { updateTouchTarget(width: screenWidth, height: screenHeight) - guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return } + guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return false } hidLog("[hid] Sending \(type) at (\(String(format:"%.3f",x)),\(String(format:"%.3f",y)))\(edge > 0 ? " edge=\(edge)" : "")") - rawSend(msg) + return rawSend(msg) } - func sendMultiTouch(type: String, x1: Double, y1: Double, x2: Double, y2: Double, screenWidth: Int, screenHeight: Int) { - guard let mouseFunc = mouseFunc else { return } + func sendMultiTouch(type: String, x1: Double, y1: Double, x2: Double, y2: Double, screenWidth: Int, screenHeight: Int) -> Bool { + guard let mouseFunc = mouseFunc else { return false } let eventType: Int32 switch type { case "begin", "move": eventType = 1 case "end": eventType = 2 - default: return + default: return false } updateTouchTarget(width: screenWidth, height: screenHeight) @@ -229,11 +239,11 @@ actor HIDInjector { var point2 = CGPoint(x: x2, y: y2) guard let rawMsg = mouseFunc(&point1, &point2, touchTarget, eventType, 1.0, 1.0, 0) else { print("[hid] IndigoHIDMessageForMouseNSEvent returned nil for multi-touch \(type)") - return + return false } hidLog("[hid] Multi-touch \(type) f1=(\(String(format:"%.3f",x1)),\(String(format:"%.3f",y1))) f2=(\(String(format:"%.3f",x2)),\(String(format:"%.3f",y2)))") - rawSend(rawMsg) + return rawSend(rawMsg) } // MARK: - Button events @@ -259,13 +269,14 @@ actor HIDInjector { /// Synchronously build + send a hardware-button message. Call only inside an /// `inputQueue` block (button sequences below run there). - private func sendHIDButton(eventSource: Int32, direction: Int32) { - guard let buttonFunc = buttonFunc else { return } + @discardableResult + private func sendHIDButton(eventSource: Int32, direction: Int32) -> Bool { + guard let buttonFunc = buttonFunc else { return false } guard let msg = buttonFunc(eventSource, direction, Self.buttonTargetHardware) else { print("[hid] IndigoHIDMessageForButton returned nil") - return + return false } - rawSend(msg) + return rawSend(msg) } // MARK: - Keyboard events @@ -425,43 +436,46 @@ actor HIDInjector { /// volume up/down, the action button, and the watch side button. /// - phase: "down" / "up" hold the button for natural long-presses (power /// off slider, side-button menus); "press" sends a momentary down+up. - func sendButtonHID(page: UInt32, usage: UInt32, phase: String) async { + /// Returns whether the press was handed to the guest. + @discardableResult + func sendButtonHID(page: UInt32, usage: UInt32, phase: String) async -> Bool { if let duoBridge { switch phase { - case "down": _ = duoBridge.send("key \(page) \(usage) 1") - case "up": _ = duoBridge.send("key \(page) \(usage) 0") + case "down": return duoBridge.send("key \(page) \(usage) 1") + case "up": return duoBridge.send("key \(page) \(usage) 0") case "press": - guard duoBridge.send("key \(page) \(usage) 1") else { return } + guard duoBridge.send("key \(page) \(usage) 1") else { return false } try? await Task.sleep(for: .milliseconds(250)) - _ = duoBridge.send("key \(page) \(usage) 0") - default: break + return duoBridge.send("key \(page) \(usage) 0") + default: return false } - return } guard let arb = hidArbitraryFunc else { print("[hid] Arbitrary HID injection unavailable (page=\(page) usage=\(usage))") - return + return false } let target = Self.buttonHIDTarget - func emit(_ direction: UInt32) { + func emit(_ direction: UInt32) -> Bool { guard let msg = arb(target, page, usage, direction) else { print("[hid] IndigoHIDMessageForHIDArbitrary returned nil (page=\(page) usage=\(usage) dir=\(direction))") - return + return false } - rawSend(msg) + return rawSend(msg) } hidLog("[hid] HID button page=\(page) usage=\(usage) phase=\(phase)") switch phase { - case "down": emit(1) - case "up": emit(2) + case "down": return emit(1) + case "up": return emit(2) default: - emit(1) + guard emit(1) else { return false } try? await Task.sleep(for: .seconds(0.05)) - emit(2) + return emit(2) } } - func sendButton(button: String, deviceUDID: String) async { + /// Returns whether the button was handed to the guest (false for an + /// unknown button or when the HID client / message builder is missing). + func sendButton(button: String, deviceUDID: String) async -> Bool { hidLog("[hid] Sending button: \(button)") switch button { @@ -473,44 +487,45 @@ actor HIDInjector { // and is functionally identical to a single home press, so use it // unconditionally rather than depending on whether the HID symbol // resolved. - launchSpringBoard(deviceUDID: deviceUDID) + return launchSpringBoard(deviceUDID: deviceUDID) case "swipe_home": - sendSwipeHome() + return sendSwipeHome() case "app_switcher": if buttonFunc != nil { // Double home press with delay for app switcher - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) + guard sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) else { return false } sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) try? await Task.sleep(for: .seconds(0.15)) sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) + return sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) } else { print("[hid] App switcher not available (IndigoHIDMessageForButton not loaded)") + return false } case "lock": if duoBridge != nil { - await sendButtonHID(page: 0x0c, usage: 0x30, phase: "press") - return + return await sendButtonHID(page: 0x0c, usage: 0x30, phase: "press") } - sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonUp) + guard sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonDown) else { return false } + return sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonUp) case "siri": // Holding Siri for ~300ms matches Simulator.app's "hold side button // to invoke Siri" gesture; a tap is ignored. - sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonDown) + guard sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonDown) else { return false } try? await Task.sleep(for: .seconds(0.3)) - sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonUp) + return sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonUp) case "side_button": - sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonUp) + guard sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonDown) else { return false } + return sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonUp) default: print("[hid] Unknown button: \(button)") + return false } } @@ -594,7 +609,7 @@ actor HIDInjector { /// Synthesize a swipe-up-from-bottom gesture (Face ID "go home" gesture). /// Uses IndigoHIDEdge.bottom to flag touches as system edge gestures, /// which iOS interprets as the home indicator swipe. - private func sendSwipeHome() { + private func sendSwipeHome() -> Bool { let xPos = 0.5 let yStart = 0.95 let yEnd = 0.35 @@ -603,7 +618,7 @@ actor HIDInjector { let edge = Self.edgeBottom // Touch down at bottom edge - rawSendTouch(type: "begin", x: xPos, y: yStart, edge: edge) + guard rawSendTouch(type: "begin", x: xPos, y: yStart, edge: edge) else { return false } Thread.sleep(forTimeInterval: stepDelay) // Interpolated moves upward @@ -615,14 +630,20 @@ actor HIDInjector { } // Touch up - rawSendTouch(type: "end", x: xPos, y: yEnd, edge: edge) + return rawSendTouch(type: "end", x: xPos, y: yEnd, edge: edge) } - private func launchSpringBoard(deviceUDID: String) { + private func launchSpringBoard(deviceUDID: String) -> Bool { let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") process.arguments = DeviceSet.simctlArguments(["launch", deviceUDID, "com.apple.springboard"]) - try? process.run() + do { + try process.run() + return true + } catch { + print("[hid] Failed to launch SpringBoard: \(error)") + return false + } } // MARK: - Orientation (GSEvent via PurpleWorkspacePort) diff --git a/packages/serve-sim/Sources/SimNative/sim-module.swift b/packages/serve-sim/Sources/SimNative/sim-module.swift index 28a656fdb..5b255b3e3 100644 --- a/packages/serve-sim/Sources/SimNative/sim-module.swift +++ b/packages/serve-sim/Sources/SimNative/sim-module.swift @@ -33,57 +33,75 @@ private func u32(_ v: Int) -> UInt32 { setupTask = Task { try await injector.setup(deviceUDID: udid, duoHelper: duoHelper) } } + // Every input method waits for `setup` first. The session is created on + // the first /ws connect, so a CLI `tap` lands milliseconds after this + // constructor; without the wait the frame reached the actor before setup + // and was dropped (no HID client yet) while the CLI exited 0 (#136). + // A failed setup (device gone / not booted) now throws to JS instead of + // silently swallowing every event. + @NodeMethod func touch(_ type: String, _ x: Double, _ y: Double, - _ w: Int, _ h: Int, _ edge: Int) async { - await injector.sendTouch(type: type, x: x, y: y, - screenWidth: w, screenHeight: h, edge: u32(edge)) + _ w: Int, _ h: Int, _ edge: Int) async throws -> Bool { + try await setupTask.value + return await injector.sendTouch(type: type, x: x, y: y, + screenWidth: w, screenHeight: h, edge: u32(edge)) } @NodeMethod func multiTouch(_ type: String, _ x1: Double, _ y1: Double, - _ x2: Double, _ y2: Double, _ w: Int, _ h: Int) async { - await injector.sendMultiTouch(type: type, x1: x1, y1: y1, x2: x2, y2: y2, - screenWidth: w, screenHeight: h) + _ x2: Double, _ y2: Double, _ w: Int, _ h: Int) async throws -> Bool { + try await setupTask.value + return await injector.sendMultiTouch(type: type, x1: x1, y1: y1, x2: x2, y2: y2, + screenWidth: w, screenHeight: h) } - @NodeMethod func button(_ button: String) async { - await injector.sendButton(button: button, deviceUDID: udid) + @NodeMethod func button(_ button: String) async throws -> Bool { + try await setupTask.value + return await injector.sendButton(button: button, deviceUDID: udid) } - @NodeMethod func buttonHid(_ page: Int, _ usage: Int, _ phase: String) async { - await injector.sendButtonHID(page: u32(page), usage: u32(usage), phase: phase) + @NodeMethod func buttonHid(_ page: Int, _ usage: Int, _ phase: String) async throws -> Bool { + try await setupTask.value + return await injector.sendButtonHID(page: u32(page), usage: u32(usage), phase: phase) } - @NodeMethod func key(_ type: String, _ usage: Int) async { + @NodeMethod func key(_ type: String, _ usage: Int) async throws { + try await setupTask.value await injector.sendKey(type: type, usage: u32(usage)) } /// NaN anchorX/anchorY mean "center" (the Swift API's nil). @NodeMethod func scroll(_ dx: Double, _ dy: Double, - _ anchorX: Double, _ anchorY: Double, _ w: Int, _ h: Int) async { + _ anchorX: Double, _ anchorY: Double, _ w: Int, _ h: Int) async throws { + try await setupTask.value await injector.sendScroll(dx: dx, dy: dy, anchorX: anchorX.isNaN ? nil : anchorX, anchorY: anchorY.isNaN ? nil : anchorY, screenWidth: w, screenHeight: h) } - @NodeMethod func digitalCrown(_ delta: Double) async { + @NodeMethod func digitalCrown(_ delta: Double) async throws { + try await setupTask.value await injector.sendDigitalCrown(delta: delta) } - @NodeMethod func orientation(_ orientation: Int) async -> Bool { - await injector.sendOrientation(orientation: u32(orientation)) + @NodeMethod func orientation(_ orientation: Int) async throws -> Bool { + try await setupTask.value + return await injector.sendOrientation(orientation: u32(orientation)) } - @NodeMethod func memoryWarning() async { + @NodeMethod func memoryWarning() async throws { + try await setupTask.value await injector.simulateMemoryWarning() } - @NodeMethod func softwareKeyboard() async { + @NodeMethod func softwareKeyboard() async throws { + try await setupTask.value await injector.toggleSoftwareKeyboard() } - @NodeMethod func caDebug(_ name: String, _ enabled: Bool) async -> Bool { - await injector.setCADebugOption(name: name, enabled: enabled) + @NodeMethod func caDebug(_ name: String, _ enabled: Bool) async throws -> Bool { + try await setupTask.value + return await injector.setCADebugOption(name: name, enabled: enabled) } @NodeMethod func isFoldable() async throws -> Bool { diff --git a/packages/serve-sim/src/__tests__/cli-input-ack.test.ts b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts new file mode 100644 index 000000000..d5816bcd7 --- /dev/null +++ b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts @@ -0,0 +1,105 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { spawn } from "child_process"; +import { createServer, type Server } from "http"; +import type { AddressInfo } from "net"; +import { join } from "path"; +import { WebSocketServer } from "ws"; + +// #136: tap/button/gesture used to fire their frame, wait 50ms and exit 0 no +// matter what the server did with it. They now ask for an ack and exit 1 when +// the server reports the injector could not deliver. A server that never acks +// (an older serve-sim) keeps the old exit-0 behaviour. + +const ACK_TAG = 0x84; +const cli = join(import.meta.dir, "..", "index.ts"); + +type Mode = "ok" | "fail" | "silent"; +const devices: Record = { "ACK-OK": "ok", "ACK-FAIL": "fail", "ACK-SILENT": "silent" }; + +let PORT = 0; +let server: Server; +const frames: Buffer[] = []; + +beforeAll(async () => { + const wss = new WebSocketServer({ noServer: true }); + server = createServer((req, res) => { + const url = new URL(req.url ?? "/", "http://x"); + if (url.pathname === "/api") { + res.writeHead(200, { "Content-Type": "application/json" }) + .end(JSON.stringify({ device: url.searchParams.get("device") })); + return; + } + res.writeHead(404).end(); + }); + server.on("upgrade", (req, socket, head) => { + const device = /^\/helper\/([^/]+)\/ws$/.exec(req.url ?? "")?.[1]; + const mode = device ? devices[device] : undefined; + if (!mode) { socket.destroy(); return; } + wss.handleUpgrade(req, socket, head, (ws) => ws.on("message", (data) => { + const frame = Buffer.from(data as Buffer); + frames.push(frame); + if (mode === "silent") return; + const body = JSON.parse(frame.subarray(1).toString()) as { ack?: number }; + if (typeof body.ack !== "number") return; + ws.send(Buffer.concat([Buffer.from([ACK_TAG]), Buffer.from(JSON.stringify({ ack: body.ack, ok: mode === "ok" }))])); + })); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + PORT = (server.address() as AddressInfo).port; +}); + +afterAll(async () => { + if (!server) return; + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); +}); + +function runCli(args: string[]): Promise<{ status: number | null; stderr: string }> { + const env = { ...process.env }; + delete env.SERVE_SIM_URL; + delete env.SERVE_SIM_AUTH_TOKEN; + return new Promise((resolve) => { + const child = spawn("bun", ["run", cli, ...args, "--url", `http://127.0.0.1:${PORT}`], { env }); + let stderr = ""; + child.stderr.on("data", (d) => (stderr += d)); + const timer = setTimeout(() => child.kill("SIGKILL"), 20_000); + child.on("close", (status) => { + clearTimeout(timer); + resolve({ status, stderr }); + }); + }); +} + +describe("input commands propagate HID delivery failures (#136)", () => { + for (const [name, args] of [ + ["tap", ["tap", "0.5", "0.5"]], + ["button", ["button", "lock"]], + ["gesture", ["gesture", '{"type":"begin","x":0.5,"y":0.5}']], + ] as const) { + test(`${name} exits 0 when the server acks delivery`, async () => { + const r = await runCli([...args, "-d", "ACK-OK"]); + expect(r.stderr).toBe(""); + expect(r.status).toBe(0); + }, 25_000); + + test(`${name} exits 1 with a clear error when the injector could not deliver`, async () => { + const r = await runCli([...args, "-d", "ACK-FAIL"]); + expect(r.status).toBe(1); + expect(r.stderr).toContain("could not deliver"); + expect(r.stderr).toContain("ACK-FAIL"); + }, 25_000); + + test(`${name} still exits 0 against a server that never acks`, async () => { + const r = await runCli([...args, "-d", "ACK-SILENT"]); + expect(r.status).toBe(0); + }, 25_000); + } + + test("tap tags begin and end with distinct ack ids", async () => { + frames.length = 0; + await runCli(["tap", "0.1", "0.2", "-d", "ACK-OK"]); + const bodies = frames.map((f) => JSON.parse(f.subarray(1).toString()) as { type: string; ack: number }); + expect(bodies.map((b) => b.type)).toEqual(["begin", "end"]); + expect(new Set(bodies.map((b) => b.ack)).size).toBe(2); + }, 25_000); +}); diff --git a/packages/serve-sim/src/__tests__/hid-input-ack.test.ts b/packages/serve-sim/src/__tests__/hid-input-ack.test.ts new file mode 100644 index 000000000..79a92a612 --- /dev/null +++ b/packages/serve-sim/src/__tests__/hid-input-ack.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { DeviceSession, type DeviceSessionDependencies } from "../device-session"; + +// #136: `serve-sim tap/button/gesture` exited 0 even when the HID injector +// could not deliver the event (injector not set up, device gone, unknown +// button). The CLI now tags its frames with an `ack` id and the session answers +// with [0x84]{ ack, ok } once the native send settles. Frames without `ack` +// (the browser viewer) must get no reply so the viewer protocol is unchanged. + +const ACK_TAG = 0x84; + +type Hid = DeviceSessionDependencies["hid"]; + +function session(hid: Partial): { + send: (tag: number, payload: unknown) => void; + replies: Array<{ tag: number; body: Record }>; + calls: unknown[][]; +} { + const calls: unknown[][] = []; + const base: Hid = { + touch: async (...args) => { calls.push(["touch", ...args]); return true; }, + multiTouch: async () => true, + button: async (...args) => { calls.push(["button", ...args]); return true; }, + buttonHid: async (...args) => { calls.push(["buttonHid", ...args]); return true; }, + key: async () => {}, + scroll: async () => {}, + digitalCrown: async () => {}, + orientation: async () => false, + memoryWarning: async () => {}, + softwareKeyboard: async () => {}, + caDebug: async () => false, + pose: async () => false, + hinge: async () => false, + }; + const s = new DeviceSession("ACK-UDID", { + capture: { + start: async () => {}, + stop: async () => {}, + subscribeMjpeg: async () => () => {}, + subscribeAvcc: async () => () => {}, + subscribeSize: async () => () => {}, + snapshotJpeg: async () => null, + setPreferredScreenSize: async () => {}, + }, + hid: { ...base, ...hid }, + }); + const replies: Array<{ tag: number; body: Record }> = []; + let receive!: (data: Buffer) => void; + s.attachHidSocket({ + send(frame: Buffer) { + // Ignore config/projection seeds; only collect input acks. + if (frame[0] === ACK_TAG) replies.push({ tag: frame[0]!, body: JSON.parse(frame.subarray(1).toString()) }); + }, + close() {}, + on(event: string, cb: (data: Buffer) => void) { if (event === "message") receive = cb; }, + } as never); + return { + send: (tag, payload) => receive(Buffer.concat([Buffer.from([tag]), Buffer.from(JSON.stringify(payload))])), + replies, + calls, + }; +} + +const settle = () => new Promise((r) => setTimeout(r, 10)); + +describe("HID input acknowledgements (#136)", () => { + test("a touch the injector delivered is acked ok", async () => { + const s = session({}); + s.send(0x03, { type: "begin", x: 0.5, y: 0.5, ack: 1 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 1, ok: true } }]); + }); + + test("a touch the injector could not deliver is acked not-ok", async () => { + const s = session({ touch: async () => false }); + s.send(0x03, { type: "begin", x: 0.5, y: 0.5, ack: 7 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 7, ok: false } }]); + }); + + test("a button the injector could not deliver is acked not-ok", async () => { + const s = session({ button: async () => false }); + s.send(0x04, { button: "lock", ack: 3 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 3, ok: false } }]); + }); + + test("an arbitrary HID button reports the native result", async () => { + const s = session({ buttonHid: async () => false }); + s.send(0x04, { button: "volume-up", page: 12, usage: 233, ack: 4 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 4, ok: false } }]); + }); + + test("a native throw is acked not-ok instead of escaping", async () => { + const s = session({ touch: async () => { throw new Error("Device not found"); } }); + s.send(0x03, { type: "end", x: 0.5, y: 0.5, ack: 9 }); + await settle(); + expect(s.replies).toEqual([{ tag: ACK_TAG, body: { ack: 9, ok: false } }]); + }); + + test("frames without an ack id (browser viewer) get no reply", async () => { + const s = session({ touch: async () => false, button: async () => false }); + s.send(0x03, { type: "begin", x: 0.5, y: 0.5 }); + s.send(0x04, { button: "home" }); + await settle(); + expect(s.replies).toEqual([]); + }); + + test("the ack id is not forwarded to the injector", async () => { + const s = session({}); + s.send(0x04, { button: "lock", ack: 2 }); + s.send(0x03, { type: "begin", x: 0.25, y: 0.75, ack: 5 }); + await settle(); + expect(s.calls[0]).toEqual(["button", "lock"]); + expect(s.calls[1]!.slice(0, 4)).toEqual(["touch", "begin", 0.25, 0.75]); + }); +}); diff --git a/packages/serve-sim/src/device-session.ts b/packages/serve-sim/src/device-session.ts index 87277261c..347cb5f40 100644 --- a/packages/serve-sim/src/device-session.ts +++ b/packages/serve-sim/src/device-session.ts @@ -65,6 +65,27 @@ const AVCC_SEED_TAG = 0x04; // WS server→client screen-config push (ClientManager.wsMsgConfig). const WS_MSG_CONFIG = 0x82; +// WS server→client input acknowledgement: [0x84]{ ack, ok }. Sent only for +// touch/button frames whose JSON carries a numeric `ack` id (the CLI's +// tap/button/gesture), so the browser viewer's protocol is unchanged (#136). +const WS_MSG_INPUT_ACK = 0x84; +const ACKABLE_INPUT_TAGS = new Set([0x03, 0x04]); + +/** The CLI's `ack` id on a touch/button frame, if it asked for one. */ +function inputAckId(frame: Buffer): number | undefined { + if (!ACKABLE_INPUT_TAGS.has(frame[0]!) || !frame.includes('"ack"', 1)) return undefined; + try { + const ack = (JSON.parse(frame.subarray(1).toString("utf8")) as { ack?: unknown }).ack; + return typeof ack === "number" && Number.isFinite(ack) ? ack : undefined; + } catch { + return undefined; + } +} + +function withoutAck(message: T): Omit { + const { ack: _ack, ...rest } = message; + return rest; +} const MJPEG_TRAILER = Buffer.from("\r\n", "ascii"); // Native capture is event-driven, so an unchanged screen can legitimately @@ -176,12 +197,12 @@ function newStreamHealth(): StreamHealth { function duoRendererErrorBody(error: DuoRendererError): { error: "duo_renderer_unavailable"; reason: string; stage: string; details: Record } { return { error: "duo_renderer_unavailable", reason: redactHome(error.reason), stage: error.stage, details: redactHome(error.details) }; } -type HidTransport = Pick< +/** Delivery-reporting sends resolve true only when the event reached the guest. */ +type ReportsDelivery = F extends (...args: infer A) => Promise ? (...args: A) => Promise : never; +type HidTransport = { + [K in "touch" | "multiTouch" | "button" | "buttonHid"]: ReportsDelivery; +} & Pick< NativeHid, - | "touch" - | "multiTouch" - | "button" - | "buttonHid" | "key" | "scroll" | "digitalCrown" @@ -853,7 +874,22 @@ export class DeviceSession { } }).catch((error) => { console.error("[hid] Pose reply failed:", error); }); } else { - void this.handleHidMessage(frame).catch((error) => { console.error("[hid] Input failed:", error); }); + const ack = inputAckId(frame); + const handled = this.handleHidMessage(frame); + if (ack === undefined) { + void handled.catch((error) => { console.error("[hid] Input failed:", error); }); + } else { + void handled + .then((result) => result === true, (error) => { + console.error("[hid] Input failed:", error); + return false; + }) + .then((ok) => { + if (this.hidSockets.has(ws)) { + ws.send(Buffer.concat([Buffer.from([WS_MSG_INPUT_ACK]), Buffer.from(JSON.stringify({ ack, ok }))])); + } + }); + } } }); ws.on("close", () => this.hidSockets.delete(ws)); @@ -876,24 +912,24 @@ export class DeviceSession { const H = this.height; switch (tag) { + // Touch and button resolve whether the injector delivered the event; the + // socket handler turns that into an ack when the frame asked for one. case 0x03: { - const m = json<{ type: string; x: number; y: number; edge?: number }>(); - if (m) { - this.recordTouchEvent(m); - this.hid.touch(m.type as "begin" | "move" | "end", m.x, m.y, W, H, m.edge ?? 0); - } - break; + const raw = json<{ type: string; x: number; y: number; edge?: number; ack?: number }>(); + if (!raw) return false; + const m = withoutAck(raw); + this.recordTouchEvent(m); + return this.hid.touch(m.type as "begin" | "move" | "end", m.x, m.y, W, H, m.edge ?? 0); } case 0x04: { - const m = json<{ button: string; page?: number; usage?: number; phase?: string }>(); - if (!m) break; + const raw = json<{ button: string; page?: number; usage?: number; phase?: string; ack?: number }>(); + if (!raw) return false; + const m = withoutAck(raw); this.recordHidEvent(tag, m); if (m.page != null && m.usage != null) { - this.hid.buttonHid(m.page, m.usage, (m.phase as "down" | "up" | "press") ?? "press"); - } else { - this.hid.button(m.button); + return this.hid.buttonHid(m.page, m.usage, (m.phase as "down" | "up" | "press") ?? "press"); } - break; + return this.hid.button(m.button); } case 0x05: { const m = json<{ type: string; x1: number; y1: number; x2: number; y2: number }>(); diff --git a/packages/serve-sim/src/index.ts b/packages/serve-sim/src/index.ts index 10e5d0079..0a01bfd0d 100755 --- a/packages/serve-sim/src/index.ts +++ b/packages/serve-sim/src/index.ts @@ -957,6 +957,90 @@ async function touchInputOrientation( } } +// Server→client input ack ([0x84]{ ack, ok }), sent by servers that support +// #136 for frames carrying an `ack` id. Older servers never reply, so a missing +// ack within INPUT_ACK_TIMEOUT_MS keeps the old fire-and-forget exit 0. Covers +// HID setup plus the new-client settle delay on a session's first input. +const WS_MSG_INPUT_ACK = 0x84; +const INPUT_ACK_TIMEOUT_MS = 4_000; + +/** + * Send touch/button frames and wait for the server to say whether the HID + * injector delivered them. Each frame goes out `delayMs` after the previous + * one regardless of acks (a tap's begin/end spacing must not stretch into a + * long-press). When the server reports a frame was not delivered, or closes + * the socket before acknowledging, prints why and sets exit code 1. + */ +function sendAckedInput( + target: InputTarget, + what: string, + frames: Array<{ tag: number; payload: object; delayMs?: number }>, +): Promise { + return new Promise((resolve, reject) => { + const ws = new WebSocket(target.wsUrl, { headers: target.headers }); + ws.binaryType = "arraybuffer"; + const pending = new Set(); + let sentAll = false; + let settled = false; + let timer: ReturnType | undefined; + const finish = (failure?: string) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + ws.onclose = null; + ws.close(); + if (failure) { + console.error(failure); + process.exitCode = 1; + } + resolve(); + }; + const undelivered = () => ( + `serve-sim could not deliver the ${what} to simulator ${target.device}: the HID injector ` + + `reported it was not sent (simulator not booted, HID setup failed, or unsupported input). ` + + `See the serve-sim server log; restart with \`serve-sim --kill ${target.device}\` and start it again.` + ); + ws.onmessage = ({ data }) => { + const frame = Buffer.from(data as ArrayBuffer); + if (frame[0] !== WS_MSG_INPUT_ACK) return; + let reply: { ack?: unknown; ok?: unknown }; + try { reply = JSON.parse(frame.subarray(1).toString()); } catch { return; } + if (typeof reply.ack !== "number" || !pending.delete(reply.ack)) return; + if (reply.ok !== true) return finish(undelivered()); + if (sentAll && pending.size === 0) finish(); + }; + ws.onclose = () => { + finish(`serve-sim closed the connection before the ${what} reached simulator ${target.device}.`); + }; + ws.onerror = () => { + console.error("Failed to connect to serve-sim server at", target.wsUrl); + settled = true; + reject(new Error("WebSocket connection failed")); + }; + ws.onopen = () => { + let ack = 0; + const sendNext = (index: number) => { + if (settled) return; + const { tag, payload } = frames[index]!; + ack += 1; + pending.add(ack); + sendHid(ws, tag, { ...payload, ack }); + const next = frames[index + 1]; + if (next) { + setTimeout(() => sendNext(index + 1), next.delayMs ?? 0); + return; + } + sentAll = true; + timer = setTimeout(() => { + debugCli("no input ack from %s after %dms; assuming an older server", target.wsUrl, INPUT_ACK_TIMEOUT_MS); + finish(); + }, INPUT_ACK_TIMEOUT_MS); + }; + sendNext(0); + }; + }); +} + async function gesture(jsonStr: string, deviceArg?: string, raw?: boolean) { const state = await resolveInputTarget(deviceArg); @@ -971,24 +1055,7 @@ async function gesture(jsonStr: string, deviceArg?: string, raw?: boolean) { const orientation = await touchInputOrientation(state, raw); const rawTouch = displayTouchToRaw(touch, orientation); - return new Promise((resolve, reject) => { - const ws = new WebSocket(state.wsUrl, { headers: state.headers }); - ws.binaryType = "arraybuffer"; - - ws.onopen = () => { - const json = new TextEncoder().encode(JSON.stringify(rawTouch)); - const msg = new Uint8Array(1 + json.length); - msg[0] = 0x03; - msg.set(json, 1); - ws.send(msg); - setTimeout(() => { ws.close(); resolve(); }, 50); - }; - - ws.onerror = () => { - console.error("Failed to connect to serve-sim server at", state.wsUrl); - reject(new Error("WebSocket connection failed")); - }; - }); + await sendAckedInput(state, "gesture", [{ tag: 0x03, payload: rawTouch }]); } async function tap(xArg: string, yArg: string, deviceArg?: string, raw?: boolean) { @@ -1002,28 +1069,10 @@ async function tap(xArg: string, yArg: string, deviceArg?: string, raw?: boolean } const state = await resolveInputTarget(deviceArg); const orientation = await touchInputOrientation(state, raw); - return new Promise((resolve, reject) => { - const ws = new WebSocket(state.wsUrl, { headers: state.headers }); - ws.binaryType = "arraybuffer"; - const send = (type: "begin" | "end") => { - const json = new TextEncoder().encode(JSON.stringify(displayTouchToRaw({ type, x, y }, orientation))); - const msg = new Uint8Array(1 + json.length); - msg[0] = 0x03; - msg.set(json, 1); - ws.send(msg); - }; - ws.onopen = () => { - send("begin"); - setTimeout(() => { - send("end"); - setTimeout(() => { ws.close(); resolve(); }, 50); - }, 40); - }; - ws.onerror = () => { - console.error("Failed to connect to serve-sim server at", state.wsUrl); - reject(new Error("WebSocket connection failed")); - }; - }); + await sendAckedInput(state, "tap", [ + { tag: 0x03, payload: displayTouchToRaw({ type: "begin", x, y }, orientation) }, + { tag: 0x03, payload: displayTouchToRaw({ type: "end", x, y }, orientation), delayMs: 40 }, + ]); } /** @@ -1318,24 +1367,7 @@ async function button(buttonName = "home", deviceArg?: string) { const hid = HID_BUTTON_CODES[buttonName]; const payload = hid ? { button: buttonName, ...hid } : { button: buttonName }; - return new Promise((resolve, reject) => { - const ws = new WebSocket(state.wsUrl, { headers: state.headers }); - ws.binaryType = "arraybuffer"; - - ws.onopen = () => { - const json = new TextEncoder().encode(JSON.stringify(payload)); - const msg = new Uint8Array(1 + json.length); - msg[0] = 0x04; - msg.set(json, 1); - ws.send(msg); - setTimeout(() => { ws.close(); resolve(); }, 50); - }; - - ws.onerror = () => { - console.error("Failed to connect to serve-sim server at", state.wsUrl); - reject(new Error("WebSocket connection failed")); - }; - }); + await sendAckedInput(state, `${buttonName} button press`, [{ tag: 0x04, payload }]); } // Send a CoreAnimation debug option toggle to the helper, which invokes diff --git a/packages/serve-sim/src/native.ts b/packages/serve-sim/src/native.ts index f0382fe17..9a10aee55 100644 --- a/packages/serve-sim/src/native.ts +++ b/packages/serve-sim/src/native.ts @@ -24,10 +24,11 @@ const require = createRequire(import.meta.url); // handle is garbage-collected (Swift `deinit`), so there are no explicit // destroy/free calls here. interface SimHIDHandle { - touch(type: TouchType, x: number, y: number, w: number, hh: number, edge: number): Promise; - multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, hh: number): Promise; - button(button: string): Promise; - buttonHid(page: number, usage: number, phase: ButtonPhase): Promise; + /** Resolves whether the event reached the HID client (addons before #136 resolve undefined). */ + touch(type: TouchType, x: number, y: number, w: number, hh: number, edge: number): Promise; + multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, hh: number): Promise; + button(button: string): Promise; + buttonHid(page: number, usage: number, phase: ButtonPhase): Promise; key(type: KeyType, usage: number): Promise; scroll(dx: number, dy: number, anchorX: number, anchorY: number, w: number, hh: number): Promise; digitalCrown(delta: number): Promise; @@ -180,6 +181,11 @@ function load(): NativeAddon { return addon; } +// How long a new HID client is given before touch/button input is sent to it. +// Measured on iOS 27.2: a tap 0.5s after the session was created landed 1/2 +// times; at 1.5s, 2/2 (and every later tap) landed. +const GUEST_HID_SETTLE_MS = 1_500; + /** * In-process HID injector for one simulator. Mirrors the WebSocket HID protocol * the spawned helper used to handle, but as direct native calls. @@ -187,10 +193,29 @@ function load(): NativeAddon { export class NativeHid { private readonly handle: SimHIDHandle; private readonly keys: KeySequencer; + /** + * Resolves once HID setup finished and the guest had time to pick up the + * new HID client. A brand-new SimDeviceLegacyHIDClient accepts messages + * (the send completion reports no error) but the guest ignores input for + * roughly the first second, so `serve-sim tap` right after a fresh start + * was silently dropped (#136). Touch/button input waits on this one shared + * promise; JS resolves its waiters in FIFO order, so a tap's begin/end can't + * be reordered. Later input passes through immediately. + */ + private readonly ready: Promise; constructor(udid: string) { this.handle = new (load().SimHID)(udid, duoHidHelperPath()); this.keys = new KeySequencer(this.handle); + this.ready = this.handle.isFoldable().then( + () => new Promise((resolve) => setTimeout(resolve, GUEST_HID_SETTLE_MS).unref?.()), + () => {}, // setup failed: let the sends run so they report the failure + ); + } + + private async settled(fn: () => PromiseLike): Promise { + await this.ready; + return fn(); } // The N-API bindings throw synchronously when a JS value can't be coerced to @@ -199,30 +224,39 @@ export class NativeHid { // so an unhandled throw here crashes the whole server — and if it lands // mid-gesture, the guest is left with a stuck finger that wedges input until // the sim reboots. The spawned helper used to absorb this in its own process; - // `guard` restores that isolation by swallowing malformed-input errors. + // `guard` restores that isolation by swallowing malformed-input errors (and + // HID setup failures, which every input method now awaits). private async guard(op: string, fn: () => PromiseLike, fallback: T): Promise { try { return await fn(); } catch (err) { - console.error(`[hid] ${op} ignored bad input:`, err instanceof Error ? err.message : err); + console.error(`[hid] ${op} failed:`, err instanceof Error ? err.message : err); return fallback; } } - touch(type: TouchType, x: number, y: number, w: number, h: number, edge = 0): Promise { - return this.guard("touch", () => this.handle.touch(type, x, y, w, h, edge), undefined); + // The four sends below resolve whether the injector handed the event to the + // guest: false when it couldn't (HID setup failed, message builder missing, + // unknown button) so `serve-sim tap/button/gesture` can exit non-zero (#136). + + touch(type: TouchType, x: number, y: number, w: number, h: number, edge = 0): Promise { + return this.delivered("touch", () => this.settled(() => this.handle.touch(type, x, y, w, h, edge))); + } + + multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, h: number): Promise { + return this.delivered("multiTouch", () => this.settled(() => this.handle.multiTouch(type, x1, y1, x2, y2, w, h))); } - multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, h: number): Promise { - return this.guard("multiTouch", () => this.handle.multiTouch(type, x1, y1, x2, y2, w, h), undefined); + button(button: string): Promise { + return this.delivered("button", () => this.settled(() => this.handle.button(button))); } - button(button: string): Promise { - return this.guard("button", () => this.handle.button(button), undefined); + buttonHid(page: number, usage: number, phase: ButtonPhase = "press"): Promise { + return this.delivered("buttonHid", () => this.settled(() => this.handle.buttonHid(page, usage, phase))); } - buttonHid(page: number, usage: number, phase: ButtonPhase = "press"): Promise { - return this.guard("buttonHid", () => this.handle.buttonHid(page, usage, phase), undefined); + private async delivered(op: string, fn: () => PromiseLike): Promise { + return (await this.guard(op, fn, false)) !== false; } key(type: KeyType, usage: number): Promise { @@ -231,11 +265,11 @@ export class NativeHid { /** anchorX/anchorY default to screen center when omitted. */ scroll(dx: number, dy: number, w: number, h: number, anchorX?: number, anchorY?: number): Promise { - return this.guard("scroll", () => this.handle.scroll(dx, dy, anchorX ?? NaN, anchorY ?? NaN, w, h), undefined); + return this.guard("scroll", () => this.settled(() => this.handle.scroll(dx, dy, anchorX ?? NaN, anchorY ?? NaN, w, h)), undefined); } digitalCrown(delta: number): Promise { - return this.guard("digitalCrown", () => this.handle.digitalCrown(delta), undefined); + return this.guard("digitalCrown", () => this.settled(() => this.handle.digitalCrown(delta)), undefined); } orientation(orientation: number): Promise { @@ -247,7 +281,7 @@ export class NativeHid { } softwareKeyboard(): Promise { - return this.guard("softwareKeyboard", () => this.handle.softwareKeyboard(), undefined); + return this.guard("softwareKeyboard", () => this.settled(() => this.handle.softwareKeyboard()), undefined); } caDebug(name: string, enabled: boolean): Promise { From 7c35f13041368735e293115c59c54dddaa5df2c3 Mon Sep 17 00:00:00 2001 From: Evan Bacon Date: Mon, 9 Nov 2026 19:25:00 -0800 Subject: [PATCH 2/3] fix(serve-sim): skip the input ack wait against servers that don't send acks (#136) A newer `serve-sim tap`/`button`/`gesture` talking to an older running server waited out the full 4s ack timeout on every call, because the old server never replies. Servers now advertise `inputAck: true` in their state file (and so in `/api`); the CLI only tags frames and waits for acks when the flag is set, and otherwise keeps the old untagged send + 50ms close. Also clear the ack timer when the socket errors after opening, so the process no longer hangs up to 4s before exiting, and note that the 1.5s guest HID settle delay is a heuristic. Tests: CLI cases for a legacy server (fast exit 0, no `ack` fields), a server that closes the socket before acking (exit 1), an advertising server that never acks (exit 0 after timeout), and the advertisement itself. --- .../src/__tests__/cli-input-ack.test.ts | 59 +++++++++++++++---- packages/serve-sim/src/index.ts | 41 +++++++++---- packages/serve-sim/src/native.ts | 6 +- packages/serve-sim/src/state.ts | 7 +++ 4 files changed, 91 insertions(+), 22 deletions(-) diff --git a/packages/serve-sim/src/__tests__/cli-input-ack.test.ts b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts index d5816bcd7..e80cb5e60 100644 --- a/packages/serve-sim/src/__tests__/cli-input-ack.test.ts +++ b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts @@ -4,17 +4,29 @@ import { createServer, type Server } from "http"; import type { AddressInfo } from "net"; import { join } from "path"; import { WebSocketServer } from "ws"; +import { previewConfigForState } from "../middleware"; +import { inProcessServeSimState } from "../state"; // #136: tap/button/gesture used to fire their frame, wait 50ms and exit 0 no // matter what the server did with it. They now ask for an ack and exit 1 when -// the server reports the injector could not deliver. A server that never acks -// (an older serve-sim) keeps the old exit-0 behaviour. +// the server reports the injector could not deliver. Servers advertise acks +// with `inputAck` in /api; one that doesn't (an older serve-sim) gets untagged +// frames and the old fast fire-and-forget exit 0. const ACK_TAG = 0x84; const cli = join(import.meta.dir, "..", "index.ts"); -type Mode = "ok" | "fail" | "silent"; -const devices: Record = { "ACK-OK": "ok", "ACK-FAIL": "fail", "ACK-SILENT": "silent" }; +// ok/fail: advertises acks and replies. close: advertises, then drops the +// socket without replying. hung: advertises but never replies. legacy: an +// older server that neither advertises nor replies. +type Mode = "ok" | "fail" | "close" | "hung" | "legacy"; +const devices: Record = { + "ACK-OK": "ok", + "ACK-FAIL": "fail", + "ACK-CLOSE": "close", + "ACK-HUNG": "hung", + "ACK-LEGACY": "legacy", +}; let PORT = 0; let server: Server; @@ -25,8 +37,10 @@ beforeAll(async () => { server = createServer((req, res) => { const url = new URL(req.url ?? "/", "http://x"); if (url.pathname === "/api") { + const device = url.searchParams.get("device"); + const advertises = device !== null && devices[device] !== undefined && devices[device] !== "legacy"; res.writeHead(200, { "Content-Type": "application/json" }) - .end(JSON.stringify({ device: url.searchParams.get("device") })); + .end(JSON.stringify({ device, ...(advertises ? { inputAck: true } : {}) })); return; } res.writeHead(404).end(); @@ -38,7 +52,8 @@ beforeAll(async () => { wss.handleUpgrade(req, socket, head, (ws) => ws.on("message", (data) => { const frame = Buffer.from(data as Buffer); frames.push(frame); - if (mode === "silent") return; + if (mode === "close") { ws.terminate(); return; } + if (mode === "hung" || mode === "legacy") return; const body = JSON.parse(frame.subarray(1).toString()) as { ack?: number }; if (typeof body.ack !== "number") return; ws.send(Buffer.concat([Buffer.from([ACK_TAG]), Buffer.from(JSON.stringify({ ack: body.ack, ok: mode === "ok" }))])); @@ -54,18 +69,19 @@ afterAll(async () => { await new Promise((r) => server.close(() => r())); }); -function runCli(args: string[]): Promise<{ status: number | null; stderr: string }> { +function runCli(args: string[]): Promise<{ status: number | null; stderr: string; ms: number }> { const env = { ...process.env }; delete env.SERVE_SIM_URL; delete env.SERVE_SIM_AUTH_TOKEN; return new Promise((resolve) => { + const started = Date.now(); const child = spawn("bun", ["run", cli, ...args, "--url", `http://127.0.0.1:${PORT}`], { env }); let stderr = ""; child.stderr.on("data", (d) => (stderr += d)); const timer = setTimeout(() => child.kill("SIGKILL"), 20_000); child.on("close", (status) => { clearTimeout(timer); - resolve({ status, stderr }); + resolve({ status, stderr, ms: Date.now() - started }); }); }); } @@ -89,12 +105,29 @@ describe("input commands propagate HID delivery failures (#136)", () => { expect(r.stderr).toContain("ACK-FAIL"); }, 25_000); - test(`${name} still exits 0 against a server that never acks`, async () => { - const r = await runCli([...args, "-d", "ACK-SILENT"]); + test(`${name} exits 1 when the server closes the socket before acking`, async () => { + const r = await runCli([...args, "-d", "ACK-CLOSE"]); + expect(r.status).toBe(1); + expect(r.stderr).toContain("closed the connection"); + }, 25_000); + + test(`${name} does not wait for acks from an older server`, async () => { + frames.length = 0; + const r = await runCli([...args, "-d", "ACK-LEGACY"]); expect(r.status).toBe(0); + // The CLI process itself takes a while to start; the point is that it + // never sits out the 4s ack timeout. + expect(r.ms).toBeLessThan(3_500); + expect(frames.length).toBeGreaterThan(0); + for (const f of frames) expect(JSON.parse(f.subarray(1).toString())).not.toHaveProperty("ack"); }, 25_000); } + test("an advertising server that never acks still exits 0 after the timeout", async () => { + const r = await runCli(["button", "lock", "-d", "ACK-HUNG"]); + expect(r.status).toBe(0); + }, 25_000); + test("tap tags begin and end with distinct ack ids", async () => { frames.length = 0; await runCli(["tap", "0.1", "0.2", "-d", "ACK-OK"]); @@ -103,3 +136,9 @@ describe("input commands propagate HID delivery failures (#136)", () => { expect(new Set(bodies.map((b) => b.ack)).size).toBe(2); }, 25_000); }); + +test("servers advertise input acks in their state file and /api (#136)", () => { + const state = inProcessServeSimState("ADVERTISE", 3999); + expect(state.inputAck).toBe(true); + expect(previewConfigForState(state, "", "serve-sim", false).inputAck).toBe(true); +}); diff --git a/packages/serve-sim/src/index.ts b/packages/serve-sim/src/index.ts index 0a01bfd0d..98891c4ac 100755 --- a/packages/serve-sim/src/index.ts +++ b/packages/serve-sim/src/index.ts @@ -711,6 +711,8 @@ interface InputTarget { url: string; wsUrl: string; headers: Record; + /** The server sends input delivery acks (#136); older servers don't. */ + inputAck: boolean; } /** @@ -731,7 +733,7 @@ async function resolveInputTarget(deviceArg?: string): Promise { console.error("No serve-sim server running. Run `serve-sim` first."); process.exit(1); } - return { device: state.device, url: state.url, wsUrl: state.wsUrl, headers }; + return { device: state.device, url: state.url, wsUrl: state.wsUrl, headers, inputAck: state.inputAck === true }; } let base: URL; @@ -761,7 +763,7 @@ async function resolveInputTarget(deviceArg?: string): Promise { console.error(`serve-sim at ${base.origin} returned HTTP ${res.status} for /api`); process.exit(1); } - const config = await res.json().catch(() => null) as { device?: string } | null; + const config = await res.json().catch(() => null) as { device?: string; inputAck?: unknown } | null; const device = config?.device; if (!device) { console.error( @@ -773,7 +775,13 @@ async function resolveInputTarget(deviceArg?: string): Promise { } const ws = new URL(`helper/${encodeURIComponent(device)}/ws`, base); ws.protocol = base.protocol === "https:" ? "wss:" : "ws:"; - return { device, url: base.toString().replace(/\/$/, ""), wsUrl: ws.toString(), headers }; + return { + device, + url: base.toString().replace(/\/$/, ""), + wsUrl: ws.toString(), + headers, + inputAck: config.inputAck === true, + }; } /** @@ -958,9 +966,10 @@ async function touchInputOrientation( } // Server→client input ack ([0x84]{ ack, ok }), sent by servers that support -// #136 for frames carrying an `ack` id. Older servers never reply, so a missing -// ack within INPUT_ACK_TIMEOUT_MS keeps the old fire-and-forget exit 0. Covers -// HID setup plus the new-client settle delay on a session's first input. +// #136 (they advertise `inputAck` in their state / `/api`) for frames carrying +// an `ack` id. Older servers never reply, so input to them stays +// fire-and-forget. INPUT_ACK_TIMEOUT_MS covers HID setup plus the new-client +// settle delay on a session's first input; a missing ack within it exits 0. const WS_MSG_INPUT_ACK = 0x84; const INPUT_ACK_TIMEOUT_MS = 4_000; @@ -969,7 +978,9 @@ const INPUT_ACK_TIMEOUT_MS = 4_000; * injector delivered them. Each frame goes out `delayMs` after the previous * one regardless of acks (a tap's begin/end spacing must not stretch into a * long-press). When the server reports a frame was not delivered, or closes - * the socket before acknowledging, prints why and sets exit code 1. + * the socket before acknowledging, prints why and sets exit code 1. A server + * that doesn't advertise acks gets the frames untagged and the socket closes + * 50ms after the last one, as before #136. */ function sendAckedInput( target: InputTarget, @@ -1013,8 +1024,10 @@ function sendAckedInput( finish(`serve-sim closed the connection before the ${what} reached simulator ${target.device}.`); }; ws.onerror = () => { + if (settled) return; console.error("Failed to connect to serve-sim server at", target.wsUrl); settled = true; + if (timer) clearTimeout(timer); reject(new Error("WebSocket connection failed")); }; ws.onopen = () => { @@ -1022,15 +1035,23 @@ function sendAckedInput( const sendNext = (index: number) => { if (settled) return; const { tag, payload } = frames[index]!; - ack += 1; - pending.add(ack); - sendHid(ws, tag, { ...payload, ack }); + if (target.inputAck) { + ack += 1; + pending.add(ack); + sendHid(ws, tag, { ...payload, ack }); + } else { + sendHid(ws, tag, payload); + } const next = frames[index + 1]; if (next) { setTimeout(() => sendNext(index + 1), next.delayMs ?? 0); return; } sentAll = true; + if (!target.inputAck) { + timer = setTimeout(() => finish(), 50); + return; + } timer = setTimeout(() => { debugCli("no input ack from %s after %dms; assuming an older server", target.wsUrl, INPUT_ACK_TIMEOUT_MS); finish(); diff --git a/packages/serve-sim/src/native.ts b/packages/serve-sim/src/native.ts index 9a10aee55..db5bcbd06 100644 --- a/packages/serve-sim/src/native.ts +++ b/packages/serve-sim/src/native.ts @@ -182,8 +182,10 @@ function load(): NativeAddon { } // How long a new HID client is given before touch/button input is sent to it. -// Measured on iOS 27.2: a tap 0.5s after the session was created landed 1/2 -// times; at 1.5s, 2/2 (and every later tap) landed. +// A heuristic, not a guarantee: measured on iOS 27.2 from only a few runs (a tap +// 0.5s after the session was created landed 1/2 times; at 1.5s, 2/2 and every +// later tap landed). If the first tap after a fresh start is flaky on a slower +// machine or runtime, suspect this value first. const GUEST_HID_SETTLE_MS = 1_500; /** diff --git a/packages/serve-sim/src/state.ts b/packages/serve-sim/src/state.ts index 54d1e6f3e..20b2311bc 100644 --- a/packages/serve-sim/src/state.ts +++ b/packages/serve-sim/src/state.ts @@ -53,6 +53,12 @@ export interface ServeSimDeviceState { * each can tell only whether its own set's simulators are booted. */ deviceSet?: string; + /** + * The server replies to input frames carrying an `ack` id with a delivery + * report (#136). Absent on servers older than that, which never reply, so + * the input subcommands only wait for acks when this is set. + */ + inputAck?: boolean; } /** @@ -81,6 +87,7 @@ export function inProcessServeSimState( url: `http://${h}:${port}`, streamUrl: `http://${h}:${port}${prefix}/helper/${udid}/stream.mjpeg`, wsUrl: `ws://${h}:${port}${prefix}/helper/${udid}/ws`, + inputAck: true, }; } From f1a5a6b8826f630d7745d276574eb7c8df3aecbe Mon Sep 17 00:00:00 2001 From: Evan Bacon Date: Mon, 9 Nov 2026 19:32:55 -0800 Subject: [PATCH 3/3] fix(serve-sim): warn when an ack-capable server never confirms input delivery (#136) --- packages/serve-sim/src/__tests__/cli-input-ack.test.ts | 4 +++- packages/serve-sim/src/index.ts | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/serve-sim/src/__tests__/cli-input-ack.test.ts b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts index e80cb5e60..ce5fe1406 100644 --- a/packages/serve-sim/src/__tests__/cli-input-ack.test.ts +++ b/packages/serve-sim/src/__tests__/cli-input-ack.test.ts @@ -123,9 +123,11 @@ describe("input commands propagate HID delivery failures (#136)", () => { }, 25_000); } - test("an advertising server that never acks still exits 0 after the timeout", async () => { + test("an advertising server that never acks exits 0 with a warning after the timeout", async () => { const r = await runCli(["button", "lock", "-d", "ACK-HUNG"]); expect(r.status).toBe(0); + expect(r.stderr).toContain("serve-sim: no delivery confirmation within 4s; input may not have reached ACK-HUNG"); + expect(r.stderr.trim().split("\n")).toHaveLength(1); }, 25_000); test("tap tags begin and end with distinct ack ids", async () => { diff --git a/packages/serve-sim/src/index.ts b/packages/serve-sim/src/index.ts index 98891c4ac..d3a45b8e9 100755 --- a/packages/serve-sim/src/index.ts +++ b/packages/serve-sim/src/index.ts @@ -969,7 +969,8 @@ async function touchInputOrientation( // #136 (they advertise `inputAck` in their state / `/api`) for frames carrying // an `ack` id. Older servers never reply, so input to them stays // fire-and-forget. INPUT_ACK_TIMEOUT_MS covers HID setup plus the new-client -// settle delay on a session's first input; a missing ack within it exits 0. +// settle delay on a session's first input; a missing ack within it warns on +// stderr and exits 0. const WS_MSG_INPUT_ACK = 0x84; const INPUT_ACK_TIMEOUT_MS = 4_000; @@ -1053,7 +1054,11 @@ function sendAckedInput( return; } timer = setTimeout(() => { - debugCli("no input ack from %s after %dms; assuming an older server", target.wsUrl, INPUT_ACK_TIMEOUT_MS); + debugCli("no input ack from %s after %dms", target.wsUrl, INPUT_ACK_TIMEOUT_MS); + console.error( + `serve-sim: no delivery confirmation within ${INPUT_ACK_TIMEOUT_MS / 1000}s; ` + + `input may not have reached ${target.device}`, + ); finish(); }, INPUT_ACK_TIMEOUT_MS); };