From fd62e5fd7973c317ee8b812d04df511d043fcdab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 16:33:59 +0100 Subject: [PATCH] feat(keyboard): report the return key in keyboard status `keyboard status` now answers what the visible keyboard's return key offers, on both platforms. Android parses the focused editor's `imeOptions` action out of the `dumpsys input_method` output the visibility read already fetched, so no extra adb call is made; `IME_FLAG_NO_ENTER_ACTION` outranks the action bits, and an app's `actionLabel` is reported beside its action kind. iOS gains a runner status command that reads the same return control `keyboard enter` presses, so the two commands cannot disagree about the same key. Both readings project into one vocabulary (`KEYBOARD_RETURN_KEYS` in the contracts package): `none` is only Android's to establish, `unknown` says a key is there this read cannot name, and a hidden keyboard reports neither field. Snapshot output and its device calls are unchanged. - Apple: `keyboardStatus` runner command (read-only, app-windows surface), `RunnerKeyboardStatus` - Android: `parseAndroidReturnKey` in `device-input-state.ts` - Daemon: projects `returnKey`/`returnKeyLabel`; a visibility-only owner publishes `unknown` - Output: `Return key:` and `Return key label:` lines, plus the JSON fields --- .../RunnerKeyboardReturnKey.swift | 8 +- .../RunnerKeyboardStatus.swift | 87 +++++++++++ .../RunnerSurfacePolicy.swift | 2 +- .../RunnerTests+CommandExecution.swift | 17 +++ .../RunnerTests+Keyboard.swift | 37 +++++ .../RunnerTests+Models.swift | 9 +- .../RunnerTests+KeyboardReturnKeyTests.swift | 139 ++++++++++++++++++ .../UnitTests/RunnerTests+ModelsTests.swift | 18 ++- .../fixtures/runner-command-read-only.json | 6 + contracts/fixtures/runner-requests.json | 1 + contracts/fixtures/runner-surface-policy.json | 1 + .../__tests__/runner-command-traits.test.ts | 1 + .../apple-runner/src/runner-command-traits.ts | 2 + packages/apple-runner/src/runner-contract.ts | 3 + .../src/system/output-keyboard.test.ts | 88 +++++++++++ packages/command-output/src/system/output.ts | 31 +++- .../src/command-output-schemas-system.test.ts | 11 ++ packages/command-surface/src/system/index.ts | 22 ++- packages/contracts/src/interactor-types.ts | 51 +++++-- packages/contracts/src/keyboard.ts | 38 ++++- .../src/platform-runtime-operations.ts | 12 +- .../contracts/src/tap-keyboard-occlusion.ts | 3 +- .../__tests__/keyboard-runtime.test.ts | 104 ++++++++++++- .../daemon/src/execution/keyboard-runtime.ts | 43 ++++-- .../src/__tests__/device-input-state.test.ts | 136 +++++++++++++++++ .../src/device-input-state.ts | 70 +++++++++ .../__tests__/index-runner-requests.test.ts | 1 + .../interactor-runner-provider.test.ts | 1 + packages/platform-apple/src/interactor.ts | 40 +++++ .../platform-apple/src/navigation/runtime.ts | 33 +++-- packages/platform-apple/src/runtime.test.ts | 27 +--- .../src/app-log-runtime.test.ts | 13 +- scripts/capability-matrix/annotations.ts | 16 +- .../runner-read-only-registry-parity.test.ts | 3 + .../schema/cli-help-command-usage.test.ts | 15 +- .../android-keyboard.test.ts | 67 +++++++++ .../provider-scenarios/ios-lifecycle.test.ts | 18 +++ .../provider-scenarios/ios-world.ts | 12 ++ test/repo-guards/capability-matrix.test.ts | 7 +- website/docs/docs/capabilities.md | 5 +- website/docs/docs/commands.md | 4 +- 41 files changed, 1081 insertions(+), 121 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardStatus.swift create mode 100644 packages/command-output/src/system/output-keyboard.test.ts create mode 100644 test/integration/provider-scenarios/android-keyboard.test.ts diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardReturnKey.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardReturnKey.swift index 570c1fed63..ce850e8d65 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardReturnKey.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardReturnKey.swift @@ -16,9 +16,11 @@ import Foundation /// `Emergency Call` places a call, which no caller of this should ever report as a submit. Every /// other Apple type has a case above. /// -/// `probeLabels` is the order `findKeyboardReturnControl(app:)` presses in. It is the runner's own -/// vocabulary, so it is not derived from the cases: it carries the two spellings of the plain -/// return key, in the order that found a hittable control before this rule existed. +/// `probeLabels` is the order `findKeyboardReturnControl(app:)` presses in. It is the +/// runner's own vocabulary, so it is not derived from the cases: it carries the two spellings of the +/// plain return key, in the order that found a hittable control before this rule existed. The +/// `keyboard status` read does not use it: it reads the keyboard's own key elements, so it needs no +/// English to find the return key. enum RunnerKeyboardReturnKey: String { case go case search diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardStatus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardStatus.swift new file mode 100644 index 0000000000..e580d2d56c --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerKeyboardStatus.swift @@ -0,0 +1,87 @@ +import Foundation +import CoreGraphics + +/// One `keyboard status` answer: whether the software keyboard is on screen and, when it is, what +/// its return key offers. +/// +/// `returnKey` is a `RunnerKeyboardReturnKey` raw value, or `nil` when the keyboard is hidden — a +/// hidden keyboard has no return key to report, which is a different fact from one this read cannot +/// name. `returnKeyLabel` is the wording the key itself carries, with surrounding whitespace +/// trimmed, so a localized or an app's custom key stays readable even where the kind is `unknown`. +/// It is present only when the read saw a key to read it from; a keyboard that exposes no readable +/// key answers `unknown` with no label. +struct RunnerKeyboardStatus: Equatable { + let visible: Bool + let returnKey: String? + let returnKeyLabel: String? +} + +extension RunnerKeyboardStatus { + /// One control the status read observed on the visible keyboard: the wording it carries in the + /// accessibility tree, and the frame it occupies. `frame` decides which control is the + /// keyboard's bottom-trailing key, which is where Apple's keyboards place the return key. + struct ObservedKeyboardKey { + let text: String? + let frame: CGRect + } + + /// Builds the answer from what the runner observed. The keyboard decides whether a return key is + /// reported at all: a hidden one has none, which is a different fact from one this read cannot + /// name. Only Android's `IME_ACTION_NONE` can establish that a field offers no action key, so + /// `none` is never this runner's answer. + /// + /// The key's own wording decides the kind, through the one rule `keyboard enter` probes with + /// (`RunnerKeyboardReturnKey.kind(of:)`), so the two commands can never disagree about the same + /// key. The label itself is kept as the accessibility tree carries it apart from its surrounding + /// whitespace, so a localized or an app's custom key stays readable even where the kind is + /// `unknown`. + static func observed( + visible: Bool, + returnControlText: String? + ) -> RunnerKeyboardStatus { + guard visible else { + return RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil) + } + guard let returnControlText else { + return RunnerKeyboardStatus( + visible: true, + returnKey: RunnerKeyboardReturnKey.unknown.rawValue, + returnKeyLabel: nil + ) + } + let label = returnControlText.trimmingCharacters(in: .whitespacesAndNewlines) + return RunnerKeyboardStatus( + visible: true, + returnKey: RunnerKeyboardReturnKey.kind(of: label).rawValue, + returnKeyLabel: label.isEmpty ? nil : label + ) + } + + /// Builds the answer from every control the read saw on the visible keyboard, so it does not need to + /// read English to find the return key. A control whose wording `kind(of:)` names a kind for is the + /// return key on any keyboard in any language, and its wording is the label. When none maps — a + /// fully localized keyboard — the return key is the measurable control in the keyboard's + /// bottom-trailing corner: the slot Apple's keyboards give it, beside and right of the space bar, + /// which no letter, shift or delete key reaches. A keyboard's padding elements report an empty + /// frame and never win on position. An unmatched key's wording is still reported, with kind + /// `unknown`: that is what the label is for. Only a keyboard exposing no worded, measurable control + /// answers `unknown` with no label. + static func observed( + visible: Bool, + keyboardKeys: [ObservedKeyboardKey] + ) -> RunnerKeyboardStatus { + guard visible else { + return RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil) + } + let worded = keyboardKeys.filter { key in + !(key.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + if let named = worded.first(where: { RunnerKeyboardReturnKey.kind(of: $0.text ?? "") != .unknown }) { + return observed(visible: true, returnControlText: named.text) + } + let returnKey = worded + .filter { !$0.frame.isEmpty } + .max { lhs, rhs in (lhs.frame.maxY, lhs.frame.maxX) < (rhs.frame.maxY, rhs.frame.maxX) } + return observed(visible: true, returnControlText: returnKey?.text) + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift index 2a36d2379f..eef4784de8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift @@ -16,7 +16,7 @@ extension CommandType { return .none case .tap, .longPress, .drag, .type, .swipe, .scroll, .findText, .querySelector, .readText, .snapshot, .backInApp, .gesture, .gestureViewport, .appSwitcher, - .keyboardDismiss, .sequence: + .keyboardDismiss, .keyboardStatus, .sequence: return .appWindows case .screenshot, .recordStart: return .observedScreen diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 6c3b307aa3..a8e9171cdc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -636,6 +636,23 @@ extension RunnerTests { wasVisible: result.wasVisible ) ) + case .keyboardStatus: + var status: RunnerKeyboardStatus? + withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { + status = keyboardStatus(app: activeApp) + } + guard let result = status else { + return Response(ok: false, error: ErrorPayload(message: "keyboardStatus produced no result")) + } + return Response( + ok: true, + data: DataPayload( + message: "keyboardStatus", + visible: result.visible, + keyboardReturnKey: result.returnKey, + keyboardReturnKeyLabel: result.returnKeyLabel + ) + ) case .alert: let action = (command.action ?? "get").lowercased() let deadline = alertDeadline ?? Date().addingTimeInterval( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift index 52725cb190..685f2d63c9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift @@ -201,6 +201,43 @@ extension RunnerTests { return visibleKeyboardFrame(app: app) != nil } + /// What `keyboard status` answers with: the keyboard's visibility and, when it is up, the return + /// key the focused field offers. It reads the keyboard's own key elements rather than looking a + /// control up by English name, so a localized keyboard (a return key reading `szukaj`) is reported + /// with its own wording instead of a label-less `unknown`. `RunnerKeyboardStatus` decides which + /// observed control is the return key, through the same `kind(of:)` rule `keyboard enter` probes + /// with, so the two commands can never disagree about the wording they can name. + func keyboardStatus(app: XCUIApplication) -> RunnerKeyboardStatus { + guard isKeyboardVisible(app: app) else { + return RunnerKeyboardStatus.observed(visible: false, returnControlText: nil) + } + return RunnerKeyboardStatus.observed(visible: true, keyboardKeys: observedKeyboardKeys(app: app)) + } + + /// Every control the visible keyboard exposes with its wording and frame. The keyboard is another + /// process, so every element read is a round trip: one pass over its keys and one over its buttons + /// is what a read can afford, and it is the same two queries a snapshot's keyboard band makes. An + /// element that answers no wording is dropped — it is neither a kind nor a label. Off iOS nothing + /// is queried and nothing is reported, so a caller must not read the empty answer as a keyboard + /// with no return key; `keyboard status` is not advertised there. + func observedKeyboardKeys(app: XCUIApplication) -> [RunnerKeyboardStatus.ObservedKeyboardKey] { +#if os(iOS) + let keyboard = app.keyboards.firstMatch + let elements: [XCUIElement] = + safely("KEYBOARD_STATUS_KEYS", [XCUIElement]()) { keyboard.keys.allElementsBoundByIndex } + + safely("KEYBOARD_STATUS_BUTTONS", [XCUIElement]()) { keyboard.buttons.allElementsBoundByIndex } + return elements.compactMap { element in + guard let label = safely("KEYBOARD_STATUS_KEY_LABEL", { element.label }) else { return nil } + let text = label.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return nil } + let frame = safely("KEYBOARD_STATUS_KEY_FRAME", CGRect.zero) { element.frame } + return RunnerKeyboardStatus.ObservedKeyboardKey(text: label, frame: frame) + } +#else + return [] +#endif + } + func dismissKeyboard(app: XCUIApplication) -> RunnerKeyboardDismissOutcome { let keyboard = app.keyboards.firstMatch let wasVisible = isKeyboardVisible(app: app) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index a5b2ebc5cf..136cf474f8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -26,6 +26,7 @@ enum CommandType: String, Codable, CaseIterable { case volumeDown case keyboardDismiss case keyboardReturn + case keyboardStatus case alert case sequence case gesture @@ -300,7 +301,7 @@ extension Command { .keyboardDismiss, .keyboardReturn, .sequence, .gesture: return .interaction - case .findText, .readText, .gestureViewport: + case .findText, .readText, .gestureViewport, .keyboardStatus: return .appRead(retainsJournalResponseJson: true) case .snapshot: @@ -536,6 +537,12 @@ struct DataPayload: Codable { var wasVisible: Bool? var dismissed: Bool? var keyboardDismissMechanism: String? + /// The kind of return key the visible keyboard offers, on a `keyboard status` answer. Absent when + /// the keyboard is hidden, which is not the same claim as a key this read could not name. + var keyboardReturnKey: String? + /// The wording that return key carries, verbatim, so a localized or an app's custom key stays + /// readable. Absent when the key carries none, or there is no key to read. + var keyboardReturnKeyLabel: String? var orientation: String? var gestureFallback: String? var gestureFallbackMessage: String? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardReturnKeyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardReturnKeyTests.swift index fe38253aac..e07d62376b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardReturnKeyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardReturnKeyTests.swift @@ -82,5 +82,144 @@ extension RunnerTests { XCTAssertEqual(RunnerKeyboardReturnKey.kind(of: label), expected[label], label) } } + + /// What a `keyboard status` read builds from each observation. The visible keyboard's return + /// control decides both fields: its wording names the kind through the same rule `keyboard enter` + /// probes with, and it is kept as the accessibility tree carries it apart from surrounding + /// whitespace, so a key this read cannot name is still readable. + func testKeyboardStatusReportsTheObservedReturnControl() { + let status = RunnerKeyboardStatus.observed(visible: true, returnControlText: " Search ") + XCTAssertEqual(status, RunnerKeyboardStatus(visible: true, returnKey: "search", returnKeyLabel: "Search")) + } + + /// The kinds `kind(of:)` names but the press list never probes are reported by the read, because + /// the read looks at the keyboard's own keys rather than looking a control up by English name. + func testKeyboardStatusReportsAKindThePressProbeListOmits() { + for (label, kind) in [("Route", "route"), ("Continue", "continue")] { + XCTAssertEqual( + RunnerKeyboardStatus.observed(visible: true, returnControlText: label), + RunnerKeyboardStatus(visible: true, returnKey: kind, returnKeyLabel: label), + label + ) + XCTAssertFalse( + RunnerKeyboardReturnKey.probeLabels.contains(label), + "\(label) must stay out of the press probe list, which the read does not use" + ) + } + } + + /// A hidden keyboard reports no return key at all. That is a different fact from one it could not + /// name, so both fields stay out rather than answering `unknown` for a key nobody can press. The + /// keyboard's keys are ignored: nothing is being submitted to. + func testKeyboardStatusReportsNoReturnKeyWhileTheKeyboardIsHidden() { + XCTAssertEqual( + RunnerKeyboardStatus.observed(visible: false, returnControlText: "Search"), + RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil) + ) + XCTAssertEqual( + RunnerKeyboardStatus + .observed(visible: false, keyboardKeys: [RunnerKeyboardStatus.ObservedKeyboardKey(text: "Done", frame: CGRect(x: 0, y: 0, width: 40, height: 40))]), + RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil) + ) + } + + /// The English keyboard, as the accessibility tree carries it: the search field's return key reads + /// `Search` in the bottom-trailing slot. The picker names the kind from its wording. + func testKeyboardStatusPicksTheReturnKeyFromTheKeyboardKeysByWording() { + let keys = observedKeyboardKeysFixture([ + ("Q", 5, 959, 39), ("W", 44, 959, 40), ("shift", 5, 1067, 51), ("Delete", 348, 1067, 52), + ("Numbers", 5, 1121, 49), ("Emoji", 54, 1121, 49), ("Space", 103, 1121, 197), + ("Search", 301, 1121, 99), ("Padding-Right", 0, 952, 0), + ]) + XCTAssertEqual( + RunnerKeyboardStatus.observed(visible: true, keyboardKeys: keys), + RunnerKeyboardStatus(visible: true, returnKey: "search", returnKeyLabel: "Search") + ) + } + + /// The whole point of reading the keyboard's own keys: a fully localized keyboard names no kind, so + /// the return key is the bottom-trailing control and its wording is reported even though the kind + /// is `unknown`. These are the frames this read measured on a Polish iOS keyboard, where `szukaj` + /// sits beside a wide space bar in the last row and the letter, shift and delete keys are higher or + /// further left. The headless test IME, which exposes keys with no wording at all, is the same + /// shape with nothing to report. + func testKeyboardStatusReportsALocalizedReturnKeyWithItsOwnWording() { + let keys = observedKeyboardKeysFixture([ + ("Padding-Left", 0, 952, 0), ("Q", 5, 959, 39), ("P", 360, 959, 40), ("shift", 5, 1067, 51), + ("M", 301, 1067, 40), ("usuń", 348, 1067, 52), ("cyfry", 5, 1121, 49), ("Emoji", 54, 1121, 49), + (" ", 103, 1121, 197), ("szukaj", 301, 1121, 99), + ]) + XCTAssertEqual( + RunnerKeyboardStatus.observed(visible: true, keyboardKeys: keys), + RunnerKeyboardStatus(visible: true, returnKey: "unknown", returnKeyLabel: "szukaj") + ) + } + + /// The picker takes a control whose wording names a kind even when another control sits further + /// down-right, so a keyboard that localizes some keys and not others is still named by wording. + /// Padding elements, which carry no usable frame, never win on position. + func testKeyboardStatusPrefersNamedWordingOverKeyboardGeometry() { + let keys = observedKeyboardKeysFixture([ + ("Done", 5, 959, 39), ("ż", 301, 1121, 99), ("Padding-Right", 0, 952, 0), + ]) + XCTAssertEqual( + RunnerKeyboardStatus.observed(visible: true, keyboardKeys: keys), + RunnerKeyboardStatus(visible: true, returnKey: "done", returnKeyLabel: "Done") + ) + } + + /// Only a visible keyboard that exposes no control worth reporting answers `unknown` with no label: + /// an empty layout, a keyboard whose keys all carry no wording (a headless test IME), and one whose + /// only wording sits on a zero-frame padding element. + func testKeyboardStatusReportsUnknownForAVisibleKeyboardWithNoReadableKey() { + let expected = RunnerKeyboardStatus(visible: true, returnKey: "unknown", returnKeyLabel: nil) + XCTAssertEqual(RunnerKeyboardStatus.observed(visible: true, keyboardKeys: []), expected) + XCTAssertEqual( + RunnerKeyboardStatus.observed( + visible: true, + keyboardKeys: observedKeyboardKeysFixture([(" ", 301, 1121, 99)]) + ), + expected + ) + XCTAssertEqual( + RunnerKeyboardStatus.observed( + visible: true, + keyboardKeys: observedKeyboardKeysFixture([("szukaj", 0, 952, 0)]) + ), + expected + ) + } + + /// The press path is unchanged by what the read can see: it still looks its own labels up in its + /// own order, and the read never consults that list. + func testKeyboardEnterProbeListStillOwnsItsOrder() { + XCTAssertEqual( + RunnerKeyboardReturnKey.probeLabels, + ["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"] + ) + } + + /// A key whose wording the rule does not recognize keeps its wording, which is the whole point of + /// reporting the label: a localized or a custom key stays readable where the kind is `unknown`. + func testKeyboardStatusKeepsAnUnrecognizedLabelVerbatim() { + for label in ["Send Message", "搜索", "完了"] { + let status = RunnerKeyboardStatus.observed(visible: true, returnControlText: label) + XCTAssertEqual(status.returnKey, "unknown", label) + XCTAssertEqual(status.returnKeyLabel, label) + } + } + + /// Builds a keyboard's observed controls from the frames this read measured; a zero width is an + /// element with no usable frame, such as the padding entries a keyboard exposes. + private func observedKeyboardKeysFixture( + _ entries: [(text: String, x: CGFloat, y: CGFloat, width: CGFloat)] + ) -> [RunnerKeyboardStatus.ObservedKeyboardKey] { + entries.map { entry in + RunnerKeyboardStatus.ObservedKeyboardKey( + text: entry.text, + frame: CGRect(x: entry.x, y: entry.y, width: entry.width, height: entry.width > 0 ? 54 : 0) + ) + } + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index 9f9ce6f832..3ffa22f4ab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -246,9 +246,10 @@ extension RunnerTests { /// Commands that did not exist at the merge-base, so no classification of its is compared with /// theirs. `appState` arrived with #2929, `pasteboardWrite` with the runner-written simulator - /// clipboard, `snapshotBridge` with the runner-served Simulator AX bridge. + /// clipboard, `snapshotBridge` with the runner-served Simulator AX bridge, `keyboardStatus` with + /// the return key in `keyboard status`. private static let commandsNewerThanTheMergeBase: Set = [ - .appState, .pasteboardWrite, .volumeUp, .volumeDown, .snapshotBridge, + .appState, .pasteboardWrite, .volumeUp, .volumeDown, .snapshotBridge, .keyboardStatus, ] /// The commands production never runs through the prepared path's body: `executeOnMain` answers @@ -262,6 +263,14 @@ extension RunnerTests { .recordStart, .recordStop, .snapshot, .snapshotBridge, ] + /// The commands the witness column below cannot evaluate against the merge-base rule: they did not + /// exist there, and that rule's `!isReadOnly` term needs the merge-base's own read-only answer for + /// them. `keyboardStatus` is the only one: it is a read that reaches the prepared path, so the + /// formula would clear a remembered tap the derived trait keeps. The table below pins its + /// `converts: false`, which is what `invalidatesRememberedTextEntryTap` reads, so the answer is + /// still asserted — against its own row rather than against a classification the merge-base lacks. + private static let commandsWithoutMergeBaseWitness: Set = [.keyboardStatus] + private func assertRememberedTextEntryWitnessInvalidation( _ command: Command, type: CommandType, @@ -281,6 +290,7 @@ extension RunnerTests { // remembered tap for it too and this column says so for every command, not just that one. let mergeBaseClears = type != .tap && type != .type && !wasReadOnlyAtMergeBase guard !Self.commandsAnsweredBeforeThePreparedPath.contains(type) else { return } + guard !Self.commandsWithoutMergeBaseWitness.contains(type) else { return } XCTAssertEqual( command.invalidatesRememberedTextEntryTap, mergeBaseClears, @@ -329,6 +339,10 @@ extension RunnerTests { ), (.keyboardDismiss, expectation(interaction: true, retry: false, launch: .mayLaunch, converts: true, retains: true, isReadOnly: false)), (.keyboardReturn, expectation(interaction: true, retry: false, launch: .mayLaunch, converts: true, retains: true, isReadOnly: false)), + ( + .keyboardStatus, + expectation(interaction: false, retry: true, launch: .existingApp, converts: false, retains: true, isReadOnly: true) + ), ( .alert, expectation(interaction: false, retry: true, launch: .presentedSurface, converts: false, retains: true, isReadOnly: true) diff --git a/contracts/fixtures/runner-command-read-only.json b/contracts/fixtures/runner-command-read-only.json index 325d83603b..92b9b28489 100644 --- a/contracts/fixtures/runner-command-read-only.json +++ b/contracts/fixtures/runner-command-read-only.json @@ -83,6 +83,12 @@ }, "readOnly": false }, + { + "command": { + "command": "keyboardStatus" + }, + "readOnly": true + }, { "command": { "command": "longPress" diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index f7a46dcfcc..115b9a5b16 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -49,6 +49,7 @@ {"name": "ios-simulator.interactor-home.press", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "home", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-dismiss.dismiss", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "keyboardDismiss", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-enter.return", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "keyboardReturn", "appBundleId": "com.example.app", "commandId": ""}}, + {"name": "ios-simulator.interactor-keyboard-status.read", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "keyboardStatus", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-read-text.point", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-set-orientation.rotate", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "rotate", "orientation": "landscape-left", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-snapshot.every-option", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "preferredBackend": "tree", "customActions": true, "depth": 3, "scope": "Go", "raw": true, "commandId": ""}}, diff --git a/contracts/fixtures/runner-surface-policy.json b/contracts/fixtures/runner-surface-policy.json index 32ec0e691d..1cb0ffccfb 100644 --- a/contracts/fixtures/runner-surface-policy.json +++ b/contracts/fixtures/runner-surface-policy.json @@ -25,6 +25,7 @@ "volumeDown": "none", "keyboardDismiss": "app-windows", "keyboardReturn": "none", + "keyboardStatus": "app-windows", "alert": "resolved-modal", "sequence": "app-windows", "recordStart": "observed-screen", diff --git a/packages/apple-runner/src/__tests__/runner-command-traits.test.ts b/packages/apple-runner/src/__tests__/runner-command-traits.test.ts index 56d5ec7bd7..a0b18e27f7 100644 --- a/packages/apple-runner/src/__tests__/runner-command-traits.test.ts +++ b/packages/apple-runner/src/__tests__/runner-command-traits.test.ts @@ -53,6 +53,7 @@ test('runner command trait table pins lifecycle-sensitive command groups', () => 'appState', 'findText', 'gestureViewport', + 'keyboardStatus', 'querySelector', 'readText', 'screenshot', diff --git a/packages/apple-runner/src/runner-command-traits.ts b/packages/apple-runner/src/runner-command-traits.ts index 146f08d10a..36e32d0ab8 100644 --- a/packages/apple-runner/src/runner-command-traits.ts +++ b/packages/apple-runner/src/runner-command-traits.ts @@ -93,6 +93,7 @@ export const RUNNER_COMMAND_TRAITS = { volumeDown: DEFAULT_TRAITS, keyboardDismiss: DEFAULT_TRAITS, keyboardReturn: DEFAULT_TRAITS, + keyboardStatus: READ_ONLY_TRAITS, alert: readAlertActionTraits, sequence: PREFLIGHT_SKIPPABLE_TOUCH_MUTATION_TRAITS, recordStart: DEFAULT_TRAITS, @@ -136,6 +137,7 @@ const RUNNER_COMMAND_SURFACE_POLICY = { volumeDown: 'none', keyboardDismiss: 'app-windows', keyboardReturn: 'none', + keyboardStatus: 'app-windows', alert: 'resolved-modal', sequence: 'app-windows', recordStart: 'observed-screen', diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 790f8d286e..5bda230c30 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -103,6 +103,9 @@ export type RunnerCommand = { | 'volumeDown' | 'keyboardDismiss' | 'keyboardReturn' + // Reads the software keyboard and the focused field's return key. A read: it reports what is on + // screen, and presses nothing. + | 'keyboardStatus' | 'alert' | 'sequence' | 'recordStart' diff --git a/packages/command-output/src/system/output-keyboard.test.ts b/packages/command-output/src/system/output-keyboard.test.ts new file mode 100644 index 0000000000..571ec2a137 --- /dev/null +++ b/packages/command-output/src/system/output-keyboard.test.ts @@ -0,0 +1,88 @@ +import { expect, test } from 'vitest'; +import { systemCliOutputFormatters } from './output.ts'; + +function keyboardText(result: Parameters[0]['result']) { + return systemCliOutputFormatters.keyboard({ + env: process.env, + stderrIsTTY: false, + input: {}, + result, + }).text; +} + +// Both platforms answer `status`, so the shared lines come first and the platform's own evidence +// follows. The return key belongs to the visible keyboard, so it is printed on iOS as much as on +// Android, before the IME fields that only Android reads. +test('keyboard status CLI output prints the return key on iOS', () => { + expect( + keyboardText({ + platform: 'ios', + action: 'status', + visible: true, + returnKey: 'search', + returnKeyLabel: 'Search', + }), + ).toBe(['Keyboard visible: yes', 'Return key: search', 'Return key label: Search'].join('\n')); +}); + +// The kind is the useful answer when the key carries no readable wording: a localized key the reader +// cannot name still reports its label. +test('keyboard status CLI output prints an unknown kind with the label it could read', () => { + expect( + keyboardText({ + platform: 'ios', + action: 'status', + visible: true, + returnKey: 'unknown', + returnKeyLabel: '\u641c\u3059', + }), + ).toBe( + ['Keyboard visible: yes', 'Return key: unknown', 'Return key label: \u641c\u3059'].join('\n'), + ); +}); + +// Closest negative: a hidden keyboard has no return key, so neither line appears — the reader must +// not be told a key exists. +test('keyboard status CLI output prints no return key while the keyboard is hidden', () => { + expect(keyboardText({ platform: 'ios', action: 'status', visible: false })).toBe( + 'Keyboard visible: no', + ); +}); + +test("keyboard status CLI output prints the return key among Android's IME fields", () => { + expect( + keyboardText({ + platform: 'android', + action: 'status', + visible: true, + type: 'text', + inputOwner: 'app', + returnKey: 'send', + returnKeyLabel: 'Send Message', + }), + ).toBe( + [ + 'Keyboard visible: yes', + 'Return key: send', + 'Return key label: Send Message', + 'Input type: text', + 'Input owner: app', + 'Next action: Keyboard is visible and focused input appears app-owned; fill/type can proceed.', + ].join('\n'), + ); +}); + +// `dismiss` and `enter` still answer with their message alone: neither reports a keyboard it did not +// look at after acting, so the status lines must not leak into them. +test('keyboard dismiss CLI output stays the message alone', () => { + expect( + keyboardText({ + platform: 'ios', + action: 'dismiss', + visible: false, + dismissed: true, + wasVisible: true, + message: 'Keyboard dismissed via its dismiss key', + }), + ).toBe('Keyboard dismissed via its dismiss key'); +}); diff --git a/packages/command-output/src/system/output.ts b/packages/command-output/src/system/output.ts index ca3d589e90..afb95d4be6 100644 --- a/packages/command-output/src/system/output.ts +++ b/packages/command-output/src/system/output.ts @@ -20,21 +20,38 @@ function appStateCliOutput(result: AppStateCommandResult): CliOutput { // fallow-ignore-next-line complexity function keyboardCliOutput(result: KeyboardCommandResult): CliOutput { - if (result.platform === 'android' && result.action === 'status') { + if (result.action === 'status') { const lines = [ `Keyboard visible: ${result.visible === true ? 'yes' : 'no'}`, - `Input type: ${result.type ?? result.inputType ?? 'unknown'}`, - `Input owner: ${result.inputOwner ?? 'unknown'}`, + ...keyboardReturnKeyLines(result), ]; - if (result.inputMethodPackage) lines.push(`Input method: ${result.inputMethodPackage}`); - if (result.focusedPackage) lines.push(`Focused package: ${result.focusedPackage}`); - if (result.focusedResourceId) lines.push(`Focused resource: ${result.focusedResourceId}`); - lines.push(`Next action: ${androidKeyboardNextAction(result.visible, result.inputOwner)}`); + if (result.platform === 'android') { + lines.push( + `Input type: ${result.type ?? result.inputType ?? 'unknown'}`, + `Input owner: ${result.inputOwner ?? 'unknown'}`, + ); + if (result.inputMethodPackage) lines.push(`Input method: ${result.inputMethodPackage}`); + if (result.focusedPackage) lines.push(`Focused package: ${result.focusedPackage}`); + if (result.focusedResourceId) lines.push(`Focused resource: ${result.focusedResourceId}`); + lines.push(`Next action: ${androidKeyboardNextAction(result.visible, result.inputOwner)}`); + } return { data: result, text: lines.join('\n') }; } return messageCliOutput(result); } +/** + * The return key `status` answers with, on a visible keyboard only: the kind, and the key's own + * wording where the platform reads it. A hidden keyboard reports neither, so nothing is printed. + */ +function keyboardReturnKeyLines(result: KeyboardCommandResult): string[] { + if (result.visible !== true || result.returnKey === undefined) return []; + return [ + `Return key: ${result.returnKey}`, + ...(result.returnKeyLabel ? [`Return key label: ${result.returnKeyLabel}`] : []), + ]; +} + function clipboardCliOutput(result: ClipboardCommandResult): CliOutput { if (result.action === 'read') return { data: result, text: result.text }; return messageCliOutput(result); diff --git a/packages/command-surface/src/command-output-schemas-system.test.ts b/packages/command-surface/src/command-output-schemas-system.test.ts index 35b4fdea40..624e168a6a 100644 --- a/packages/command-surface/src/command-output-schemas-system.test.ts +++ b/packages/command-surface/src/command-output-schemas-system.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { KEYBOARD_RETURN_KEYS } from '@agent-device/contracts/keyboard'; import { COMMAND_OWNER_FILES, ownerFilesForCommand, @@ -61,3 +62,13 @@ test('every system-owned command claims this module, and the module claims nothi 'a command the registry attributes to this module is missing from SYSTEM_COMMAND_OUTPUT_SCHEMAS (or vice versa)', ); }); + +// The `returnKey` word list is written out in the schema rather than spread from the contract, +// because the module's eager closure is budgeted and one value import of the contract's vocabulary +// would grow it. This is where the two stay in step: the schema may not advertise a kind the +// command's result type cannot carry, nor drop one it can. +test('the keyboard output schema publishes exactly the contract return-key vocabulary', () => { + assert.deepEqual(SYSTEM_COMMAND_OUTPUT_SCHEMAS.keyboard.properties?.returnKey?.enum, [ + ...KEYBOARD_RETURN_KEYS, + ]); +}); diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 10834c6d55..5ac78beaf5 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -263,6 +263,24 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { focusedPackage: stringSchema(), focusedResourceId: stringSchema(), inputOwner: enumSchema(['app', 'ime', 'unknown']), + // `status` on a visible keyboard only: what its return key offers, and where the platform can + // read it, the wording the key shows. `command-output-schemas-system.test.ts` pins these twelve + // words to `KEYBOARD_RETURN_KEYS`, the one owner of the vocabulary. + returnKey: enumSchema([ + 'go', + 'search', + 'send', + 'next', + 'done', + 'previous', + 'join', + 'route', + 'continue', + 'enter', + 'none', + 'unknown', + ]), + returnKeyLabel: stringSchema(), message: stringSchema(), }, ['platform', 'action'], @@ -301,7 +319,7 @@ const foldCommandDescription = `Fold or unfold a foldable iPhone simulator (iPho const appSwitcherCommandDescription = 'Open the device app switcher to inspect or change foreground apps. This changes the visible system UI and may move focus away from the current app.'; const keyboardCommandDescription = - 'Inspect Android keyboard visibility/type or press/dismiss the device keyboard. The dismiss action hides the keyboard. It taps the keyboard dismiss/hide key when one is exposed, verifies the keyboard closed, and reports UNSUPPORTED_OPERATION when the keyboard stays up \u2014 background taps are never attempted. On iOS a keyboard with no dismiss key adds error.details.reason keyboard_dismiss_unavailable with dispatched: no; one whose key was tapped and stayed visible has no reason and dispatched: unknown. Android reports keyboard_dismiss_unavailable with dispatched: unknown, since its own dismiss keyevents were already sent.'; + "Report or act on the device keyboard. The status action reports whether the keyboard is visible and, when it is, what its return key offers: `returnKey` is go, search, send, next, done, previous, join, route, continue, enter, none, or unknown, and `returnKeyLabel` is the wording the key itself shows when the platform can read it, trimmed of surrounding whitespace, which can be an app's custom or a localized string. `none` means the field offers no action key, which only Android can establish; `unknown` means this read could not name the key, not that the keyboard has one: on iOS the readable key carries wording this read does not map, or the keyboard exposes no readable key at all; on Android the editor's imeOptions name no action. A hidden keyboard reports neither field. The get action answers the same as status. The dismiss action hides the keyboard. It taps the keyboard dismiss/hide key when one is exposed, verifies the keyboard closed, and reports UNSUPPORTED_OPERATION when the keyboard stays up \u2014 background taps are never attempted. On iOS a keyboard with no dismiss key adds error.details.reason keyboard_dismiss_unavailable with dispatched: no; one whose key was tapped and stayed visible has no reason and dispatched: unknown. Android reports keyboard_dismiss_unavailable with dispatched: unknown, since its own dismiss keyevents were already sent. The enter action presses the return key."; const clipboardCommandDescription = 'Read the current device clipboard text, or replace its contents with the given text. Android runs both through the clipboard service shell command, and a build that implements none (Android 16 does not) refuses with UNSUPPORTED_OPERATION rather than reporting an empty clipboard.'; const actionButtonCommandDescription = @@ -705,7 +723,7 @@ const keyboardCommandFacet = defineCommandFacet({ effect: 'destructive', text: { summary: 'Inspect, press, or dismiss the device keyboard', - searchHint: 'return', + searchHint: 'ime gboard submit', }, metadata: keyboardCommandMetadata, run: (client, input) => client.command.keyboard(input), diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index a89de50458..06abd92ec9 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -1,5 +1,6 @@ import type { ScreenshotScreenMask } from './screenshot.ts'; import type { AppStateRuntimeResult } from './app-state-runtime.ts'; +import type { KeyboardReturnKey } from './keyboard.ts'; import type { ElementTextTarget } from './element-text-target.ts'; import type { BackMode } from './back-mode.ts'; import type { IosSystemSurfaceProvenance } from './ios-system-surface.ts'; @@ -205,22 +206,39 @@ export type SnapshotOptions = BaseSnapshotOptions & { }; /** - * Android's live IME status read. Optional on {@link Interactor}: parity with the retired leaf, - * which refused `status`/`get` on every other family (no other platform's runner exposes one). - * A single-member discriminated union rather than a bare object so the daemon can derive its wire - * `platform` label from `kind` the same way it does for dismiss and enter — an owner can only ever - * produce its own result shape. + * What `keyboard status` answers with, discriminated by which owner produced it: Android's IME + * probe fields, or Apple's keyboard read (visible plus the focused field's return key). The daemon + * derives the wire `platform` label from `kind` rather than re-deriving it from the device, so an + * owner can only ever produce its own result shape. + * + * `returnKey` is the one field both owners carry, and it is present only on a visible keyboard: a + * hidden one has no return key, so neither owner reports one. Each owner names the kinds its own + * reading establishes; neither guesses the other's. */ -export type KeyboardStatusResult = Readonly<{ - kind: 'ime-probe'; - visible: boolean; - inputType?: string; - type?: string; - inputMethodPackage?: string; - focusedPackage?: string; - focusedResourceId?: string; - inputOwner?: string; -}>; +export type KeyboardStatusResult = + | Readonly<{ + kind: 'ime-probe'; + visible: boolean; + inputType?: string; + type?: string; + inputMethodPackage?: string; + focusedPackage?: string; + focusedResourceId?: string; + inputOwner?: string; + returnKey?: KeyboardReturnKey; + returnKeyLabel?: string; + }> + | Readonly<{ + kind: 'keyboard'; + visible: boolean; + /** + * The kinds the Apple reading can establish. `none` is excluded: Apple's return control always + * offers a key when one is reachable, so this owner can only name one or answer `unknown`, + * never establish absence (Android's `IME_ACTION_NONE` can). + */ + returnKey?: Exclude; + returnKeyLabel?: string; + }>; /** * Owner-shaped dismiss evidence, discriminated by which owner produced it: Android's IME probe @@ -482,7 +500,8 @@ export type Interactor = { mediaRewind?(): Promise; /** Optional (parity with `mediaPlayPause`): presses the media fast-forward key once. */ mediaFastForward?(): Promise; - /** Optional: only Android implements a live status read (see {@link KeyboardStatusResult}). */ + /** Optional: owners with no live keyboard status read leave it undefined. Each owner answers with + * its own {@link KeyboardStatusResult} kind. */ keyboardStatus?(): Promise; /** Optional: platforms with no keyboard-dismiss concept leave it undefined. */ keyboardDismiss?(): Promise; diff --git a/packages/contracts/src/keyboard.ts b/packages/contracts/src/keyboard.ts index 4bafe39c3f..c8e8b49ce0 100644 --- a/packages/contracts/src/keyboard.ts +++ b/packages/contracts/src/keyboard.ts @@ -1,9 +1,38 @@ +/** + * The return key one platform reports for the focused field, in the vocabulary both platforms' + * readings project into: Apple's `UITextInput.returnKeyType` wording and Android's + * `EditorInfo.imeOptions` action bits each map onto these kinds. + * + * `unknown` means this read could not name the key, not that the keyboard has one: the Apple runner + * answers it when the return key it found carries wording this vocabulary does not map (its label is + * still reported), or when the keyboard exposes no readable key at all; Android answers it when the + * focused editor's `imeOptions` name no action. `none` means the platform established that the field + * offers no action key at all (Android's `IME_ACTION_NONE`); no other platform can establish absence, + * so only Android reports it. + */ +export const KEYBOARD_RETURN_KEYS = [ + 'go', + 'search', + 'send', + 'next', + 'done', + 'previous', + 'join', + 'route', + 'continue', + 'enter', + 'none', + 'unknown', +] as const; + +export type KeyboardReturnKey = (typeof KEYBOARD_RETURN_KEYS)[number]; + /** * Closed result of the `keyboard` command, grounded in the runtime operation * results projected by `packages/daemon/src/execution/keyboard-runtime.ts`. * * `platform` and `action` are always present; the remaining fields appear per - * branch (Android `status`/`dismiss` carry the keyboard-state fields; `enter` + * branch (`status` carries the keyboard-state fields, and a visible one its return key; `enter` * and iOS `dismiss` carry a `message`). It is kept as a flat closed shape rather * than a five-way `platform`×`action` union because the per-branch field sets * overlap heavily and the underlying Android keyboard-state types live in the @@ -24,6 +53,13 @@ export type KeyboardCommandResult = { focusedPackage?: string; focusedResourceId?: string; inputOwner?: 'app' | 'ime' | 'unknown'; + /** `status` on a visible keyboard: what its return key offers. Omitted when the keyboard is not + * visible — a hidden keyboard has no return key to report. */ + returnKey?: KeyboardReturnKey; + /** `status` on a visible keyboard whose platform reads the key's own wording: the label the key + * shows with its surrounding whitespace trimmed, which can be an app's custom or a localized + * string. Both producers read it from a dump or the accessibility tree, and trim it there. */ + returnKeyLabel?: string; message?: string; /** iOS `dismiss` only (#1598): which mechanism actually resigned the * keyboard — 'dismissKey' (tapped the keyboard's own Hide/Dismiss/Done diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index e9da08378e..7b06d3d45e 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -853,11 +853,13 @@ export const alertRuntimePlanUses = Object.freeze([ ] as const); /** - * `keyboard`'s action-selected uses (ADR 0019 §9: one bind per handler). `status` is Android-only - * (parity with the retired leaf's per-family rejection of `status`/`get` everywhere else); - * `dismiss` and `enter` are the cross-platform legs. The daemon's `keyboard-runtime.ts` admits - * and binds exactly one of the three exported uses per request, keyed by the parsed action — - * never all three. + * `keyboard`'s action-selected uses (ADR 0019 §9: one bind per handler). `status` reads the + * keyboard, `dismiss` and `enter` press it; the three legs differ in what they do to the device, so + * the daemon's `keyboard-runtime.ts` admits and binds exactly one of the three exported uses per + * request, keyed by the parsed action — never all three. Which legs one owner serves is the owner's + * own facts to say, and no owner serves the same set: Android and the Apple runner serve all three + * (the Apple runner on iOS and iPadOS only), HarmonyOS serves `dismiss` and `enter`, and WebDriver + * serves all three on Android but only `dismiss` on iOS, whose probe has no `status` result shape. */ export const keyboardRuntimePlanUses = Object.freeze([ keyboardStatusUse, diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 74a0dfad06..4d1bba3561 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -51,7 +51,8 @@ export const TAP_KEYBOARD_OCCLUDES_TARGET_REASON = 'tap_keyboard_occludes_target * stance `scroll-gesture.ts` takes for the surface it refuses to swipe. * * The verdict is read from the tree the command measured against, and no owner re-probes the - * keyboard to confirm it: `keyboard status`/`get` are Android-only, and #1542's double-check + * keyboard to confirm it: `keyboard status`/`get` answer only what a visible keyboard's return key + * offers, and #1542's double-check * confirms the target's own live rect, which a covering keyboard leaves intact. So the hint names * the re-measurement instead of pretending the refusal already has it. */ diff --git a/packages/daemon/src/execution/__tests__/keyboard-runtime.test.ts b/packages/daemon/src/execution/__tests__/keyboard-runtime.test.ts index 0342812f20..bdcb5d03f4 100644 --- a/packages/daemon/src/execution/__tests__/keyboard-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/keyboard-runtime.test.ts @@ -58,7 +58,7 @@ const available = Object.freeze({ available: true } as const); const unavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf' as const, - hint: 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', + hint: 'keyboard status is not available through the public HarmonyOS HDC API; use keyboard dismiss or enter', }); function runtimeHarness( @@ -114,6 +114,8 @@ test('android status admits keyboardStatusUse and reports the platform-shaped st focusedPackage: 'com.example.app', focusedResourceId: 'com.example.app:id/field', inputOwner: 'app', + returnKey: 'search', + returnKeyLabel: 'Search the store', }); const resolved = await resolveBoundKeyboardRuntime({ @@ -137,6 +139,100 @@ test('android status admits keyboardStatusUse and reports the platform-shaped st focusedPackage: 'com.example.app', focusedResourceId: 'com.example.app:id/field', inputOwner: 'app', + returnKey: 'search', + returnKeyLabel: 'Search the store', + }); +}); + +// The Apple runner answers with its own `keyboard` shape: two fields, and the wire platform follows +// the kind that came back rather than a second guess from the device. +test("iOS status reports the runner's keyboard shape and its return key", async () => { + const harness = runtimeHarness(iosDevice, 'apple', { + unsupported: unavailable, + status: available, + dismiss: available, + enter: available, + }); + harness.keyboardStatus.mockResolvedValue({ + kind: 'keyboard', + visible: true, + returnKey: 'done', + returnKeyLabel: 'Done', + }); + + const resolved = await resolveBoundKeyboardRuntime({ + platformServices: daemonPlatformServicesFixture(), + device: iosDevice, + positionals: ['status'], + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }); + + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + expect(await resolved.execute({})).toEqual({ + platform: 'ios', + action: 'status', + visible: true, + returnKey: 'done', + returnKeyLabel: 'Done', + }); +}); + +// A visible keyboard whose owner reads visibility alone (a WebDriver session's probe) names no kind. +// The wire still has to answer the question, so it publishes `unknown` rather than silence: an +// omitted `returnKey` means there was no visible keyboard, which is a different claim. +test('a visible keyboard whose owner names no return key publishes unknown', async () => { + const harness = runtimeHarness(androidDevice, 'android', { + unsupported: unavailable, + status: available, + dismiss: available, + enter: available, + }); + harness.keyboardStatus.mockResolvedValue({ kind: 'ime-probe', visible: true }); + + const resolved = await resolveBoundKeyboardRuntime({ + platformServices: daemonPlatformServicesFixture(), + device: androidDevice, + positionals: ['status'], + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }); + + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + const result = await resolved.execute({}); + expect(result).toMatchObject({ visible: true, returnKey: 'unknown' }); + // A label belongs to a named key, so the fallback never carries one. + expect(result).not.toHaveProperty('returnKeyLabel'); +}); + +// Closest negative: a hidden keyboard has no return key, so both fields stay off the wire even +// though the owner's own reading could still see the last editor's action. +test('status reports no return key while the keyboard is hidden', async () => { + const harness = runtimeHarness(iosDevice, 'apple', { + unsupported: unavailable, + status: available, + dismiss: available, + enter: available, + }); + harness.keyboardStatus.mockResolvedValue({ kind: 'keyboard', visible: false }); + + const resolved = await resolveBoundKeyboardRuntime({ + platformServices: daemonPlatformServicesFixture(), + device: iosDevice, + positionals: ['status'], + inspectFacts: harness.inspectFacts, + bindDevice: harness.bindDevice, + }); + + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + expect(await resolved.execute({})).toEqual({ + platform: 'ios', + action: 'status', + visible: false, + returnKey: undefined, }); }); @@ -180,8 +276,8 @@ test('`return` is an alias for `enter`', async () => { expect(harness.bindDevice).toHaveBeenCalledWith(androidDevice, keyboardEnterUse); }); -test('android status is refused on iOS with the retired in-handler hint', async () => { - const harness = runtimeHarness(iosDevice, 'apple', { +test("status is refused on HarmonyOS with this owner's leaf hint", async () => { + const harness = runtimeHarness(harmonyDevice, 'harmonyos', { unsupported: unavailable, dismiss: available, enter: available, @@ -189,7 +285,7 @@ test('android status is refused on iOS with the retired in-handler hint', async const resolved = await resolveBoundKeyboardRuntime({ platformServices: daemonPlatformServicesFixture(), - device: iosDevice, + device: harmonyDevice, positionals: ['status'], inspectFacts: harness.inspectFacts, bindDevice: harness.bindDevice, diff --git a/packages/daemon/src/execution/keyboard-runtime.ts b/packages/daemon/src/execution/keyboard-runtime.ts index 71e394afc5..8448651840 100644 --- a/packages/daemon/src/execution/keyboard-runtime.ts +++ b/packages/daemon/src/execution/keyboard-runtime.ts @@ -77,6 +77,7 @@ const KEYBOARD_STATUS_PLATFORM_LABEL: Record< 'android' | 'harmonyos' | 'ios' > = { 'ime-probe': 'android', + keyboard: 'ios', }; const KEYBOARD_DISMISS_PLATFORM_LABEL: Record< @@ -139,22 +140,40 @@ async function admitKeyboardAction< return { ok: true, execute: (context) => execute(runtime, context) }; } -/** `keyboard status` — Android-only; every other owner refuses. */ +/** + * `keyboard status`. + * + * The return key is projected so the wire contract holds for every owner: the two owners that read + * a key name it, and a visible keyboard whose owner reads visibility alone (`keyboardStatus` on a + * WebDriver session answers only that) publishes `unknown` rather than silence — an omitted + * `returnKey` means there was no visible keyboard to report one for. + */ function executeKeyboardStatus( runtime: BoundDeviceRuntime, context: DaemonCommandContext, ): Promise> { - return runtime.operations.keyboardStatus(keyboardActionInput(context)).then((state) => ({ - platform: KEYBOARD_STATUS_PLATFORM_LABEL[state.kind], - action: 'status', - visible: state.visible, - inputType: state.inputType, - type: state.type, - inputMethodPackage: state.inputMethodPackage, - focusedPackage: state.focusedPackage, - focusedResourceId: state.focusedResourceId, - inputOwner: state.inputOwner, - })); + return runtime.operations.keyboardStatus(keyboardActionInput(context)).then((state) => { + const returnKey = state.returnKey ?? (state.visible === true ? 'unknown' : undefined); + const base = { + platform: KEYBOARD_STATUS_PLATFORM_LABEL[state.kind], + action: 'status', + visible: state.visible, + returnKey, + ...(state.returnKeyLabel === undefined || returnKey === undefined + ? {} + : { returnKeyLabel: state.returnKeyLabel }), + }; + if (state.kind === 'keyboard') return base; + return { + ...base, + inputType: state.inputType, + type: state.type, + inputMethodPackage: state.inputMethodPackage, + focusedPackage: state.focusedPackage, + focusedResourceId: state.focusedResourceId, + inputOwner: state.inputOwner, + }; + }); } /** `keyboard dismiss`. */ diff --git a/packages/platform-android/src/__tests__/device-input-state.test.ts b/packages/platform-android/src/__tests__/device-input-state.test.ts index 8e15bb51d1..87347ed74d 100644 --- a/packages/platform-android/src/__tests__/device-input-state.test.ts +++ b/packages/platform-android/src/__tests__/device-input-state.test.ts @@ -43,10 +43,146 @@ test('getAndroidKeyboardStatusWithAdb exposes active input method package', asyn focusedPackage: undefined, focusedResourceId: undefined, inputOwner: 'unknown', + // Visible with no `imeOptions` to read: the keyboard is up, so this read cannot say what its + // return key does, which is `unknown` rather than a guessed action. + returnKey: 'unknown', }); }); }); +// Each `EditorInfo.IME_ACTION_*` the current editor declares, and the kind `keyboard status` +// reports for it. An unspecified action is the IME's own choice, so it stays `unknown`. +test('getAndroidKeyboardState reports the current editor action as the return key', async () => { + const cases: readonly [imeOptions: string, returnKey: string][] = [ + ['0x00000002', 'go'], + ['0x00000003', 'search'], + ['0x00000004', 'send'], + ['0x00000005', 'next'], + ['0x00000006', 'done'], + ['0x00000007', 'previous'], + ['0x00000001', 'none'], + ['0x00000000', 'unknown'], + ]; + for (const [imeOptions, returnKey] of cases) { + await withFakeAdb( + (args) => + args.join(' ') === 'shell dumpsys input_method' + ? [ + 'mInputShown=true mIsInputViewShown=true', + `inputType=0x1 imeOptions=${imeOptions} privateImeOptions=null`, + ].join('\n') + : { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }, + async ({ device }) => { + const state = await getAndroidKeyboardState(device); + assert.equal(state.returnKey, returnKey, imeOptions); + assert.equal(state.returnKeyLabel, undefined, imeOptions); + }, + ); + } +}); + +// Action bits beyond `IME_MASK_ACTION` are flags, not action. The only one `keyboard status` reads +// is `IME_FLAG_NO_ENTER_ACTION`, which asks the IME to keep the action off the enter key — so the +// key the user presses is enter, whatever action the editor declared. +test('getAndroidKeyboardState reports enter when the editor withholds the action from the key', async () => { + await withFakeAdb( + (args) => + args.join(' ') === 'shell dumpsys input_method' + ? [ + 'mInputShown=true mIsInputViewShown=true', + 'inputType=0x1 imeOptions=0x60000006', + 'actionLabel=Send Message privateKey=null', + ].join('\n') + : { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }, + async ({ device }) => { + const state = await getAndroidKeyboardState(device); + assert.equal(state.returnKey, 'enter'); + // The withheld action's own label is not what the key shows, so it is not reported either. + assert.equal(state.returnKeyLabel, undefined); + }, + ); +}); + +// An app's `actionLabel` is the wording its key shows. The kind stays the declared action's: the +// label is evidence about the key's wording, not a second claim about what it does. +test('getAndroidKeyboardState reports a custom action label beside its action kind', async () => { + await withFakeAdb( + (args) => + args.join(' ') === 'shell dumpsys input_method' + ? [ + 'mInputShown=true mIsInputViewShown=true', + 'inputType=0x1 imeOptions=0x00000004', + 'actionLabel=Send Message privateKey=null', + ].join('\n') + : { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }, + async ({ device }) => { + const state = await getAndroidKeyboardState(device); + assert.equal(state.returnKey, 'send'); + assert.equal(state.returnKeyLabel, 'Send Message'); + }, + ); +}); + +// An app's `actionLabel` can hold a comma. Only a comma that opens another `key=value` field ends +// the label, so a label that spells out two actions keeps both words. +test('getAndroidKeyboardState keeps commas inside a custom action label', async () => { + await withFakeAdb( + (args) => + args.join(' ') === 'shell dumpsys input_method' + ? [ + 'mInputShown=true mIsInputViewShown=true', + 'inputType=0x1 imeOptions=0x00000005', + 'actionLabel=Next, then Send privateKey=null', + ].join('\n') + : { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }, + async ({ device }) => { + const state = await getAndroidKeyboardState(device); + assert.equal(state.returnKey, 'next'); + assert.equal(state.returnKeyLabel, 'Next, then Send'); + }, + ); +}); + +// The other dump shape, where `EditorInfo.toString()` wraps the editor in braces, so the label ends +// at `}` rather than at another named field. A comma inside the label survives there too. +test('getAndroidKeyboardState keeps a comma in an action label a brace-delimited dump carries', async () => { + await withFakeAdb( + (args) => + args.join(' ') === 'shell dumpsys input_method' + ? [ + 'mInputShown=true mIsInputViewShown=true', + 'inputType=0x1 imeOptions=0x00000004', + 'editorInfo=EditorInfo{imeOptions=0x4, actionLabel=Save, now}', + ].join('\n') + : { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }, + async ({ device }) => { + const state = await getAndroidKeyboardState(device); + assert.equal(state.returnKey, 'send'); + assert.equal(state.returnKeyLabel, 'Save, now'); + }, + ); +}); + +// Closest negative: a hidden keyboard keeps its last editor's attributes in the dump. Nothing is +// being typed into, so no key offers anything, and both fields stay out of the state. +test('getAndroidKeyboardState reports no return key while the keyboard is hidden', async () => { + await withFakeAdb( + (args) => + args.join(' ') === 'shell dumpsys input_method' + ? [ + 'mInputShown=false mIsInputViewShown=false', + 'inputType=0x1 imeOptions=0x00000003 actionLabel=Go', + ].join('\n') + : { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }, + async ({ device }) => { + const state = await getAndroidKeyboardState(device); + assert.equal(state.visible, false); + assert.equal('returnKey' in state, false); + assert.equal('returnKeyLabel' in state, false); + }, + ); +}); + test('getAndroidKeyboardStatusWithAdb classifies tolerated adb failures with actionable hints', async () => { // allowFailure regression: the executor returns the nonzero result instead of // throwing, so the classified hint must come from the result-to-error path. diff --git a/packages/platform-android/src/device-input-state.ts b/packages/platform-android/src/device-input-state.ts index 930f3d6447..0682fab75a 100644 --- a/packages/platform-android/src/device-input-state.ts +++ b/packages/platform-android/src/device-input-state.ts @@ -18,6 +18,7 @@ import { isFallbackAndroidInputMethodResource, readAndroidActiveInputMethodPackage, } from '@agent-device/contracts/android-input-ownership'; +import type { KeyboardReturnKey } from '@agent-device/contracts/keyboard'; const ANDROID_INPUT_TYPE_CLASS_MASK = 0x0000000f; const ANDROID_INPUT_TYPE_CLASS_TEXT = 0x00000001; @@ -30,6 +31,10 @@ const ANDROID_TEXT_VARIATION_WEB_EMAIL_ADDRESS = 0x000000d0; const ANDROID_TEXT_VARIATION_PASSWORD = 0x00000080; const ANDROID_TEXT_VARIATION_WEB_PASSWORD = 0x000000e0; const ANDROID_TEXT_VARIATION_VISIBLE_PASSWORD = 0x00000090; +/** `EditorInfo.IME_MASK_ACTION`: the bits of `imeOptions` that name the return key's action. */ +const ANDROID_IME_MASK_ACTION = 0x000000ff; +/** `EditorInfo.IME_FLAG_NO_ENTER_ACTION`: the action stays off the enter key, which is then enter. */ +const ANDROID_IME_FLAG_NO_ENTER_ACTION = 0x40000000; const ANDROID_KEYBOARD_DISMISS_MAX_ATTEMPTS = 2; const ANDROID_KEYBOARD_DISMISS_RETRY_DELAY_MS = 120; const ANDROID_KEYCODE_ESCAPE = '111'; @@ -72,6 +77,17 @@ type AndroidKeyboardType = | 'datetime' | 'unknown'; +/** `EditorInfo.IME_ACTION_*`: the action one masked bit pattern names. */ +const ANDROID_IME_RETURN_KEY_BY_ACTION: ReadonlyMap = new Map([ + [0x01, 'none'], + [0x02, 'go'], + [0x03, 'search'], + [0x04, 'send'], + [0x05, 'next'], + [0x06, 'done'], + [0x07, 'previous'], +]); + export type AndroidKeyboardState = { visible: boolean; inputType?: string; @@ -80,6 +96,11 @@ export type AndroidKeyboardState = { focusedPackage?: string; focusedResourceId?: string; inputOwner: AndroidInputOwner; + /** What the current editor's return key offers; absent when the keyboard is not visible. */ + returnKey?: KeyboardReturnKey; + /** The editor's own custom action label, with its surrounding whitespace trimmed; absent when the + * editor names none. */ + returnKeyLabel?: string; }; export type AndroidKeyboardDismissResult = AndroidKeyboardState & { @@ -182,6 +203,9 @@ function parseAndroidKeyboardState(stdout: string): AndroidKeyboardState { focusedResourceId, inputMethodPackage, ); + // The return key belongs to the visible keyboard: a hidden one keeps the editor's attributes in + // its dump while nothing is being typed into, so its action would not be what any key offers. + const returnKeyFacts = visible === true ? parseAndroidReturnKey(stdout) : {}; return { visible: visible ?? false, @@ -191,9 +215,55 @@ function parseAndroidKeyboardState(stdout: string): AndroidKeyboardState { focusedPackage, focusedResourceId, inputOwner, + ...returnKeyFacts, }; } +/** + * What the current editor's `imeOptions` say its return key does, read from the same + * `dumpsys input_method` output the rest of this state comes from. + * + * `IME_FLAG_NO_ENTER_ACTION` outranks the action bits: it asks the IME to keep the action off the + * enter key, so the key the user presses is enter whatever the action is. Otherwise the masked + * action names the key, and an app's custom `actionLabel` is reported beside it as the wording the + * key shows. An unspecified action is the IME's own choice, so it answers `unknown` rather than + * guessing which key a given IME offers. No `imeOptions` to read is also `unknown`: the keyboard is + * up, but this read cannot say what its key does. + */ +function parseAndroidReturnKey( + stdout: string, +): Readonly<{ returnKey?: KeyboardReturnKey; returnKeyLabel?: string }> { + const rawImeOptions = parseLastDumpsysValue(stdout, /\bimeOptions=0x([0-9a-fA-F]+)\b/gi); + const imeOptions = rawImeOptions ? Number.parseInt(rawImeOptions, 16) >>> 0 : Number.NaN; + const actionLabel = parseLastDumpsysActionLabel(stdout); + if (Number.isNaN(imeOptions)) { + return actionLabel + ? { returnKey: 'unknown', returnKeyLabel: actionLabel } + : { returnKey: 'unknown' }; + } + if ((imeOptions & ANDROID_IME_FLAG_NO_ENTER_ACTION) !== 0) { + return { returnKey: 'enter' }; + } + const action = imeOptions & ANDROID_IME_MASK_ACTION; + const returnKey = ANDROID_IME_RETURN_KEY_BY_ACTION.get(action) ?? 'unknown'; + return actionLabel ? { returnKey, returnKeyLabel: actionLabel } : { returnKey }; +} + +/** + * The editor's own `actionLabel`, with its surrounding whitespace trimmed. `EditorInfo.toString()` + * writes one space-separated `key=value` run, so the label — which can hold spaces and commas, such + * as `Submit, then Continue` — ends at the next `key=value` field or at the line. A comma only ends + * it when a field name follows, which is how a brace-delimited dump separates its own fields. + */ +function parseLastDumpsysActionLabel(stdout: string): string | undefined { + let label: string | undefined; + for (const match of stdout.matchAll(/\bactionLabel=([^\n]*?)(?=\s+\w+=|,\s*\w+=|[}\]]|\s*$)/g)) { + const value = match[1]?.trim(); + if (value && value !== 'null') label = value; + } + return label; +} + function parseLegacyImeWindowVisibility(stdout: string): boolean | null { const imeWindowVisibility = stdout.match(/\bmImeWindowVis=0x([0-9a-fA-F]+)\b/); const rawFlags = imeWindowVisibility?.[1]; diff --git a/packages/platform-apple/src/__tests__/index-runner-requests.test.ts b/packages/platform-apple/src/__tests__/index-runner-requests.test.ts index 3f86e304cd..1c34d343b3 100644 --- a/packages/platform-apple/src/__tests__/index-runner-requests.test.ts +++ b/packages/platform-apple/src/__tests__/index-runner-requests.test.ts @@ -190,6 +190,7 @@ const INTERACTOR_SITES: Record = { 'tvos.interactor-tv-remote.hold': [TVOS_SIMULATOR, (i) => i.tvRemote!('select', 500)], 'ios-simulator.interactor-keyboard-dismiss.dismiss': [IOS_SIMULATOR, (i) => i.keyboardDismiss!()], 'ios-simulator.interactor-keyboard-enter.return': [IOS_SIMULATOR, (i) => i.keyboardEnter!()], + 'ios-simulator.interactor-keyboard-status.read': [IOS_SIMULATOR, (i) => i.keyboardStatus!()], 'ios-simulator.interactor-write-clipboard.text': [ IOS_SIMULATOR, (i) => i.writeClipboard!('one-time code 246810'), diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index e5988872d6..08f324f3ea 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -84,6 +84,7 @@ const RUNNER_TRANSPORT_METHODS: Record< tvRemote: { invoke: (i) => i.tvRemote!('select'), runnerCommand: 'remotePress' }, keyboardDismiss: { invoke: (i) => i.keyboardDismiss!(), runnerCommand: 'keyboardDismiss' }, keyboardEnter: { invoke: (i) => i.keyboardEnter!(), runnerCommand: 'keyboardReturn' }, + keyboardStatus: { invoke: (i) => i.keyboardStatus!(), runnerCommand: 'keyboardStatus' }, writeClipboard: { invoke: (i) => i.writeClipboard!('hi'), runnerCommand: 'pasteboardWrite' }, // R59: same reading as `readTextAtPoint` — the macOS-helper branch is reachable only for a // local desktop surface, which a provider-owned mobile device never carries, so every diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 56c8eeb510..e4fab3635b 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -18,6 +18,7 @@ import { type RunnerCommand, } from '@agent-device/apple-runner'; import { toAppleTvRemoteButton } from '@agent-device/contracts/tv-remote'; +import { KEYBOARD_RETURN_KEYS, type KeyboardReturnKey } from '@agent-device/contracts/keyboard'; import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; import { macOsHelperSurface, @@ -221,6 +222,19 @@ export function createAppleInteractor( runnerOpts, ); }, + keyboardStatus: async () => { + const result = await runAppleRunnerCommand( + device, + { command: 'keyboardStatus', appBundleId: runnerContext.appBundleId }, + runnerOpts, + ); + return { + kind: 'keyboard', + visible: readRunnerBoolean(result, 'visible') === true, + returnKey: readRunnerKeyboardReturnKey(result), + returnKeyLabel: readRunnerString(result, 'keyboardReturnKeyLabel'), + }; + }, keyboardDismiss: async () => { const result = await runAppleRunnerCommand( device, @@ -460,6 +474,32 @@ function readRunnerString(result: Record, key: string): string return typeof value === 'string' ? value : undefined; } +/** + * The kinds this owner can answer with: the shared vocabulary without `none`, which only Android's + * `IME_ACTION_NONE` reading establishes. Derived from it rather than restated, so one list owns the + * words. + */ +const APPLE_KEYBOARD_RETURN_KEYS = KEYBOARD_RETURN_KEYS.filter( + (key): key is Exclude => key !== 'none', +); + +/** + * The return key the runner named, degraded to `unknown` when the wire carries a value this owner + * cannot answer with: an older runner must not publish a kind the command's contract disallows, and + * `none` is one — Apple has no reading that establishes a field offers no action key, so a runner + * that answers it is not speaking this vocabulary. Absent is absent — a hidden keyboard reports no + * key — which the caller keeps distinct. + */ +function readRunnerKeyboardReturnKey( + result: Record, +): Exclude | undefined { + const value = readRunnerString(result, 'keyboardReturnKey'); + if (value === undefined) return undefined; + return (APPLE_KEYBOARD_RETURN_KEYS as readonly string[]).includes(value) + ? (value as Exclude) + : 'unknown'; +} + async function setRunnerOrientation( device: DeviceInfo, orientation: DeviceRotation, diff --git a/packages/platform-apple/src/navigation/runtime.ts b/packages/platform-apple/src/navigation/runtime.ts index 08740751e7..7af61493df 100644 --- a/packages/platform-apple/src/navigation/runtime.ts +++ b/packages/platform-apple/src/navigation/runtime.ts @@ -123,21 +123,24 @@ const keyboardCellUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', } as const); -const keyboardStatusUnsupported = Object.freeze({ - available: false, - reason: 'unsupported-platform-leaf', - hint: 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', -} as const); -/** Apple never had a live keyboard status read: every eligible cell still refuses `status`/`get` - * with the retired in-handler hint. */ -function appleKeyboardStatusFact(device: DeviceInfo): RuntimeOperationFact { - return appleMobileInputEligible(device) ? keyboardStatusUnsupported : keyboardCellUnavailable; -} -function appleKeyboardDismissFact(device: DeviceInfo): RuntimeOperationFact { +/** + * `dismiss` and `enter` press the software keyboard, which every handheld Apple leaf has, so both + * read `orientation`'s eligibility: unavailable on tvOS, macOS and watchOS, available on iOS and + * visionOS, where the keyboard exists even though the runner reads it differently. + */ +function appleKeyboardFact(device: DeviceInfo): RuntimeOperationFact { return appleMobileInputEligible(device) ? available : keyboardCellUnavailable; } -function appleKeyboardEnterFact(device: DeviceInfo): RuntimeOperationFact { - return appleMobileInputEligible(device) ? available : keyboardCellUnavailable; +/** + * `status` is the one leg the runner can only serve on iOS and iPadOS: it answers from the key + * elements `observedKeyboardKeys(app:)` reads, and that read is `#if os(iOS)` code which queries + * `app.keyboards` and answers nothing elsewhere. On visionOS the same guard makes the visibility read + * answer `nil`, so a caller would get "keyboard hidden" for a keyboard that may be up. visionOS hears + * the same refusal as the leaves with no keyboard at all. + */ +function appleKeyboardStatusFact(device: DeviceInfo): RuntimeOperationFact { + if (!appleMobileInputEligible(device)) return keyboardCellUnavailable; + return isHandheldAppleDevice(device) ? available : keyboardCellUnavailable; } /** Every system button this owner knows is named below; the family denial covers none today. */ @@ -222,8 +225,8 @@ export function appleNavigationFacts(device: DeviceInfo) { ...keyboardRuntimeOperationFacts({ unsupported: keyboardCellUnavailable, status: appleKeyboardStatusFact(device), - dismiss: appleKeyboardDismissFact(device), - enter: appleKeyboardEnterFact(device), + dismiss: appleKeyboardFact(device), + enter: appleKeyboardFact(device), }), }); } diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 146ab7a220..e180f745cd 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -540,34 +540,23 @@ function expectNavigationAndKeyboardFacts( expectOperationAvailability(binding, 'appSwitcher', springboard); // orientation and keyboard dismiss/enter share mobile-input eligibility: unavailable on tvOS - // (focus-only XCUIRemote navigation), macOS (an AppKit desktop host), and watchOS. + // (focus-only XCUIRemote navigation), macOS (an AppKit desktop host), and watchOS. `status` is + // the narrower leg: the runner answers it from the keyboard's own key elements, an iOS-only read, + // so visionOS is refused there even though it can press the keyboard. const mobileInputEligible = device.appleOs !== 'tvos' && device.appleOs !== 'macos' && device.appleOs !== 'watchos'; expectOperationAvailability(binding, 'setOrientation', mobileInputEligible); expectOperationAvailability(binding, 'unlockOrientation', false); expectOperationAvailability(binding, 'keyboardDismiss', mobileInputEligible); expectOperationAvailability(binding, 'keyboardEnter', mobileInputEligible); - - expectKeyboardStatusFact(binding, mobileInputEligible); + expectOperationAvailability( + binding, + 'keyboardStatus', + mobileInputEligible && device.appleOs !== 'visionos', + ); expectTvRemoteFact(binding, device); } -/** Apple never had a live keyboard status read: every eligible leaf still refuses status/get with - * the retired in-handler hint; ineligible leaves fall through the outer cell instead. */ -function expectKeyboardStatusFact( - binding: DeviceBinding, - mobileInputEligible: boolean, -): void { - expect(binding.facts.operations.keyboardStatus.available).toBe(false); - if (mobileInputEligible) { - expect(binding.facts.operations.keyboardStatus).toHaveProperty( - 'hint', - expect.stringContaining('keyboard status/get is currently supported only on Android'), - ); - } - expect(binding.operations.keyboardStatus).toBeUndefined(); -} - /** tv-remote is available only for tvOS, which drives navigation through XCUIRemote presses. */ function expectTvRemoteFact( binding: DeviceBinding, diff --git a/packages/provider-limrun/src/app-log-runtime.test.ts b/packages/provider-limrun/src/app-log-runtime.test.ts index effdccdc3d..3249608e85 100644 --- a/packages/provider-limrun/src/app-log-runtime.test.ts +++ b/packages/provider-limrun/src/app-log-runtime.test.ts @@ -421,16 +421,11 @@ function expectLimrunNavigationAndKeyboardFacts( ? 'tv-remote is supported only on Android TV targets.' : 'tv-remote is supported only on tvOS devices.', }); - if (!isAndroid) { - expect(binding.facts.operations.keyboardStatus).toMatchObject({ - reason: 'unsupported-platform-leaf', - hint: 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', - }); - } + // Limrun iOS rides the shared Apple runner facts, so all three keyboard legs answer there, as they + // do on any other mobile Apple leaf. for (const operation of ['keyboardStatus', 'keyboardDismiss', 'keyboardEnter'] as const) { - const available = isAndroid || operation !== 'keyboardStatus'; - expect(binding.facts.operations[operation].available).toBe(available); - expect(binding.operations[operation]).toBeTypeOf(available ? 'function' : 'undefined'); + expect(binding.facts.operations[operation].available).toBe(true); + expect(binding.operations[operation]).toBeTypeOf('function'); } } diff --git a/scripts/capability-matrix/annotations.ts b/scripts/capability-matrix/annotations.ts index fb846fbc27..9b2769e989 100644 --- a/scripts/capability-matrix/annotations.ts +++ b/scripts/capability-matrix/annotations.ts @@ -163,22 +163,8 @@ export const MATRIX_SECTIONS: readonly MatrixSection[] = [ label: 'Keyboard dismiss, enter, and status', commands: ['keyboard'], link: docs('commands', 'keyboard'), + note: "`status` reports the focused field's return key on a visible keyboard. It does not claim the key can be named: only Android can report `none`, and the Apple runner reports `unknown` for a key whose wording names no action it knows.", cells: { - 'limrun-ios': { - text: 'Dismiss and enter; status unsupported', - admits: ['keyboardDismiss', 'keyboardEnter'], - refuses: ['keyboardStatus'], - }, - 'ios-sim': { - text: 'Dismiss and enter; status unsupported', - admits: ['keyboardDismiss', 'keyboardEnter'], - refuses: ['keyboardStatus'], - }, - 'ios-device': { - text: 'Dismiss and enter; status unsupported', - admits: ['keyboardDismiss', 'keyboardEnter'], - refuses: ['keyboardStatus'], - }, 'harmonyos-emulator': { text: 'Dismiss and enter; status unsupported', admits: ['keyboardDismiss', 'keyboardEnter'], diff --git a/src/__tests__/runner-read-only-registry-parity.test.ts b/src/__tests__/runner-read-only-registry-parity.test.ts index 2fa703d755..97c17b170e 100644 --- a/src/__tests__/runner-read-only-registry-parity.test.ts +++ b/src/__tests__/runner-read-only-registry-parity.test.ts @@ -50,6 +50,9 @@ const REGISTRY_COUNTERPART: Record = { volumeDown: { command: 'hardware-button', positionals: ['volume-down'] }, keyboardDismiss: { command: 'keyboard', positionals: ['dismiss'] }, keyboardReturn: { command: 'keyboard', positionals: ['enter'] }, + // The new runner read pairs with the registry's own read leg, so the parity check below compares + // the runner's read-only trait with `keyboard status`'s recording effect. + keyboardStatus: { command: 'keyboard', positionals: ['status'] }, pasteboardWrite: { command: 'clipboard', positionals: ['write', 'x'] }, }; diff --git a/src/commands/schema/cli-help-command-usage.test.ts b/src/commands/schema/cli-help-command-usage.test.ts index ea84cc839f..c15226c78c 100644 --- a/src/commands/schema/cli-help-command-usage.test.ts +++ b/src/commands/schema/cli-help-command-usage.test.ts @@ -416,10 +416,21 @@ test('keyboard command usage is documented', async () => { const help = await usageForCommand('keyboard'); if (help === null) throw new Error('Expected command help text'); assert.match(help, /keyboard \[status\|get\|dismiss\|enter\|return\]/); + assert.match(help, /Report or act on the device keyboard/); +}); + +// `keyboard status`'s own output vocabulary, so a caller reading help knows what `returnKey` can say +// and which platform can establish that a field offers no key. The complete list, `none` and +// `unknown` included: dropping either from the description must fail here. +test('usageForCommand documents the keyboard return key read', async () => { + const help = await usageForCommand('keyboard'); + assert.equal(help === null, false); assert.match( - help, - /Inspect Android keyboard visibility\/type or press\/dismiss the device keyboard/, + help ?? '', + /`returnKey` is go, search, send, next, done, previous, join, route, continue, enter, none, or unknown/, ); + assert.match(help ?? '', /`returnKeyLabel` is the wording the key itself shows/); + assert.match(help ?? '', /only Android can establish/); }); test('orientation command usage is documented', async () => { diff --git a/test/integration/provider-scenarios/android-keyboard.test.ts b/test/integration/provider-scenarios/android-keyboard.test.ts new file mode 100644 index 0000000000..fd29c4fa4f --- /dev/null +++ b/test/integration/provider-scenarios/android-keyboard.test.ts @@ -0,0 +1,67 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAndroidSettingsWorld } from './android-world.ts'; +import { withProviderScenarioResource } from './harness.ts'; + +type AndroidKeyboardWorld = Awaited>; + +// The world's own `dumpsys input_method` fixture keeps the keyboard hidden, which is what the long +// Settings flow needs. Every case here scripts its own IME dump instead: the return key comes from +// the same output the visibility comes from, so the only thing a case has to say is what that one +// dump reports. +function createAndroidKeyboardWorld(dump: string) { + return async () => + await createAndroidSettingsWorld({ + onAdbExec: (args) => + args.join(' ') === 'shell dumpsys input_method' + ? { stdout: dump, stderr: '', exitCode: 0 } + : undefined, + }); +} + +function countImeDumps(world: AndroidKeyboardWorld): number { + return world.adbCalls.filter((call) => call.join(' ') === 'shell dumpsys input_method').length; +} + +// Reading the return key is not a second device round-trip: it comes from the one `dumpsys +// input_method` the visibility read already made. +test('Provider-backed integration Android keyboard status names the focused field return key', async () => { + const createWorld = createAndroidKeyboardWorld( + [ + 'mInputShown=true mIsInputViewShown=true', + 'mCurMethodId=com.google.android.inputmethod.latin/.LatinIME', + 'inputType=0x200001 imeOptions=0x00000004 actionLabel=Send Message packageName=com.example.demo', + ].join('\n'), + ); + await withProviderScenarioResource(createWorld, async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + const status = await client.command.keyboard({ action: 'status', ...world.selection }); + assert.equal(status.platform, 'android'); + assert.equal(status.action, 'status'); + assert.equal(status.visible, true); + assert.equal(status.returnKey, 'send'); + assert.equal(status.returnKeyLabel, 'Send Message'); + assert.equal(status.inputOwner, 'app'); + assert.equal(countImeDumps(world), 1, JSON.stringify(world.adbCalls)); + }); +}); + +// Closest negative: a hidden keyboard keeps its last editor's attributes in the dump. Nothing is +// being typed into, so no key offers anything, and both fields stay off the answer rather than +// reporting an action no key carries. +test('Provider-backed integration Android keyboard status reports no return key while the keyboard is hidden', async () => { + const createWorld = createAndroidKeyboardWorld( + ['mInputShown=false mIsInputViewShown=false', 'inputType=0x1 imeOptions=0x00000003'].join('\n'), + ); + await withProviderScenarioResource(createWorld, async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + const status = await client.command.keyboard({ action: 'status', ...world.selection }); + assert.equal(status.visible, false); + assert.equal('returnKey' in status, false); + assert.equal('returnKeyLabel' in status, false); + }); +}); diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts index 1873a030d6..dac3eeb479 100644 --- a/test/integration/provider-scenarios/ios-lifecycle.test.ts +++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts @@ -251,6 +251,24 @@ test('Provider-backed integration iOS Settings flow uses scripted simctl and run positionals: ['read'], expectData: { text: 'runner otp 246810' }, }, + // The runner reads the keyboard and the focused field's return control, and the wire shape is + // the runner's own two fields — no Android IME fields on an iOS answer. + { + name: 'read keyboard status', + command: 'keyboard', + positionals: ['status'], + expectData: { + platform: 'ios', + action: 'status', + visible: true, + returnKey: 'search', + returnKeyLabel: 'Search', + }, + assert: (response) => { + assert.equal(response.json?.result?.data?.inputType, undefined); + assert.equal(response.json?.result?.data?.inputOwner, undefined); + }, + }, { name: 'dismiss keyboard', command: 'keyboard', diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index fe0a765ea9..9d7ef91025 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -186,6 +186,18 @@ export async function createIosSettingsWorld(): Promise { request: { command: 'keyboardDismiss', appBundleId: 'com.apple.Preferences' }, result: { dismissed: true }, }, + { + command: 'ios.runner.keyboardStatus', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + platform: 'apple', + request: { command: 'keyboardStatus', appBundleId: 'com.apple.Preferences' }, + result: { + message: 'keyboardStatus', + visible: true, + keyboardReturnKey: 'search', + keyboardReturnKeyLabel: 'Search', + }, + }, ]); let clipboardText = ''; const scriptedRunner = createAppleRunnerProviderFromTranscript(runnerTranscript, 'ios.runner'); diff --git a/test/repo-guards/capability-matrix.test.ts b/test/repo-guards/capability-matrix.test.ts index b3ade7e35f..2b9272db8c 100644 --- a/test/repo-guards/capability-matrix.test.ts +++ b/test/repo-guards/capability-matrix.test.ts @@ -148,11 +148,12 @@ describe('capabilities page matches runtime facts', () => { ), ); assert.throws(() => renderPage({ ...data, operations: lost }), /keyboardDismiss/); - // And a refusal a cell names must fail the same way once the fact grants it. + // And a refusal a cell names must fail the same way once the fact grants it. HarmonyOS is the + // column whose keyboard cell still refuses `keyboardStatus`, so granting it there must fail. const gained = new Map(operations); gained.set( - 'ios-sim', - new Set([...(operations.get('ios-sim') as Set), 'keyboardStatus']), + 'harmonyos-emulator', + new Set([...(operations.get('harmonyos-emulator') as Set), 'keyboardStatus']), ); assert.throws(() => renderPage({ ...data, operations: gained }), /keyboardStatus/); }); diff --git a/website/docs/docs/capabilities.md b/website/docs/docs/capabilities.md index f380545e69..204183a696 100644 --- a/website/docs/docs/capabilities.md +++ b/website/docs/docs/capabilities.md @@ -75,7 +75,7 @@ What a **Yes** does not claim: | [Gesture plans and swipes](/docs/commands#pointer-and-gestures) | Yes | Yes | Yes | Yes | Yes | Yes | | [Hover and viewport size](/docs/commands#web-automation) | No | No | No | No | No | No | | [Read, accept, and dismiss system alerts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | No | No | -| [Keyboard dismiss, enter, and status](/docs/commands#keyboard) | Dismiss and enter; status unsupported | Dismiss and enter; status unsupported | Yes | Yes | Dismiss and enter; status unsupported | Dismiss and enter; status unsupported | +| [Keyboard dismiss, enter, and status](/docs/commands#keyboard) | Yes | Yes | Yes | Yes | Dismiss and enter; status unsupported | Dismiss and enter; status unsupported | | [React Native overlay dismissal](/docs/commands#react-native-overlays) | Yes | Yes | Yes | Yes | Yes | Yes | ### Interact with the app: TV targets @@ -117,7 +117,7 @@ What a **Yes** does not claim: | [Gesture plans and swipes](/docs/commands#pointer-and-gestures) | Yes | Yes | Yes | Yes | Yes | Yes | No | No | | [Hover and viewport size](/docs/commands#web-automation) | No | No | No | No | No | No | No | No | | [Read, accept, and dismiss system alerts](/docs/commands#wait-and-alerts) | No | No | No | No | Yes | Yes | No | No | -| [Keyboard dismiss, enter, and status](/docs/commands#keyboard) | Yes | Dismiss only; status and enter unsupported | Yes | Dismiss only; status and enter unsupported | Dismiss and enter; status unsupported | Yes | No | No | +| [Keyboard dismiss, enter, and status](/docs/commands#keyboard) | Yes | Dismiss only; status and enter unsupported | Yes | Dismiss only; status and enter unsupported | Yes | Yes | No | No | | [React Native overlay dismissal](/docs/commands#react-native-overlays) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | What a **Yes** does not claim: @@ -125,6 +125,7 @@ What a **Yes** does not claim: - **Tap, press, and fill**: Android types non-ASCII text through the test input method: automatically on emulators, and on physical devices only with `open --test-ime`. - **Type into the focused field**: Revyl enters text only at a point, so it refuses `type`; use `fill` on the field instead. - **Gesture plans and swipes**: Multi-touch plans are part of `gesture` where the target runs them; a plan using an operation the target lacks fails at that step. +- **Keyboard dismiss, enter, and status**: `status` reports the focused field's return key on a visible keyboard. It does not claim the key can be named: only Android can report `none`, and the Apple runner reports `unknown` for a key whose wording names no action it knows. - **React Native overlay dismissal**: The command runs wherever tapping runs; it answers `detected: false` on a surface with no React Native overlay. Only iOS and Android apps present dismissable development overlays. ## Apps and devices diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 90dac45f0a..18ab3a5766 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -922,9 +922,11 @@ agent-device clipboard write "" # clear clipboard agent-device keyboard status agent-device keyboard get agent-device keyboard dismiss +agent-device keyboard enter ``` -- `keyboard status` (or `keyboard get`) reports keyboard visibility and a best-effort input type on Android emulators and devices. +- `keyboard status` (or `keyboard get`) reports whether the keyboard is visible, on every platform that serves the command. On a visible keyboard it also reports `returnKey` — what that keyboard's return key offers, as `go`, `search`, `send`, `next`, `done`, `previous`, `join`, `route`, `continue`, `enter`, `none`, or `unknown` — and `returnKeyLabel`, the wording the key itself shows where the platform can read it, trimmed of surrounding whitespace, which can be an app's custom or a localized string. `none` means the field offers no action key, which only Android establishes. `unknown` means the read could not name the visible keyboard's return key, not that one is there: on iOS the key's wording names no action this read knows (its label is still reported), or the keyboard exposes no readable key at all (then there is no label); on Android it means the editor's `imeOptions` named no action. Treat it as "read the keyboard before submitting", not as a safe key to press. A hidden keyboard reports neither field. Android adds its best-effort input type and IME fields; iOS adds nothing else. +- `keyboard enter` presses the return key. - `keyboard dismiss` taps a safe control such as `Done` when available and checks that the keyboard closed. It works on Android and, best-effort, on iOS simulators and devices. Both report `UNSUPPORTED_OPERATION` when the keyboard stays up. On iOS, a keyboard that exposes no dismiss control adds `error.details.reason: keyboard_dismiss_unavailable` with `dispatched: no`, because nothing was tapped; one whose control was tapped and stayed open has no `reason` and `dispatched: unknown`. Android reports `error.details.reason: keyboard_dismiss_unavailable` with `dispatched: unknown` — its escape keyevents were already sent — alongside its own `attempts` and IME fields. Either way, press a visible app control such as `Done` only when that is the intended fallback. - Works with the session device or a device selector (`--platform`, `--device`, `--udid`, `--serial`).