Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@ import Foundation
/// `Emergency Call` places a call, which no caller of this should ever report as a submit. Every
/// other Apple type has a case above.
///
/// `probeLabels` is the order `findKeyboardReturnControl(app:)` presses in. It is the runner's own
/// vocabulary, so it is not derived from the cases: it carries the two spellings of the plain
/// return key, in the order that found a hittable control before this rule existed.
/// `probeLabels` is the order `findKeyboardReturnControl(app:)` presses in. It is the
/// runner's own vocabulary, so it is not derived from the cases: it carries the two spellings of the
/// plain return key, in the order that found a hittable control before this rule existed. The
/// `keyboard status` read does not use it: it reads the keyboard's own key elements, so it needs no
/// English to find the return key.
enum RunnerKeyboardReturnKey: String {
case go
case search
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import Foundation
import CoreGraphics

/// One `keyboard status` answer: whether the software keyboard is on screen and, when it is, what
/// its return key offers.
///
/// `returnKey` is a `RunnerKeyboardReturnKey` raw value, or `nil` when the keyboard is hidden — a
/// hidden keyboard has no return key to report, which is a different fact from one this read cannot
/// name. `returnKeyLabel` is the wording the key itself carries, with surrounding whitespace
/// trimmed, so a localized or an app's custom key stays readable even where the kind is `unknown`.
/// It is present only when the read saw a key to read it from; a keyboard that exposes no readable
/// key answers `unknown` with no label.
struct RunnerKeyboardStatus: Equatable {
let visible: Bool
let returnKey: String?
let returnKeyLabel: String?
}

extension RunnerKeyboardStatus {
/// One control the status read observed on the visible keyboard: the wording it carries in the
/// accessibility tree, and the frame it occupies. `frame` decides which control is the
/// keyboard's bottom-trailing key, which is where Apple's keyboards place the return key.
struct ObservedKeyboardKey {
let text: String?
let frame: CGRect
}

/// Builds the answer from what the runner observed. The keyboard decides whether a return key is
/// reported at all: a hidden one has none, which is a different fact from one this read cannot
/// name. Only Android's `IME_ACTION_NONE` can establish that a field offers no action key, so
/// `none` is never this runner's answer.
///
/// The key's own wording decides the kind, through the one rule `keyboard enter` probes with
/// (`RunnerKeyboardReturnKey.kind(of:)`), so the two commands can never disagree about the same
/// key. The label itself is kept as the accessibility tree carries it apart from its surrounding
/// whitespace, so a localized or an app's custom key stays readable even where the kind is
/// `unknown`.
static func observed(
visible: Bool,
returnControlText: String?
) -> RunnerKeyboardStatus {
guard visible else {
return RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil)
}
guard let returnControlText else {
return RunnerKeyboardStatus(
visible: true,
returnKey: RunnerKeyboardReturnKey.unknown.rawValue,
returnKeyLabel: nil
)
}
let label = returnControlText.trimmingCharacters(in: .whitespacesAndNewlines)
return RunnerKeyboardStatus(
visible: true,
returnKey: RunnerKeyboardReturnKey.kind(of: label).rawValue,
returnKeyLabel: label.isEmpty ? nil : label
)
}

/// Builds the answer from every control the read saw on the visible keyboard, so it does not need to
/// read English to find the return key. A control whose wording `kind(of:)` names a kind for is the
/// return key on any keyboard in any language, and its wording is the label. When none maps — a
/// fully localized keyboard — the return key is the measurable control in the keyboard's
/// bottom-trailing corner: the slot Apple's keyboards give it, beside and right of the space bar,
/// which no letter, shift or delete key reaches. A keyboard's padding elements report an empty
/// frame and never win on position. An unmatched key's wording is still reported, with kind
/// `unknown`: that is what the label is for. Only a keyboard exposing no worded, measurable control
/// answers `unknown` with no label.
static func observed(
visible: Bool,
keyboardKeys: [ObservedKeyboardKey]
) -> RunnerKeyboardStatus {
guard visible else {
return RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil)
}
let worded = keyboardKeys.filter { key in
!(key.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
if let named = worded.first(where: { RunnerKeyboardReturnKey.kind(of: $0.text ?? "") != .unknown }) {
return observed(visible: true, returnControlText: named.text)
}
let returnKey = worded
.filter { !$0.frame.isEmpty }
.max { lhs, rhs in (lhs.frame.maxY, lhs.frame.maxX) < (rhs.frame.maxY, rhs.frame.maxX) }
return observed(visible: true, returnControlText: returnKey?.text)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ extension CommandType {
return .none
case .tap, .longPress, .drag, .type, .swipe, .scroll, .findText, .querySelector, .readText,
.snapshot, .backInApp, .gesture, .gestureViewport, .appSwitcher,
.keyboardDismiss, .sequence:
.keyboardDismiss, .keyboardStatus, .sequence:
return .appWindows
case .screenshot, .recordStart:
return .observedScreen
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,23 @@ extension RunnerTests {
wasVisible: result.wasVisible
)
)
case .keyboardStatus:
var status: RunnerKeyboardStatus?
withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) {
status = keyboardStatus(app: activeApp)
}
guard let result = status else {
return Response(ok: false, error: ErrorPayload(message: "keyboardStatus produced no result"))
}
return Response(
ok: true,
data: DataPayload(
message: "keyboardStatus",
visible: result.visible,
keyboardReturnKey: result.returnKey,
keyboardReturnKeyLabel: result.returnKeyLabel
)
)
case .alert:
let action = (command.action ?? "get").lowercased()
let deadline = alertDeadline ?? Date().addingTimeInterval(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,43 @@ extension RunnerTests {
return visibleKeyboardFrame(app: app) != nil
}

/// What `keyboard status` answers with: the keyboard's visibility and, when it is up, the return
/// key the focused field offers. It reads the keyboard's own key elements rather than looking a
/// control up by English name, so a localized keyboard (a return key reading `szukaj`) is reported
/// with its own wording instead of a label-less `unknown`. `RunnerKeyboardStatus` decides which
/// observed control is the return key, through the same `kind(of:)` rule `keyboard enter` probes
/// with, so the two commands can never disagree about the wording they can name.
func keyboardStatus(app: XCUIApplication) -> RunnerKeyboardStatus {
guard isKeyboardVisible(app: app) else {
return RunnerKeyboardStatus.observed(visible: false, returnControlText: nil)
}
return RunnerKeyboardStatus.observed(visible: true, keyboardKeys: observedKeyboardKeys(app: app))
}

/// Every control the visible keyboard exposes with its wording and frame. The keyboard is another
/// process, so every element read is a round trip: one pass over its keys and one over its buttons
/// is what a read can afford, and it is the same two queries a snapshot's keyboard band makes. An
/// element that answers no wording is dropped — it is neither a kind nor a label. Off iOS nothing
/// is queried and nothing is reported, so a caller must not read the empty answer as a keyboard
/// with no return key; `keyboard status` is not advertised there.
func observedKeyboardKeys(app: XCUIApplication) -> [RunnerKeyboardStatus.ObservedKeyboardKey] {
#if os(iOS)
let keyboard = app.keyboards.firstMatch
let elements: [XCUIElement] =
safely("KEYBOARD_STATUS_KEYS", [XCUIElement]()) { keyboard.keys.allElementsBoundByIndex }
+ safely("KEYBOARD_STATUS_BUTTONS", [XCUIElement]()) { keyboard.buttons.allElementsBoundByIndex }
return elements.compactMap { element in
guard let label = safely("KEYBOARD_STATUS_KEY_LABEL", { element.label }) else { return nil }
let text = label.trimmingCharacters(in: .whitespacesAndNewlines)
guard !text.isEmpty else { return nil }
let frame = safely("KEYBOARD_STATUS_KEY_FRAME", CGRect.zero) { element.frame }
return RunnerKeyboardStatus.ObservedKeyboardKey(text: label, frame: frame)
}
#else
return []
#endif
}
Comment thread
thymikee marked this conversation as resolved.
Resolved

func dismissKeyboard(app: XCUIApplication) -> RunnerKeyboardDismissOutcome {
let keyboard = app.keyboards.firstMatch
let wasVisible = isKeyboardVisible(app: app)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ enum CommandType: String, Codable, CaseIterable {
case volumeDown
case keyboardDismiss
case keyboardReturn
case keyboardStatus
case alert
case sequence
case gesture
Expand Down Expand Up @@ -300,7 +301,7 @@ extension Command {
.keyboardDismiss, .keyboardReturn, .sequence, .gesture:
return .interaction

case .findText, .readText, .gestureViewport:
case .findText, .readText, .gestureViewport, .keyboardStatus:
return .appRead(retainsJournalResponseJson: true)

case .snapshot:
Expand Down Expand Up @@ -536,6 +537,12 @@ struct DataPayload: Codable {
var wasVisible: Bool?
var dismissed: Bool?
var keyboardDismissMechanism: String?
/// The kind of return key the visible keyboard offers, on a `keyboard status` answer. Absent when
/// the keyboard is hidden, which is not the same claim as a key this read could not name.
var keyboardReturnKey: String?
/// The wording that return key carries, verbatim, so a localized or an app's custom key stays
/// readable. Absent when the key carries none, or there is no key to read.
var keyboardReturnKeyLabel: String?
var orientation: String?
var gestureFallback: String?
var gestureFallbackMessage: String?
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,5 +82,144 @@ extension RunnerTests {
XCTAssertEqual(RunnerKeyboardReturnKey.kind(of: label), expected[label], label)
}
}

/// What a `keyboard status` read builds from each observation. The visible keyboard's return
/// control decides both fields: its wording names the kind through the same rule `keyboard enter`
/// probes with, and it is kept as the accessibility tree carries it apart from surrounding
/// whitespace, so a key this read cannot name is still readable.
func testKeyboardStatusReportsTheObservedReturnControl() {
let status = RunnerKeyboardStatus.observed(visible: true, returnControlText: " Search ")
XCTAssertEqual(status, RunnerKeyboardStatus(visible: true, returnKey: "search", returnKeyLabel: "Search"))
}

/// The kinds `kind(of:)` names but the press list never probes are reported by the read, because
/// the read looks at the keyboard's own keys rather than looking a control up by English name.
func testKeyboardStatusReportsAKindThePressProbeListOmits() {
for (label, kind) in [("Route", "route"), ("Continue", "continue")] {
XCTAssertEqual(
RunnerKeyboardStatus.observed(visible: true, returnControlText: label),
RunnerKeyboardStatus(visible: true, returnKey: kind, returnKeyLabel: label),
label
)
XCTAssertFalse(
RunnerKeyboardReturnKey.probeLabels.contains(label),
"\(label) must stay out of the press probe list, which the read does not use"
)
}
}

/// A hidden keyboard reports no return key at all. That is a different fact from one it could not
/// name, so both fields stay out rather than answering `unknown` for a key nobody can press. The
/// keyboard's keys are ignored: nothing is being submitted to.
func testKeyboardStatusReportsNoReturnKeyWhileTheKeyboardIsHidden() {
XCTAssertEqual(
RunnerKeyboardStatus.observed(visible: false, returnControlText: "Search"),
RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil)
)
XCTAssertEqual(
RunnerKeyboardStatus
.observed(visible: false, keyboardKeys: [RunnerKeyboardStatus.ObservedKeyboardKey(text: "Done", frame: CGRect(x: 0, y: 0, width: 40, height: 40))]),
RunnerKeyboardStatus(visible: false, returnKey: nil, returnKeyLabel: nil)
)
}

/// The English keyboard, as the accessibility tree carries it: the search field's return key reads
/// `Search` in the bottom-trailing slot. The picker names the kind from its wording.
func testKeyboardStatusPicksTheReturnKeyFromTheKeyboardKeysByWording() {
let keys = observedKeyboardKeysFixture([
("Q", 5, 959, 39), ("W", 44, 959, 40), ("shift", 5, 1067, 51), ("Delete", 348, 1067, 52),
("Numbers", 5, 1121, 49), ("Emoji", 54, 1121, 49), ("Space", 103, 1121, 197),
("Search", 301, 1121, 99), ("Padding-Right", 0, 952, 0),
])
XCTAssertEqual(
RunnerKeyboardStatus.observed(visible: true, keyboardKeys: keys),
RunnerKeyboardStatus(visible: true, returnKey: "search", returnKeyLabel: "Search")
)
}

/// The whole point of reading the keyboard's own keys: a fully localized keyboard names no kind, so
/// the return key is the bottom-trailing control and its wording is reported even though the kind
/// is `unknown`. These are the frames this read measured on a Polish iOS keyboard, where `szukaj`
/// sits beside a wide space bar in the last row and the letter, shift and delete keys are higher or
/// further left. The headless test IME, which exposes keys with no wording at all, is the same
/// shape with nothing to report.
func testKeyboardStatusReportsALocalizedReturnKeyWithItsOwnWording() {
let keys = observedKeyboardKeysFixture([
("Padding-Left", 0, 952, 0), ("Q", 5, 959, 39), ("P", 360, 959, 40), ("shift", 5, 1067, 51),
("M", 301, 1067, 40), ("usuń", 348, 1067, 52), ("cyfry", 5, 1121, 49), ("Emoji", 54, 1121, 49),
(" ", 103, 1121, 197), ("szukaj", 301, 1121, 99),
])
XCTAssertEqual(
RunnerKeyboardStatus.observed(visible: true, keyboardKeys: keys),
RunnerKeyboardStatus(visible: true, returnKey: "unknown", returnKeyLabel: "szukaj")
)
}

/// The picker takes a control whose wording names a kind even when another control sits further
/// down-right, so a keyboard that localizes some keys and not others is still named by wording.
/// Padding elements, which carry no usable frame, never win on position.
func testKeyboardStatusPrefersNamedWordingOverKeyboardGeometry() {
let keys = observedKeyboardKeysFixture([
("Done", 5, 959, 39), ("ż", 301, 1121, 99), ("Padding-Right", 0, 952, 0),
])
XCTAssertEqual(
RunnerKeyboardStatus.observed(visible: true, keyboardKeys: keys),
RunnerKeyboardStatus(visible: true, returnKey: "done", returnKeyLabel: "Done")
)
}

/// Only a visible keyboard that exposes no control worth reporting answers `unknown` with no label:
/// an empty layout, a keyboard whose keys all carry no wording (a headless test IME), and one whose
/// only wording sits on a zero-frame padding element.
func testKeyboardStatusReportsUnknownForAVisibleKeyboardWithNoReadableKey() {
let expected = RunnerKeyboardStatus(visible: true, returnKey: "unknown", returnKeyLabel: nil)
XCTAssertEqual(RunnerKeyboardStatus.observed(visible: true, keyboardKeys: []), expected)
XCTAssertEqual(
RunnerKeyboardStatus.observed(
visible: true,
keyboardKeys: observedKeyboardKeysFixture([(" ", 301, 1121, 99)])
),
expected
)
XCTAssertEqual(
RunnerKeyboardStatus.observed(
visible: true,
keyboardKeys: observedKeyboardKeysFixture([("szukaj", 0, 952, 0)])
),
expected
)
}

/// The press path is unchanged by what the read can see: it still looks its own labels up in its
/// own order, and the read never consults that list.
func testKeyboardEnterProbeListStillOwnsItsOrder() {
XCTAssertEqual(
RunnerKeyboardReturnKey.probeLabels,
["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"]
)
}

/// A key whose wording the rule does not recognize keeps its wording, which is the whole point of
/// reporting the label: a localized or a custom key stays readable where the kind is `unknown`.
func testKeyboardStatusKeepsAnUnrecognizedLabelVerbatim() {
for label in ["Send Message", "搜索", "完了"] {
let status = RunnerKeyboardStatus.observed(visible: true, returnControlText: label)
XCTAssertEqual(status.returnKey, "unknown", label)
XCTAssertEqual(status.returnKeyLabel, label)
}
}

/// Builds a keyboard's observed controls from the frames this read measured; a zero width is an
/// element with no usable frame, such as the padding entries a keyboard exposes.
private func observedKeyboardKeysFixture(
_ entries: [(text: String, x: CGFloat, y: CGFloat, width: CGFloat)]
) -> [RunnerKeyboardStatus.ObservedKeyboardKey] {
entries.map { entry in
RunnerKeyboardStatus.ObservedKeyboardKey(
text: entry.text,
frame: CGRect(x: entry.x, y: entry.y, width: entry.width, height: entry.width > 0 ? 54 : 0)
)
}
}
}
#endif
Loading
Loading