Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,11 @@ extension RunnerTests {
guard pressTvRemote(button, duration: duration) else {
return Response(
ok: false,
error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: "remotePress is only supported on tvOS")
error: ErrorPayload(
code: RunnerWireErrorCode.hardwareControlUnavailable,
message: "remotePress is only supported on tvOS",
hint: "Only a tvOS target has a Siri Remote. On a TV target use tv-remote; on a phone or Mac, press the app's own control by its ref."
)
)
}
return Response(ok: true, data: DataPayload(message: "remote pressed"))
Expand Down Expand Up @@ -567,7 +571,7 @@ extension RunnerTests {
return Response(
ok: false,
error: ErrorPayload(
code: "UNSUPPORTED_OPERATION",
code: RunnerWireErrorCode.hardwareControlUnavailable,
message: "actionButton requires a device model with an Action Button",
hint: "The Action Button is on iPhone 15 Pro and later and iPad Pro (M4) and later. Assign a Shortcut or App Intent to it in Settings > Action Button."
)
Expand All @@ -579,7 +583,7 @@ extension RunnerTests {
return Response(
ok: false,
error: ErrorPayload(
code: "UNSUPPORTED_OPERATION",
code: RunnerWireErrorCode.hardwareControlUnavailable,
message: "\(command.command) requires a physical iPhone or iPad; XCUIDevice exposes no volume press on a simulator",
hint: "hardware-button volume-up/volume-down is supported on physical iOS devices through the runner or Appium; iOS Simulators have no pressable volume key."
)
Expand Down Expand Up @@ -608,7 +612,7 @@ extension RunnerTests {
)
)
case .keyboardReturn:
var press: (wasVisible: Bool, pressed: Bool, visible: Bool)?
var press: (wasVisible: Bool, pressed: Bool, visible: Bool, fieldAttempted: Bool)?
withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .preEventSkipped) {
press = pressKeyboardReturn(app: activeApp)
}
Expand All @@ -618,10 +622,7 @@ extension RunnerTests {
if !result.pressed {
return Response(
ok: false,
error: ErrorPayload(
code: "UNSUPPORTED_OPERATION",
message: "Unable to press the iOS keyboard return key"
)
error: result.fieldAttempted ? keyboardReturnFieldAttemptFailure : keyboardReturnRefusal
)
}
return Response(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,28 @@ extension RunnerTests {
"An element whose center sits behind the on-screen keyboard is refused with tap_keyboard_occludes_target; one whose center stays above the keys presses normally. To end editing, tap the app's own Done/Cancel control, or use keyboard enter to press the return key when submission is wanted."
}

/// The refusal a `keyboard enter` answers with when the control is missing: the keyboard exposes no
/// return-like key, and typing a return found no field to type into. It carries the runner's own code
/// so the host publishes `details.reason` instead of leaving callers to read the prose. The dispatch
/// disclosure stays `unknown` because typing the return can fail after XCTest sent it.
var keyboardReturnRefusal: ErrorPayload {
return ErrorPayload(
code: RunnerWireErrorCode.keyboardReturnUnavailable,
message: "Unable to press the keyboard return key: the keyboard exposes no return-like key and no text field was available to submit",
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
hint: "Tap the app's own submit control by its ref, or focus a text field with fill or type before asking keyboard enter to submit it."
)
}

/// The refusal for a sole text field that was found and tapped but would not take the return. It is
/// not an absent control, so it has no typed reason; the tap may have landed.
var keyboardReturnFieldAttemptFailure: ErrorPayload {
return ErrorPayload(
code: "UNSUPPORTED_OPERATION",
message: "Unable to press the keyboard return key: the only text field was found, but tapping it and typing a return failed",
hint: "Focus the field with fill or type, or tap the app's own submit control by its ref."
)
}


// AX-free on purpose (screenshot bytes, not the accessibility tree) so it holds
// under the same AX degradation the synthesized gesture lane is built to survive.
Expand Down Expand Up @@ -332,26 +354,30 @@ extension RunnerTests {
return runnerPngData(for: image)
}

func pressKeyboardReturn(app: XCUIApplication) -> (wasVisible: Bool, pressed: Bool, visible: Bool) {
/// `fieldAttempted` is true when the sole-text-field route found a field and tapped it before typing,
/// so a failed press there is a failed attempt rather than a missing control.
func pressKeyboardReturn(
app: XCUIApplication
) -> (wasVisible: Bool, pressed: Bool, visible: Bool, fieldAttempted: Bool) {
#if os(tvOS)
return (wasVisible: false, pressed: pressTvRemote(.select), visible: false)
return (wasVisible: false, pressed: pressTvRemote(.select), visible: false, fieldAttempted: false)
#elseif os(iOS)
let wasVisible = isKeyboardVisible(app: app)
if tapKeyboardReturnControl(app: app) {
sleepFor(0.2)
return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app))
return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app), fieldAttempted: false)
}

guard typeKeyboardReturn({ app.typeText(XCUIKeyboardKey.return.rawValue) }) else {
if let singleTarget = singleTextEntryElement(app: app) {
return pressKeyboardReturn(on: singleTarget, app: app, wasVisible: wasVisible)
}
return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app))
return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app), fieldAttempted: false)
}
sleepFor(0.2)
return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app))
return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app), fieldAttempted: false)
#else
return (wasVisible: false, pressed: false, visible: false)
return (wasVisible: false, pressed: false, visible: false, fieldAttempted: false)
#endif
}

Expand All @@ -373,7 +399,7 @@ extension RunnerTests {
on element: XCUIElement,
app: XCUIApplication,
wasVisible: Bool
) -> (wasVisible: Bool, pressed: Bool, visible: Bool) {
) -> (wasVisible: Bool, pressed: Bool, visible: Bool, fieldAttempted: Bool) {
#if os(iOS)
// `element` is bound by index, and the keyboard the tap raises shifts those indices, so the
// return goes to whatever the tap focused rather than to a re-resolved `element`.
Expand All @@ -382,12 +408,12 @@ extension RunnerTests {
app.typeText(XCUIKeyboardKey.return.rawValue)
})
guard typed else {
return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app))
return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app), fieldAttempted: true)
}
sleepFor(0.2)
return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app))
return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app), fieldAttempted: true)
#else
return (wasVisible: wasVisible, pressed: false, visible: false)
return (wasVisible: wasVisible, pressed: false, visible: false, fieldAttempted: false)
#endif
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ extension RunnerTests {
func performInAppBack(app: XCUIApplication) throws -> InAppBackOutcome {
#if os(macOS)
guard let back = macOSNavigationBackElement(app: app) else {
return .refused(ErrorPayload(message: "in-app back control is not available"))
return .refused(Self.macOSInAppBackControlRefusal)
}
let frame = back.frame
if frame.isEmpty {
Expand Down Expand Up @@ -602,6 +602,15 @@ extension RunnerTests {
)
}

/// macOS `back` presses the app's own back control, read from its tree; when the app exposes none,
/// there is no gesture left to try, so it sends nothing. The refusal carries the runner's own code so
/// the host publishes `details.reason` and a caller never reads the refusal off the message prose.
static let macOSInAppBackControlRefusal = ErrorPayload(
code: RunnerWireErrorCode.inAppBackControlUnavailable,
message: "in-app back control is not available",
hint: "Press the app's visible back control by its ref."
)

#if os(iOS)
/// The last settle sample: the baseline the swipe's change is measured from. `stableSinceMs` is
/// when the run of agreeing samples it ends began, and `sampledMs` when it was captured.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ final class RunnerTests: XCTestCase {
/// `BACK_SWIPE_UNVERIFIED` is one whose screen kept changing on its own before the swipe, or could
/// not be sampled afterwards. Both ran the swipe. `BACK_SWIPE_NOT_DISPATCHED` is a back swipe that
/// was refused before any touch went down, so dispatched: no.
/// `IN_APP_BACK_CONTROL_UNAVAILABLE` is macOS `back` finding no in-app back control, so it pressed
/// nothing. `KEYBOARD_RETURN_UNAVAILABLE` is `keyboard enter` finding nothing to submit.
/// `HARDWARE_CONTROL_UNAVAILABLE` is a press the runner answers that its target cannot perform.
/// The host publishes `details.reason` for all three.
enum RunnerWireErrorCode {
static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT"
static let appNotRunning = "APP_NOT_RUNNING"
Expand All @@ -62,6 +66,20 @@ final class RunnerTests: XCTestCase {
/// runner vouches for (#1606). The host reports it under `UNSUPPORTED_OPERATION` and publishes
/// `details.reason`, so a caller never reads the refusal off the message prose.
static let keyboardDismissUnavailable = "KEYBOARD_DISMISS_UNAVAILABLE"
/// A `keyboard enter` that found no return key to press and no text entry to submit through, so
/// nothing was submitted. The host reports it under `UNSUPPORTED_OPERATION` and publishes
/// `details.reason`; the disclosure stays `unknown` because typing the return can fail after XCTest
/// sent it. A sole field that was found but would not take the return is not this refusal.
static let keyboardReturnUnavailable = "KEYBOARD_RETURN_UNAVAILABLE"
/// A press of a hardware or Siri Remote control this target cannot perform: the press helper refuses
/// before pressing anything, so the host reports `UNSUPPORTED_OPERATION` with `details.reason` and
/// `dispatched: no`. Through the daemon the reachable case today is the Action Button on an iPhone
/// or iPad model without one; the simulator volume and non-tvOS remote cases are refused by runtime
/// admission before the runner is asked.
static let hardwareControlUnavailable = "HARDWARE_CONTROL_UNAVAILABLE"
/// macOS `back` found no in-app back control to press, so it sent nothing. The host keeps the
/// `COMMAND_FAILED` code such a code-less refusal always had and publishes `details.reason`.
static let inAppBackControlUnavailable = "IN_APP_BACK_CONTROL_UNAVAILABLE"
/// Refuses a repeat of a `commandId` the runner already finished, when that command's own
/// classification keeps no response body to replay. Nothing ran for the repeat, but the daemon's
/// dispatch disclosure stays `unknown` — only the runner knows whether an earlier send of that id
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,22 @@ extension RunnerTests {
XCTAssertNotEqual(dispatched.code, notDispatched.code)
}

/// `keyboard enter` refusing for having nothing to submit is its own typed refusal, and the host
/// reads this code's literal to publish `details.reason`. The wire code alone is the contract.
func testKeyboardReturnRefusalNamesItsOwnCode() {
let refusal = keyboardReturnRefusal
XCTAssertEqual(refusal.code, "KEYBOARD_RETURN_UNAVAILABLE")
XCTAssertNotNil(refusal.hint)
}

/// A sole field that was found and tapped but would not take the return is a failed attempt, not an
/// absent control, so it must not publish the unavailable reason.
func testKeyboardReturnFieldAttemptFailureIsNotTheTypedRefusal() {
let failure = keyboardReturnFieldAttemptFailure
XCTAssertEqual(failure.code, "UNSUPPORTED_OPERATION")
XCTAssertNotEqual(failure.code, keyboardReturnRefusal.code)
}

func testKeyboardBandFactPublishesAbsenceWhenTheCompleteCaptureHasNoKeyboard() {
let fact = runnerKeyboardBandFact(exists: false, frame: .zero)
XCTAssertEqual(fact, .absent)
Expand Down Expand Up @@ -312,7 +328,10 @@ extension RunnerTests {

XCTAssertEqual(textEntryInputCount(), 0)
XCTAssertEqual(recordedFailures, 0)
XCTAssertEqual(response.error?.code, "UNSUPPORTED_OPERATION")
// The refusal names itself, so the host publishes its `details.reason`; the host still reports the
// command's own `UNSUPPORTED_OPERATION` for the caller.
XCTAssertEqual(response.error?.code, RunnerWireErrorCode.keyboardReturnUnavailable)
XCTAssertNotNil(response.error?.hint)
}

@MainActor
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -580,6 +580,15 @@ extension RunnerTests {

#endif

/// macOS `back` has one route: the app's own back control. Finding none sends nothing, so the refusal
/// names itself — the host publishes its `details.reason`, and a caller never reads the refusal off
/// the message. The literal is pinned as the wire value the host reads.
func testInAppBackControlRefusalNamesItsOwnCode() {
let refusal = Self.macOSInAppBackControlRefusal
XCTAssertEqual(refusal.code, "IN_APP_BACK_CONTROL_UNAVAILABLE")
XCTAssertNotNil(refusal.hint)
}

func testNavigationVisualObservationSeparatesChangeFromRefusal() {
// A refused side makes the pair uncomparable, not "unchanged"; reading it as "unchanged" would
// let a capture that refused become the reason `back` claims its swipe changed nothing.
Expand Down
14 changes: 14 additions & 0 deletions apple/runner/RUNNER_PROTOCOL.md
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,20 @@ next step also carries `hint`, omitted here. A keyboard that stayed up answers o
and the keyboard stayed visible. The host reports both under `UNSUPPORTED_OPERATION` and publishes the
first one's `details.reason`.

Three more refusals name the control the target could not supply, so a caller never reads the refusal
off `message`. `KEYBOARD_RETURN_UNAVAILABLE` is a `keyboard enter` whose keyboard exposed no return-like
key and for which no text field could be found; it proves no submission, not that nothing ran, because
typing the return can fail after XCTest sent it. A sole text field that was found and tapped but would
not take the return answers `UNSUPPORTED_OPERATION` with no code, since the control exists and the tap
may have landed. `HARDWARE_CONTROL_UNAVAILABLE` is a press the runner answers that its target cannot
perform. The runner emits it for the Action Button on an iPhone or iPad model that has none, for a
volume press on a simulator, and for a Siri Remote press off tvOS. Runtime admission refuses the last
two before the runner is asked, so through the daemon only the Action Button case reaches a caller.
`IN_APP_BACK_CONTROL_UNAVAILABLE` is macOS `back` finding no in-app back control in the app's tree. The
last two press nothing. The host maps each to the code its command already refused with
(the first two to `UNSUPPORTED_OPERATION`, the macOS back refusal to the `COMMAND_FAILED` a code-less
reply produced) and publishes its `details.reason`.

`data` is command-specific. Common fields include snapshot nodes, text lookup results, gesture timing, visibility metadata, and screenshot or recording output details.

Capture-plan snapshots carry `snapshotViewport`, the acquisition's declared viewport evidence.
Expand Down
24 changes: 24 additions & 0 deletions contracts/fixtures/dispatch-disclosure.json
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,24 @@
"trigger": "structured runner reply with code KEYBOARD_DISMISS_UNAVAILABLE: `keyboard dismiss` refused because the visible keyboard exposes no dismiss key, before any tap",
"dispatched": "no"
},
{
"id": "ios-runner.reply.KEYBOARD_RETURN_UNAVAILABLE",
"producer": "ios-runner",
"trigger": "structured runner reply with code KEYBOARD_RETURN_UNAVAILABLE: `keyboard enter` found no return key to press and no text field to submit through; typing the return can fail after XCTest sent it",
"dispatched": "unknown"
},
{
"id": "ios-runner.reply.HARDWARE_CONTROL_UNAVAILABLE",
"producer": "ios-runner",
"trigger": "structured runner reply with code HARDWARE_CONTROL_UNAVAILABLE: this target cannot perform the asked-for hardware or Siri Remote press, so nothing was pressed",
"dispatched": "no"
},
{
"id": "ios-runner.reply.IN_APP_BACK_CONTROL_UNAVAILABLE",
"producer": "ios-runner",
"trigger": "structured runner reply with code IN_APP_BACK_CONTROL_UNAVAILABLE: macOS `back` found no in-app back control in the app's tree and pressed nothing",
"dispatched": "no"
},
{
"id": "ios-runner.reply.unlisted-code",
"producer": "ios-runner",
Expand Down Expand Up @@ -493,6 +511,12 @@
"trigger": "fill's first clear-and-retype pass typed and failed verification; the second pass's adb input text then failed, so the fill reports unknown with details.dispatchedSteps counting the steps both passes sent",
"dispatched": "unknown"
},
{
"id": "android-adb.keyboard-dismiss.ime-ignored-escape",
"producer": "android-adb",
"trigger": "keyboard dismiss: the IME kept the keyboard visible through every escape keyevent, which were already sent (details.reason keyboard_dismiss_unavailable)",
"dispatched": "unknown"
},
{
"id": "android-helper.gesture.reported-failure",
"producer": "android-helper",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,11 @@ const DRIVERS: Record<string, () => Promise<unknown>> = {
replyFailure('SYNTHESIZED_GESTURE_OUTCOME_UNKNOWN'),
'ios-runner.reply.KEYBOARD_DISMISS_UNAVAILABLE': () =>
replyFailure('KEYBOARD_DISMISS_UNAVAILABLE'),
'ios-runner.reply.KEYBOARD_RETURN_UNAVAILABLE': () => replyFailure('KEYBOARD_RETURN_UNAVAILABLE'),
'ios-runner.reply.HARDWARE_CONTROL_UNAVAILABLE': () =>
replyFailure('HARDWARE_CONTROL_UNAVAILABLE'),
'ios-runner.reply.IN_APP_BACK_CONTROL_UNAVAILABLE': () =>
replyFailure('IN_APP_BACK_CONTROL_UNAVAILABLE'),
'ios-runner.reply.unlisted-code': () => replyFailure('XCTEST_RECORDED_FAILURE'),
'ios-runner.series.later-chunk-refused': pressSeriesRefusedOnSecondChunk,
'ios-runner.series.invalid-step-before-send': pressSeriesWithInvalidStep,
Expand Down
Loading
Loading