diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index bc88aee555..1bddbd560c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -587,7 +587,7 @@ extension RunnerTests { } return Response(ok: true, data: DataPayload(message: command.command.rawValue)) case .keyboardDismiss: - var dismissal: (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?)? + var dismissal: RunnerKeyboardDismissOutcome? withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { dismissal = dismissKeyboard(app: activeApp) } @@ -595,15 +595,7 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(message: "keyboardDismiss produced no result")) } if result.wasVisible && !result.dismissed { - return Response( - ok: false, - error: ErrorPayload( - code: "UNSUPPORTED_OPERATION", - message: "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)", - hint: - "An element whose center sits behind the on-screen keyboard is refused with tap_keyboard_occludes_target; one whose center stays above the keys presses normally. To end editing, tap the app's own Done/Cancel control, or use keyboard enter to press the return key when submission is wanted." - ) - ) + return Response(ok: false, error: keyboardDismissRefusal(dispatchedDismissal: result.dispatchedDismissal)) } return Response( ok: true, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift index 797113a8b9..78d3d3e6c3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift @@ -31,6 +31,18 @@ enum RunnerKeyboardDismissMechanism: String { case dismissKey } +/// One `keyboard dismiss` attempt, plus the fact that decides how a refusal is classified: +/// `dispatchedDismissal` says whether the runner ever dispatched a dismissal. +struct RunnerKeyboardDismissOutcome { + let wasVisible: Bool + let dismissed: Bool + let visible: Bool + let mechanism: RunnerKeyboardDismissMechanism? + /// Whether a dismissal was dispatched — the keyboard's own control was tapped, or the tvOS remote's + /// Menu press landed. Only its absence proves the command sent nothing. + let dispatchedDismissal: Bool +} + /// Keyboard coverage read from the capture's complete pre-projection root snapshot on iOS XCTest. /// Visible frames include the input assistant and share the nodes' normalized coordinate space. /// A complete scan with no keyboard establishes absence; projected or partial trees never do. @@ -189,20 +201,32 @@ extension RunnerTests { return visibleKeyboardFrame(app: app) != nil } - func dismissKeyboard( - app: XCUIApplication - ) -> (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?) { + func dismissKeyboard(app: XCUIApplication) -> RunnerKeyboardDismissOutcome { let keyboard = app.keyboards.firstMatch let wasVisible = isKeyboardVisible(app: app) guard wasVisible else { - return (wasVisible: false, dismissed: false, visible: false, mechanism: nil) + return RunnerKeyboardDismissOutcome( + wasVisible: false, + dismissed: false, + visible: false, + mechanism: nil, + dispatchedDismissal: false + ) } #if os(tvOS) _ = pressTvRemote(.menu) sleepFor(0.2) - let visible = isKeyboardVisible(app: app) - return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey) + let tvVisible = isKeyboardVisible(app: app) + return RunnerKeyboardDismissOutcome( + wasVisible: true, + dismissed: !tvVisible, + visible: tvVisible, + mechanism: tvVisible ? nil : .dismissKey, + // On tvOS the Menu press always goes out, so a refusal after it did dispatch and keeps the + // generic code. + dispatchedDismissal: true + ) #else if tapKeyboardDismissControl(app: app) { _ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout) @@ -213,7 +237,13 @@ extension RunnerTests { requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches ) let visible = isKeyboardVisible(app: app) - return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey) + return RunnerKeyboardDismissOutcome( + wasVisible: true, + dismissed: !visible, + visible: visible, + mechanism: visible ? nil : .dismissKey, + dispatchedDismissal: true + ) } // #1606 review P1 (twice): generic background-tap dismissal is @@ -222,10 +252,41 @@ extension RunnerTests { // indistinguishable from an inert backdrop, so a "safe-area" tap can // navigate or submit while reporting a successful dismiss. The dismiss // key is the only mechanism the runner can vouch for. - return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app), mechanism: nil) + return RunnerKeyboardDismissOutcome( + wasVisible: true, + dismissed: false, + visible: isKeyboardVisible(app: app), + mechanism: nil, + dispatchedDismissal: false + ) #endif } + /// The refusal a visible keyboard that stayed up answers with. The two ways it stays up answer + /// different questions and must not share one classification: finding no dismiss control sent + /// nothing and is the runner's own typed refusal (`KEYBOARD_DISMISS_UNAVAILABLE`, so the host + /// publishes its `details.reason` instead of leaving callers to read the prose), while a control + /// that was tapped and left the keys up did dispatch and keeps the generic code. + func keyboardDismissRefusal(dispatchedDismissal: Bool) -> ErrorPayload { + if dispatchedDismissal { + return ErrorPayload( + code: "UNSUPPORTED_OPERATION", + message: "Unable to dismiss the keyboard: a dismiss mechanism ran and the keyboard stayed visible", + hint: keyboardDismissRefusalHint + ) + } + return ErrorPayload( + code: RunnerWireErrorCode.keyboardDismissUnavailable, + message: "Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)", + hint: keyboardDismissRefusalHint + ) + } + + private var keyboardDismissRefusalHint: String { + return + "An element whose center sits behind the on-screen keyboard is refused with tap_keyboard_occludes_target; one whose center stays above the keys presses normally. To end editing, tap the app's own Done/Cancel control, or use keyboard enter to press the return key when submission is wanted." + } + // AX-free on purpose (screenshot bytes, not the accessibility tree) so it holds // under the same AX degradation the synthesized gesture lane is built to survive. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 09bfe4e010..931dc150db 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -58,6 +58,10 @@ final class RunnerTests: XCTestCase { static let backSwipeNoChange = "BACK_SWIPE_NO_CHANGE" static let backSwipeUnverified = "BACK_SWIPE_UNVERIFIED" static let backSwipeNotDispatched = "BACK_SWIPE_NOT_DISPATCHED" + /// A `keyboard dismiss` the visible keyboard would not come down for: it exposes no dismiss key the + /// runner vouches for (#1606). The host reports it under `UNSUPPORTED_OPERATION` and publishes + /// `details.reason`, so a caller never reads the refusal off the message prose. + static let keyboardDismissUnavailable = "KEYBOARD_DISMISS_UNAVAILABLE" /// Refuses a repeat of a `commandId` the runner already finished, when that command's own /// classification keeps no response body to replay. Nothing ran for the repeat, but the daemon's /// dispatch disclosure stays `unknown` — only the runner knows whether an earlier send of that id diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift index ecd62afcc2..207b05af92 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift @@ -139,6 +139,21 @@ extension RunnerTests { #endif + /// The two ways a visible keyboard stays up answer different questions, so they must not share one + /// code (#1606). Finding no dismiss control sent nothing and is the runner's own refusal; a control + /// that was tapped and left the keys up did dispatch and keeps the generic code. The host reads the + /// first code's literal to publish `details.reason`, so it is pinned here as the wire value. + func testKeyboardDismissRefusalNamesItsOwnCodeOnlyWhenNothingWasDispatched() { + let notDispatched = keyboardDismissRefusal(dispatchedDismissal: false) + XCTAssertEqual(notDispatched.code, "KEYBOARD_DISMISS_UNAVAILABLE") + XCTAssertNotNil(notDispatched.hint) + + let dispatched = keyboardDismissRefusal(dispatchedDismissal: true) + XCTAssertEqual(dispatched.code, "UNSUPPORTED_OPERATION") + XCTAssertNotNil(dispatched.hint) + XCTAssertNotEqual(dispatched.code, notDispatched.code) + } + func testKeyboardBandFactPublishesAbsenceWhenTheCompleteCaptureHasNoKeyboard() { let fact = runnerKeyboardBandFact(exists: false, frame: .zero) XCTAssertEqual(fact, .absent) diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index d88201deee..2c91cc62ac 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -252,12 +252,19 @@ Successful and failed responses use the same top-level envelope: { "ok": false, "error": { - "code": "UNSUPPORTED_OPERATION", - "message": "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key (background taps are never attempted)" + "code": "KEYBOARD_DISMISS_UNAVAILABLE", + "message": "Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)" } } ``` +The example is a `keyboard dismiss` the keyboard would not come down for; a refusal that can name a +next step also carries `hint`, omitted here. A keyboard that stayed up answers one of two codes: +`KEYBOARD_DISMISS_UNAVAILABLE` when it found no dismiss control and so sent nothing, or +`UNSUPPORTED_OPERATION` when a dismiss control was tapped, or the tvOS remote's Menu press was sent, +and the keyboard stayed visible. The host reports both under `UNSUPPORTED_OPERATION` and publishes the +first one's `details.reason`. + `data` is command-specific. Common fields include snapshot nodes, text lookup results, gesture timing, visibility metadata, and screenshot or recording output details. Capture-plan snapshots carry `snapshotViewport`, the acquisition's declared viewport evidence. @@ -310,7 +317,9 @@ declaration for the current rules. the main-thread watchdog in [`RunnerTests+Transport.swift`](AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift). The daemon classifies a runner-reported code in `classifyRunnerReportedError` ([`runner-contract.ts`](../../packages/apple-runner/src/runner-contract.ts)): codes listed in `DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES` - arrive as `COMMAND_FAILED` with `details.runnerErrorCode`, and other codes pass through as typed - codes. How the daemon reacts to a failure (retry, resend, session-fatal) is declared in + arrive as `COMMAND_FAILED` with `details.runnerErrorCode`, unless their row names the wire code the + caller already reads that family under. Other codes pass through as typed codes. A code with a + `RUNNER_REPORTED_ERROR_REASONS` row also publishes `details.reason`, so a caller never reads the + refusal off `message`. How the daemon reacts to a failure (retry, resend, session-fatal) is declared in `RUNNER_ERROR_RULES` and the helpers beside it ([`runner-error-classification.ts`](../../packages/apple-runner/src/runner-error-classification.ts)). diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 1a12a36228..261f216a7d 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -301,6 +301,12 @@ "trigger": "structured runner reply with code COMMAND_ALREADY_FINISHED: the journal refused a repeat of a spent commandId and kept no outcome to replay, so it cannot say what that command did", "dispatched": "unknown" }, + { + "id": "ios-runner.reply.KEYBOARD_DISMISS_UNAVAILABLE", + "producer": "ios-runner", + "trigger": "structured runner reply with code KEYBOARD_DISMISS_UNAVAILABLE: `keyboard dismiss` refused because the visible keyboard exposes no dismiss key, before any tap", + "dispatched": "no" + }, { "id": "ios-runner.reply.unlisted-code", "producer": "ios-runner", diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index 1972d92a9c..81a94e248a 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -192,6 +192,8 @@ const DRIVERS: Record Promise> = { replyFailure('SYNTHESIZED_GESTURE_NOT_DISPATCHED'), 'ios-runner.reply.SYNTHESIZED_GESTURE_OUTCOME_UNKNOWN': () => replyFailure('SYNTHESIZED_GESTURE_OUTCOME_UNKNOWN'), + 'ios-runner.reply.KEYBOARD_DISMISS_UNAVAILABLE': () => + replyFailure('KEYBOARD_DISMISS_UNAVAILABLE'), 'ios-runner.reply.unlisted-code': () => replyFailure('XCTEST_RECORDED_FAILURE'), 'ios-runner.series.later-chunk-refused': pressSeriesRefusedOnSecondChunk, 'ios-runner.series.invalid-step-before-send': pressSeriesWithInvalidStep, diff --git a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts index 41a6ea7e98..44640bf88b 100644 --- a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts +++ b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts @@ -3,6 +3,7 @@ import { test } from 'vitest'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { COMMAND_ALREADY_FINISHED_RUNNER_CODE, + KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE, MAIN_THREAD_TIMEOUT_RUNNER_CODE, NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES, @@ -22,7 +23,10 @@ import { shouldRetryRunnerConnectError, } from '../runner-error-classification.ts'; import { runnerConnectFailure, unwrittenConnectRefusal } from './runner-session-fixtures.ts'; -import { requireSwiftThrownErrorWireCode } from './runner-swift-settlement-fixtures.ts'; +import { + requireSwiftKeyboardDismissRefusalWireCode, + requireSwiftThrownErrorWireCode, +} from './runner-swift-settlement-fixtures.ts'; const TAP = { command: 'tap' } as const; const SNAPSHOT = { command: 'snapshot' } as const; @@ -169,6 +173,42 @@ test('a back swipe that proved nothing stays COMMAND_FAILED, unknown, with its o assert.equal(notDispatched.details.dispatched, 'no'); }); +// The keyboard stays up two different ways, and only one of them sent nothing. The literal is what the +// Swift runner encodes, so it is the contract and not the constant name: a rename on one side has to +// fail here rather than silently split the pair. +test('a keyboard dismiss refused for having no dismiss key keeps its code and publishes its reason', () => { + assert.equal(KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE, 'KEYBOARD_DISMISS_UNAVAILABLE'); + const refusal = classifyRunnerReportedError(KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE); + // The code contract for `keyboard dismiss` is unchanged: the refusal travels under the code that + // command already refused with, so a caller keyed on the code sees what it saw before. + assert.equal(refusal.code, 'UNSUPPORTED_OPERATION'); + assert.equal(refusal.details.runnerErrorCode, 'KEYBOARD_DISMISS_UNAVAILABLE'); + assert.equal(refusal.details.reason, 'keyboard_dismiss_unavailable'); + assert.equal(refusal.details.dispatched, 'no'); + assert.equal(refusal.details.retriable, undefined); + // The whole published shape: the runner's own code survives, the wire code it hides under does not. + assert.deepEqual(refusal.details, { + runnerErrorCode: 'KEYBOARD_DISMISS_UNAVAILABLE', + reason: 'keyboard_dismiss_unavailable', + dispatched: 'no', + }); + const refusalError = new AppError(refusal.code, refusal.details.reason ?? '', refusal.details); + assert.equal(isRetryableRunnerError(refusalError), false); + assert.equal(isRunnerBusyError(refusalError), false); + // The closest negative: the same refusal in prose alone, from an unclassified reply, earns no + // reason, so a consumer keyed on `reason` cannot mistake it for the pre-dispatch refusal. + const untyped = commandFailed( + 'Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted', + ); + assert.equal(untyped.details?.reason, undefined); + // A dismiss control that was tapped and left the keys up did dispatch, so it keeps the generic code + // and says nothing about dispatch. + const dispatched = classifyRunnerReportedError('UNSUPPORTED_OPERATION'); + assert.equal(dispatched.code, 'UNSUPPORTED_OPERATION'); + assert.equal(dispatched.details.reason, undefined); + assert.equal(dispatched.details.dispatched, 'unknown'); +}); + test('retryable requires an AppError with COMMAND_FAILED', () => { assert.equal(isRetryableRunnerError(new Error('fetch failed')), false); assert.equal(isRetryableRunnerError(new AppError('DEVICE_NOT_FOUND', 'fetch failed')), false); @@ -560,6 +600,15 @@ test('the runner declares the wire codes the daemon reads for occupancy', () => ); }); +test('the runner declares the wire code the keyboard-dismiss refusal publishes its reason under', () => { + assert.equal( + requireSwiftKeyboardDismissRefusalWireCode(), + KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE, + 'the runner must encode its no-dismiss-key refusal under the code the daemon maps to ' + + 'details.reason, or that reason becomes unreachable', + ); +}); + // --- runner that predates the listener token --- test('a runner that does not require the token is session-fatal and never retried or replayed', () => { diff --git a/packages/apple-runner/src/__tests__/runner-session-response.test.ts b/packages/apple-runner/src/__tests__/runner-session-response.test.ts index 5c08ab0959..516fa2a96d 100644 --- a/packages/apple-runner/src/__tests__/runner-session-response.test.ts +++ b/packages/apple-runner/src/__tests__/runner-session-response.test.ts @@ -10,13 +10,14 @@ import { isRetryableRunnerError } from '../runner-error-classification.ts'; import type { RunnerLogAttempt } from '../runner-failure-diagnostics.ts'; import { parseRunnerResponse } from '../runner-exchange.ts'; -test('parseRunnerResponse preserves runner unsupported-operation codes', async () => { +test('parseRunnerResponse keeps the keyboard-dismiss refusal under the code the command reports', async () => { const response = new Response( JSON.stringify({ ok: false, error: { - code: 'UNSUPPORTED_OPERATION', - message: 'Unable to dismiss the iOS keyboard without a safe native dismiss control', + code: 'KEYBOARD_DISMISS_UNAVAILABLE', + message: + 'Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)', }, }), ); @@ -26,8 +27,12 @@ test('parseRunnerResponse preserves runner unsupported-operation codes', async ( () => parseRunnerResponse(response, session, runnerLogAttempt('/tmp/runner.log')), (error: unknown) => { assert.ok(error instanceof AppError); + // The runner's own code stays off the wire: `keyboard dismiss` keeps refusing with the code it + // always refused with, and the reason is what tells the two ways the keyboard stays up apart. assert.equal(error.code, 'UNSUPPORTED_OPERATION'); - assert.match(error.message, /Unable to dismiss the iOS keyboard/i); + assert.equal(error.details?.reason, 'keyboard_dismiss_unavailable'); + assert.equal(error.details?.runnerErrorCode, 'KEYBOARD_DISMISS_UNAVAILABLE'); + assert.equal(error.details?.dispatched, 'no'); return true; }, ); @@ -40,8 +45,9 @@ test('parseRunnerResponse surfaces the keyboard-dismiss hint naming the occlusio JSON.stringify({ ok: false, error: { - code: 'UNSUPPORTED_OPERATION', - message: 'Unable to dismiss the iOS keyboard without a safe native dismiss control', + code: 'KEYBOARD_DISMISS_UNAVAILABLE', + message: + 'Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)', hint, }, }), @@ -54,9 +60,6 @@ test('parseRunnerResponse surfaces the keyboard-dismiss hint naming the occlusio assert.ok(error instanceof AppError); assert.equal(error.code, 'UNSUPPORTED_OPERATION'); assert.equal(error.details?.hint, hint); - assert.match(String(error.details?.hint), /tap_keyboard_occludes_target/); - assert.match(String(error.details?.hint), /center stays above the keys/i); - assert.match(String(error.details?.hint), /keyboard enter/i); return true; }, ); diff --git a/packages/apple-runner/src/__tests__/runner-swift-settlement-fixtures.ts b/packages/apple-runner/src/__tests__/runner-swift-settlement-fixtures.ts index 758111c07e..46d110dfb9 100644 --- a/packages/apple-runner/src/__tests__/runner-swift-settlement-fixtures.ts +++ b/packages/apple-runner/src/__tests__/runner-swift-settlement-fixtures.ts @@ -175,3 +175,20 @@ export function requireSwiftThrownErrorWireCode(): { finishedCode, }; } + +/** + * The Swift declaration for the code a `keyboard dismiss` refuses with when the keyboard exposes no + * dismiss key (#1606). The refusal is thrown on a device lane, so the tie that keeps the host's + * `details.reason` row attached to it reads the declaration rather than a second copy of the literal. + */ +export function requireSwiftKeyboardDismissRefusalWireCode(): string { + const codes = readSwiftWireErrorCodes(); + const refusalCode = codes['keyboardDismissUnavailable']; + assert.ok( + refusalCode !== undefined, + 'RunnerWireErrorCode must declare keyboardDismissUnavailable: the daemon publishes ' + + "details.reason 'keyboard_dismiss_unavailable' for this code, and a runner that stopped " + + 'encoding it would leave that reason unreachable.', + ); + return refusalCode; +} diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 7ad4c2b374..7c17a89784 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -3,6 +3,7 @@ import { createRequestCanceledError, discloseDispatch, toAppErrorCode, + type AppErrorCode, type DispatchDisclosure, } from '@agent-device/kernel/errors'; import crypto from 'node:crypto'; @@ -290,6 +291,15 @@ export function resolveRunnerStartupSignal( */ export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE = 'SCROLL_KEYBOARD_OCCLUDES_SURFACE'; +/** + * The runner's own code refusing a `keyboard dismiss` whose visible keyboard exposes no dismiss key + * (#1606). Background taps are never attempted, so no geometry can prove a tap outside the keys is + * side-effect-free; the refusal is the runner's decision and it answers before dispatching anything. + * It is the runner's vocabulary, so it is declared here beside the set that keeps it off the wire, + * and consumers read `details.reason`, never the message prose. + */ +export const KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE = 'KEYBOARD_DISMISS_UNAVAILABLE'; + /** * The codes the XCTest runner answers with when a resolved-display capture it was asked to make did * not happen (#2728): no window resolved so no display could be named, the resolved window named no @@ -355,17 +365,22 @@ const BACK_SWIPE_NOT_DISPATCHED_RUNNER_CODE = 'BACK_SWIPE_NOT_DISPATCHED'; /** * Runner codes that classify a failure for the host without renaming it on the wire. They stay - * `COMMAND_FAILED` and survive as `details.runnerErrorCode`, which is what family policy reads: - * `RUNNER_BUSY` for retriable contention, `ALERT_NOT_FOUND` for an alert that is not there yet, - * the scroll keyboard refusal for a surface the runner declined to swipe under the keys, and the - * retriable `APP_NOT_RUNNING` for a read the runner refused rather than launch the session app. + * `COMMAND_FAILED` unless `DIAGNOSTIC_RUNNER_ERROR_WIRE_CODES` names the code the caller already + * reads the family under, and survive as `details.runnerErrorCode`, which is what family policy + * reads: `RUNNER_BUSY` for retriable contention, `ALERT_NOT_FOUND` for an alert that is not there + * yet, the scroll keyboard refusal for a surface the runner declined to swipe under the keys, and + * the retriable `APP_NOT_RUNNING` for a read the runner refused rather than launch the session app. */ -const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlyMap = new Map([ +const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlyMap< + string, + Readonly<{ retriable?: true }> +> = new Map([ [RUNNER_BUSY_RUNNER_CODE, { retriable: true }], [MAIN_THREAD_TIMEOUT_RUNNER_CODE, {}], [APP_NOT_RUNNING_RUNNER_CODE, { retriable: true }], [ALERT_NOT_FOUND_RUNNER_CODE, {}], [SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, {}], + [KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE, {}], [SYNTHESIZED_GESTURE_NOT_DISPATCHED_RUNNER_CODE, {}], [SYNTHESIZED_GESTURE_OUTCOME_UNKNOWN_RUNNER_CODE, {}], [BACK_SWIPE_NO_CHANGE_RUNNER_CODE, {}], @@ -374,6 +389,15 @@ const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlyMap [code, {}] as const), ]); +/** + * Diagnostic-only codes that keep the wire code their command already refuses with instead of + * `COMMAND_FAILED`: the keyboard-dismiss refusal stays `UNSUPPORTED_OPERATION`, the code + * `keyboard dismiss` already refuses with. + */ +const DIAGNOSTIC_RUNNER_ERROR_WIRE_CODES: ReadonlyMap = new Map([ + [KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE, 'UNSUPPORTED_OPERATION'], +]); + /** * `details.reason` for the runner codes a consumer acts on, so it reads one field instead of * `details.runnerErrorCode`, which stays for the runner's own vocabulary. Both the codes that stay @@ -389,6 +413,7 @@ const RUNNER_REPORTED_ERROR_REASONS: ReadonlyMap = new [ALERT_UNRESOLVED_RUNNER_CODE, 'no'], [SYNTHESIZED_GESTURE_NOT_DISPATCHED_RUNNER_CODE, 'no'], [BACK_SWIPE_NOT_DISPATCHED_RUNNER_CODE, 'no'], + [KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE, 'no'], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, 'no'] as const), ]); @@ -431,7 +457,8 @@ type RunnerReportedErrorReason = | 'synthesized_gesture_outcome_unknown' | 'back_swipe_no_change' | 'back_swipe_unverified' - | 'back_swipe_not_dispatched'; + | 'back_swipe_not_dispatched' + | 'keyboard_dismiss_unavailable'; /** Wire code plus the details every path must publish for one runner-reported error code. */ export type RunnerReportedErrorClass = Readonly<{ @@ -457,13 +484,17 @@ export function classifyRunnerReportedError( runnerErrorCode === undefined ? undefined : DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES.get(runnerErrorCode); + const wireCode = + runnerErrorCode === undefined + ? undefined + : DIAGNOSTIC_RUNNER_ERROR_WIRE_CODES.get(runnerErrorCode); const reason = runnerErrorCode === undefined ? undefined : RUNNER_REPORTED_ERROR_REASONS.get(runnerErrorCode); return Object.freeze({ - code: diagnosticOnly ? 'COMMAND_FAILED' : toAppErrorCode(runnerErrorCode), + code: diagnosticOnly ? (wireCode ?? 'COMMAND_FAILED') : toAppErrorCode(runnerErrorCode), details: Object.freeze({ runnerErrorCode, - ...diagnosticOnly, + ...(diagnosticOnly?.retriable === true ? { retriable: true as const } : {}), ...(reason === undefined ? {} : { reason }), dispatched: (runnerErrorCode === undefined diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 293b2fe6de..2fcef204f9 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -301,7 +301,7 @@ const foldCommandDescription = `Fold or unfold a foldable iPhone simulator (iPho const appSwitcherCommandDescription = 'Open the device app switcher to inspect or change foreground apps. This changes the visible system UI and may move focus away from the current app.'; const keyboardCommandDescription = - 'Inspect Android keyboard visibility/type or press/dismiss the device keyboard. The dismiss action hides the keyboard. It taps the keyboard dismiss/hide key when one is exposed, verifies the keyboard closed, and reports UNSUPPORTED_OPERATION when no dismiss key exists \u2014 background taps are never attempted.'; + 'Inspect Android keyboard visibility/type or press/dismiss the device keyboard. The dismiss action hides the keyboard. It taps the keyboard dismiss/hide key when one is exposed, verifies the keyboard closed, and reports UNSUPPORTED_OPERATION when the keyboard stays up \u2014 background taps are never attempted. On iOS a keyboard with no dismiss key adds error.details.reason keyboard_dismiss_unavailable with dispatched: no; one whose key was tapped and stayed visible has no reason and dispatched: unknown.'; const clipboardCommandDescription = 'Read the current device clipboard text, or replace its contents with the given text. Android runs both through the clipboard service shell command, and a build that implements none (Android 16 does not) refuses with UNSUPPORTED_OPERATION rather than reporting an empty clipboard.'; const actionButtonCommandDescription = diff --git a/src/commands/schema/cli-help-command-usage.test.ts b/src/commands/schema/cli-help-command-usage.test.ts index f7d3b26726..ea84cc839f 100644 --- a/src/commands/schema/cli-help-command-usage.test.ts +++ b/src/commands/schema/cli-help-command-usage.test.ts @@ -85,6 +85,7 @@ test('usageForCommand documents keyboard dismissal flow', async () => { assert.match(help ?? '', /keyboard \[status\|get\|dismiss\|enter\|return\]/); assert.match(help ?? '', /taps the keyboard dismiss\/hide key when one is exposed/); assert.match(help ?? '', /UNSUPPORTED_OPERATION/); + assert.match(help ?? '', /error\.details\.reason keyboard_dismiss_unavailable/); }); test('usageForCommand explains focused perf evidence', async () => { diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index c9b6fad1d1..869e8455b6 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -105,8 +105,8 @@ Selectors: Text entry: fill replaces; type appends to an already-focused field: fill 'id="field-email"' "qa@example.com"; type "Handle with care" --delay-ms 80 fill "" clears the field (replace with nothing); the empty argument must be present -- fill alone is a missing argument. - For dropped iOS text, retry with --delay-ms before clipboard paste. This also applies to Limrun iOS, which uses the same Apple runner. - Element presses behind the keyboard are refused with tap_keyboard_occludes_target. keyboard dismiss taps its own dismiss key when one exists, else UNSUPPORTED_OPERATION. Android: try dismiss before back. iOS: when both fail, do not tap a static text/heading hoping it is safe; prefer type "\\n" to submit. + For dropped iOS text, retry with --delay-ms before clipboard paste (also Limrun iOS). + Element presses behind the keyboard are refused with tap_keyboard_occludes_target. keyboard dismiss taps its own dismiss key when one exists, else UNSUPPORTED_OPERATION (iOS: error.details.reason keyboard_dismiss_unavailable, dispatched: no). Android: try dismiss before back. iOS: when both fail, never tap static text/headings; prefer type "\\n" to submit. iOS paste-prompt limits and Android IME/handwriting capture quirks: help debugging. Session ordering: diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a93e539340..159ef9cf6e 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -458,7 +458,7 @@ agent-device alert dismiss - The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators, including Limrun, the XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. On iOS simulators, including Limrun, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline during action or confirmation after the alert is resolved fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. -- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). See Back above. +- iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. A synthesized long press, `scroll`, `swipe` (sent as a planned fling `gesture`), or other planned `gesture` that the event synthesis refuses before its event record is handed to XCTest reports `COMMAND_FAILED` with `synthesized_gesture_not_dispatched` (`dispatched: no`), and so does a planned `gesture` refused while its window or display is resolved (the window cannot be named, or the sole display cannot be measured); one that fails after the record was handed to XCTest (its events may or may not have run) reports `synthesized_gesture_outcome_unknown` (`dispatched: unknown`) and is never repeated through an XCTest fallback, so check the screen with `snapshot` before you retry. Any other refusal, such as a platform without touch input, keeps `UNSUPPORTED_OPERATION`. A long-press step in a press series reports the same reasons; once an earlier step has run, the series is `dispatched: unknown`. `back_swipe_no_change` means the iOS `back` swipe was sent and the screen did not change, and `back_swipe_unverified` means it was sent but may or may not have navigated (both `dispatched: unknown`); `back_swipe_not_dispatched` means it was refused before any touch went down (`dispatched: no`). A `keyboard dismiss` the keyboard would not come down for because it exposes no dismiss key keeps `UNSUPPORTED_OPERATION` and reports `keyboard_dismiss_unavailable` (`dispatched: no`, nothing was tapped); one whose dismiss key was tapped and left the keyboard visible keeps `UNSUPPORTED_OPERATION` with no `reason` and `dispatched: unknown`. See Back above. - On the Apple XCUITest runner, an alert search that fails, runs out of time, or finds an app alert, Safari dialog, or dismiss-popup window with no actionable controls fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check remained unresolved. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. - On the Apple XCUITest runner, `alert get` describes a readable alert even when all its buttons are disabled, and lists those disabled buttons. `alert accept` and `alert dismiss` fail with `ALERT_BUTTON_NOT_FOUND`. - If a sheet or permission prompt is visible but `alert` reports none, `alert` did not recognize it. Run `snapshot -i -s ""`, then `press @ref` on the visible button. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. @@ -914,7 +914,7 @@ agent-device keyboard dismiss ``` - `keyboard status` (or `keyboard get`) reports keyboard visibility and a best-effort input type on Android emulators and devices. -- `keyboard dismiss` taps a safe control such as `Done` when available and checks that the keyboard closed. It works on Android and, best-effort, on iOS simulators and devices. If it returns `UNSUPPORTED_OPERATION`, press a visible app control such as `Done` only when that is the intended fallback. +- `keyboard dismiss` taps a safe control such as `Done` when available and checks that the keyboard closed. It works on Android and, best-effort, on iOS simulators and devices. Both report `UNSUPPORTED_OPERATION` when the keyboard stays up. On iOS, a keyboard that exposes no dismiss control adds `error.details.reason: keyboard_dismiss_unavailable` with `dispatched: no`, because nothing was tapped; one whose control was tapped and stayed open has no `reason` and `dispatched: unknown`. Android reports its own `attempts` and IME fields with no `reason`. Either way, press a visible app control such as `Done` only when that is the intended fallback. - Works with the session device or a device selector (`--platform`, `--device`, `--udid`, `--serial`). ## Performance diagnostics