Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -587,23 +587,15 @@ extension RunnerTests {
}
return Response(ok: true, data: DataPayload(message: command.command.rawValue))
case .keyboardDismiss:
var dismissal: (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?)?
var dismissal: RunnerKeyboardDismissOutcome?
withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) {
dismissal = dismissKeyboard(app: activeApp)
}
guard let result = dismissal else {
return Response(ok: false, error: ErrorPayload(message: "keyboardDismiss produced no result"))
}
if result.wasVisible && !result.dismissed {
return Response(
ok: false,
error: ErrorPayload(
code: "UNSUPPORTED_OPERATION",
message: "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)",
hint:
"An element whose center sits behind the on-screen keyboard is refused with tap_keyboard_occludes_target; one whose center stays above the keys presses normally. To end editing, tap the app's own Done/Cancel control, or use keyboard enter to press the return key when submission is wanted."
)
)
return Response(ok: false, error: keyboardDismissRefusal(dispatchedDismissal: result.dispatchedDismissal))
}
return Response(
ok: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,18 @@ enum RunnerKeyboardDismissMechanism: String {
case dismissKey
}

/// One `keyboard dismiss` attempt, plus the fact that decides how a refusal is classified:
/// `dispatchedDismissal` says whether the runner ever dispatched a dismissal.
struct RunnerKeyboardDismissOutcome {
let wasVisible: Bool
let dismissed: Bool
let visible: Bool
let mechanism: RunnerKeyboardDismissMechanism?
/// Whether a dismissal was dispatched — the keyboard's own control was tapped, or the tvOS remote's
/// Menu press landed. Only its absence proves the command sent nothing.
let dispatchedDismissal: Bool
}

/// Keyboard coverage read from the capture's complete pre-projection root snapshot on iOS XCTest.
/// Visible frames include the input assistant and share the nodes' normalized coordinate space.
/// A complete scan with no keyboard establishes absence; projected or partial trees never do.
Expand Down Expand Up @@ -189,20 +201,32 @@ extension RunnerTests {
return visibleKeyboardFrame(app: app) != nil
}

func dismissKeyboard(
app: XCUIApplication
) -> (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?) {
func dismissKeyboard(app: XCUIApplication) -> RunnerKeyboardDismissOutcome {
let keyboard = app.keyboards.firstMatch
let wasVisible = isKeyboardVisible(app: app)
guard wasVisible else {
return (wasVisible: false, dismissed: false, visible: false, mechanism: nil)
return RunnerKeyboardDismissOutcome(
wasVisible: false,
dismissed: false,
visible: false,
mechanism: nil,
dispatchedDismissal: false
)
}

#if os(tvOS)
_ = pressTvRemote(.menu)
sleepFor(0.2)
let visible = isKeyboardVisible(app: app)
return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
let tvVisible = isKeyboardVisible(app: app)
return RunnerKeyboardDismissOutcome(
wasVisible: true,
dismissed: !tvVisible,
visible: tvVisible,
mechanism: tvVisible ? nil : .dismissKey,
// On tvOS the Menu press always goes out, so a refusal after it did dispatch and keeps the
// generic code.
dispatchedDismissal: true
)
Comment thread
thymikee marked this conversation as resolved.
Resolved
#else
if tapKeyboardDismissControl(app: app) {
_ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout)
Expand All @@ -213,7 +237,13 @@ extension RunnerTests {
requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches
)
let visible = isKeyboardVisible(app: app)
return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
return RunnerKeyboardDismissOutcome(
wasVisible: true,
dismissed: !visible,
visible: visible,
mechanism: visible ? nil : .dismissKey,
dispatchedDismissal: true
)
}

// #1606 review P1 (twice): generic background-tap dismissal is
Expand All @@ -222,10 +252,41 @@ extension RunnerTests {
// indistinguishable from an inert backdrop, so a "safe-area" tap can
// navigate or submit while reporting a successful dismiss. The dismiss
// key is the only mechanism the runner can vouch for.
return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app), mechanism: nil)
return RunnerKeyboardDismissOutcome(
wasVisible: true,
dismissed: false,
visible: isKeyboardVisible(app: app),
mechanism: nil,
dispatchedDismissal: false
)
#endif
}

/// The refusal a visible keyboard that stayed up answers with. The two ways it stays up answer
/// different questions and must not share one classification: finding no dismiss control sent
/// nothing and is the runner's own typed refusal (`KEYBOARD_DISMISS_UNAVAILABLE`, so the host
/// publishes its `details.reason` instead of leaving callers to read the prose), while a control
/// that was tapped and left the keys up did dispatch and keeps the generic code.
func keyboardDismissRefusal(dispatchedDismissal: Bool) -> ErrorPayload {
if dispatchedDismissal {
return ErrorPayload(
code: "UNSUPPORTED_OPERATION",
message: "Unable to dismiss the keyboard: a dismiss mechanism ran and the keyboard stayed visible",
hint: keyboardDismissRefusalHint
)
}
return ErrorPayload(
code: RunnerWireErrorCode.keyboardDismissUnavailable,
message: "Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)",
hint: keyboardDismissRefusalHint
)
}

private var keyboardDismissRefusalHint: String {
return
"An element whose center sits behind the on-screen keyboard is refused with tap_keyboard_occludes_target; one whose center stays above the keys presses normally. To end editing, tap the app's own Done/Cancel control, or use keyboard enter to press the return key when submission is wanted."
}


// AX-free on purpose (screenshot bytes, not the accessibility tree) so it holds
// under the same AX degradation the synthesized gesture lane is built to survive.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ final class RunnerTests: XCTestCase {
static let backSwipeNoChange = "BACK_SWIPE_NO_CHANGE"
static let backSwipeUnverified = "BACK_SWIPE_UNVERIFIED"
static let backSwipeNotDispatched = "BACK_SWIPE_NOT_DISPATCHED"
/// A `keyboard dismiss` the visible keyboard would not come down for: it exposes no dismiss key the
/// runner vouches for (#1606). The host reports it under `UNSUPPORTED_OPERATION` and publishes
/// `details.reason`, so a caller never reads the refusal off the message prose.
static let keyboardDismissUnavailable = "KEYBOARD_DISMISS_UNAVAILABLE"
/// Refuses a repeat of a `commandId` the runner already finished, when that command's own
/// classification keeps no response body to replay. Nothing ran for the repeat, but the daemon's
/// dispatch disclosure stays `unknown` — only the runner knows whether an earlier send of that id
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,21 @@ extension RunnerTests {

#endif

/// The two ways a visible keyboard stays up answer different questions, so they must not share one
/// code (#1606). Finding no dismiss control sent nothing and is the runner's own refusal; a control
/// that was tapped and left the keys up did dispatch and keeps the generic code. The host reads the
/// first code's literal to publish `details.reason`, so it is pinned here as the wire value.
func testKeyboardDismissRefusalNamesItsOwnCodeOnlyWhenNothingWasDispatched() {
let notDispatched = keyboardDismissRefusal(dispatchedDismissal: false)
XCTAssertEqual(notDispatched.code, "KEYBOARD_DISMISS_UNAVAILABLE")
XCTAssertNotNil(notDispatched.hint)

let dispatched = keyboardDismissRefusal(dispatchedDismissal: true)
XCTAssertEqual(dispatched.code, "UNSUPPORTED_OPERATION")
XCTAssertNotNil(dispatched.hint)
XCTAssertNotEqual(dispatched.code, notDispatched.code)
}

func testKeyboardBandFactPublishesAbsenceWhenTheCompleteCaptureHasNoKeyboard() {
let fact = runnerKeyboardBandFact(exists: false, frame: .zero)
XCTAssertEqual(fact, .absent)
Expand Down
17 changes: 13 additions & 4 deletions apple/runner/RUNNER_PROTOCOL.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,12 +252,19 @@ Successful and failed responses use the same top-level envelope:
{
"ok": false,
"error": {
"code": "UNSUPPORTED_OPERATION",
"message": "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key (background taps are never attempted)"
"code": "KEYBOARD_DISMISS_UNAVAILABLE",
"message": "Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)"
}
}
```

The example is a `keyboard dismiss` the keyboard would not come down for; a refusal that can name a
next step also carries `hint`, omitted here. A keyboard that stayed up answers one of two codes:
`KEYBOARD_DISMISS_UNAVAILABLE` when it found no dismiss control and so sent nothing, or
`UNSUPPORTED_OPERATION` when a dismiss control was tapped, or the tvOS remote's Menu press was sent,
and the keyboard stayed visible. The host reports both under `UNSUPPORTED_OPERATION` and publishes the
first one's `details.reason`.

`data` is command-specific. Common fields include snapshot nodes, text lookup results, gesture timing, visibility metadata, and screenshot or recording output details.

Capture-plan snapshots carry `snapshotViewport`, the acquisition's declared viewport evidence.
Expand Down Expand Up @@ -310,7 +317,9 @@ declaration for the current rules.
the main-thread watchdog in [`RunnerTests+Transport.swift`](AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift). The daemon
classifies a runner-reported code in `classifyRunnerReportedError`
([`runner-contract.ts`](../../packages/apple-runner/src/runner-contract.ts)): codes listed in `DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES`
arrive as `COMMAND_FAILED` with `details.runnerErrorCode`, and other codes pass through as typed
codes. How the daemon reacts to a failure (retry, resend, session-fatal) is declared in
arrive as `COMMAND_FAILED` with `details.runnerErrorCode`, unless their row names the wire code the
caller already reads that family under. Other codes pass through as typed codes. A code with a
`RUNNER_REPORTED_ERROR_REASONS` row also publishes `details.reason`, so a caller never reads the
refusal off `message`. How the daemon reacts to a failure (retry, resend, session-fatal) is declared in
`RUNNER_ERROR_RULES` and the helpers beside it
([`runner-error-classification.ts`](../../packages/apple-runner/src/runner-error-classification.ts)).
6 changes: 6 additions & 0 deletions contracts/fixtures/dispatch-disclosure.json
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,12 @@
"trigger": "structured runner reply with code COMMAND_ALREADY_FINISHED: the journal refused a repeat of a spent commandId and kept no outcome to replay, so it cannot say what that command did",
"dispatched": "unknown"
},
{
"id": "ios-runner.reply.KEYBOARD_DISMISS_UNAVAILABLE",
"producer": "ios-runner",
"trigger": "structured runner reply with code KEYBOARD_DISMISS_UNAVAILABLE: `keyboard dismiss` refused because the visible keyboard exposes no dismiss key, before any tap",
"dispatched": "no"
},
{
"id": "ios-runner.reply.unlisted-code",
"producer": "ios-runner",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ const DRIVERS: Record<string, () => Promise<unknown>> = {
replyFailure('SYNTHESIZED_GESTURE_NOT_DISPATCHED'),
'ios-runner.reply.SYNTHESIZED_GESTURE_OUTCOME_UNKNOWN': () =>
replyFailure('SYNTHESIZED_GESTURE_OUTCOME_UNKNOWN'),
'ios-runner.reply.KEYBOARD_DISMISS_UNAVAILABLE': () =>
replyFailure('KEYBOARD_DISMISS_UNAVAILABLE'),
'ios-runner.reply.unlisted-code': () => replyFailure('XCTEST_RECORDED_FAILURE'),
'ios-runner.series.later-chunk-refused': pressSeriesRefusedOnSecondChunk,
'ios-runner.series.invalid-step-before-send': pressSeriesWithInvalidStep,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { test } from 'vitest';
import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors';
import {
COMMAND_ALREADY_FINISHED_RUNNER_CODE,
KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE,
MAIN_THREAD_TIMEOUT_RUNNER_CODE,
NO_TEXT_INPUT_AT_POINT_RUNNER_CODE,
RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES,
Expand All @@ -22,7 +23,10 @@ import {
shouldRetryRunnerConnectError,
} from '../runner-error-classification.ts';
import { runnerConnectFailure, unwrittenConnectRefusal } from './runner-session-fixtures.ts';
import { requireSwiftThrownErrorWireCode } from './runner-swift-settlement-fixtures.ts';
import {
requireSwiftKeyboardDismissRefusalWireCode,
requireSwiftThrownErrorWireCode,
} from './runner-swift-settlement-fixtures.ts';

const TAP = { command: 'tap' } as const;
const SNAPSHOT = { command: 'snapshot' } as const;
Expand Down Expand Up @@ -169,6 +173,42 @@ test('a back swipe that proved nothing stays COMMAND_FAILED, unknown, with its o
assert.equal(notDispatched.details.dispatched, 'no');
});

// The keyboard stays up two different ways, and only one of them sent nothing. The literal is what the
// Swift runner encodes, so it is the contract and not the constant name: a rename on one side has to
// fail here rather than silently split the pair.
test('a keyboard dismiss refused for having no dismiss key keeps its code and publishes its reason', () => {
assert.equal(KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE, 'KEYBOARD_DISMISS_UNAVAILABLE');
const refusal = classifyRunnerReportedError(KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE);
// The code contract for `keyboard dismiss` is unchanged: the refusal travels under the code that
// command already refused with, so a caller keyed on the code sees what it saw before.
assert.equal(refusal.code, 'UNSUPPORTED_OPERATION');
assert.equal(refusal.details.runnerErrorCode, 'KEYBOARD_DISMISS_UNAVAILABLE');
assert.equal(refusal.details.reason, 'keyboard_dismiss_unavailable');
assert.equal(refusal.details.dispatched, 'no');
assert.equal(refusal.details.retriable, undefined);
// The whole published shape: the runner's own code survives, the wire code it hides under does not.
assert.deepEqual(refusal.details, {
runnerErrorCode: 'KEYBOARD_DISMISS_UNAVAILABLE',
reason: 'keyboard_dismiss_unavailable',
dispatched: 'no',
});
const refusalError = new AppError(refusal.code, refusal.details.reason ?? '', refusal.details);
assert.equal(isRetryableRunnerError(refusalError), false);
assert.equal(isRunnerBusyError(refusalError), false);
// The closest negative: the same refusal in prose alone, from an unclassified reply, earns no
// reason, so a consumer keyed on `reason` cannot mistake it for the pre-dispatch refusal.
const untyped = commandFailed(
'Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted',
);
assert.equal(untyped.details?.reason, undefined);
// A dismiss control that was tapped and left the keys up did dispatch, so it keeps the generic code
// and says nothing about dispatch.
const dispatched = classifyRunnerReportedError('UNSUPPORTED_OPERATION');
assert.equal(dispatched.code, 'UNSUPPORTED_OPERATION');
assert.equal(dispatched.details.reason, undefined);
assert.equal(dispatched.details.dispatched, 'unknown');
});

test('retryable requires an AppError with COMMAND_FAILED', () => {
assert.equal(isRetryableRunnerError(new Error('fetch failed')), false);
assert.equal(isRetryableRunnerError(new AppError('DEVICE_NOT_FOUND', 'fetch failed')), false);
Expand Down Expand Up @@ -560,6 +600,15 @@ test('the runner declares the wire codes the daemon reads for occupancy', () =>
);
});

test('the runner declares the wire code the keyboard-dismiss refusal publishes its reason under', () => {
assert.equal(
requireSwiftKeyboardDismissRefusalWireCode(),
KEYBOARD_DISMISS_UNAVAILABLE_RUNNER_CODE,
'the runner must encode its no-dismiss-key refusal under the code the daemon maps to ' +
'details.reason, or that reason becomes unreachable',
);
});

// --- runner that predates the listener token ---

test('a runner that does not require the token is session-fatal and never retried or replayed', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ import { isRetryableRunnerError } from '../runner-error-classification.ts';
import type { RunnerLogAttempt } from '../runner-failure-diagnostics.ts';
import { parseRunnerResponse } from '../runner-exchange.ts';

test('parseRunnerResponse preserves runner unsupported-operation codes', async () => {
test('parseRunnerResponse keeps the keyboard-dismiss refusal under the code the command reports', async () => {
const response = new Response(
JSON.stringify({
ok: false,
error: {
code: 'UNSUPPORTED_OPERATION',
message: 'Unable to dismiss the iOS keyboard without a safe native dismiss control',
code: 'KEYBOARD_DISMISS_UNAVAILABLE',
message:
'Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)',
},
}),
);
Expand All @@ -26,8 +27,12 @@ test('parseRunnerResponse preserves runner unsupported-operation codes', async (
() => parseRunnerResponse(response, session, runnerLogAttempt('/tmp/runner.log')),
(error: unknown) => {
assert.ok(error instanceof AppError);
// The runner's own code stays off the wire: `keyboard dismiss` keeps refusing with the code it
// always refused with, and the reason is what tells the two ways the keyboard stays up apart.
assert.equal(error.code, 'UNSUPPORTED_OPERATION');
assert.match(error.message, /Unable to dismiss the iOS keyboard/i);
assert.equal(error.details?.reason, 'keyboard_dismiss_unavailable');
assert.equal(error.details?.runnerErrorCode, 'KEYBOARD_DISMISS_UNAVAILABLE');
assert.equal(error.details?.dispatched, 'no');
return true;
},
);
Expand All @@ -40,8 +45,9 @@ test('parseRunnerResponse surfaces the keyboard-dismiss hint naming the occlusio
JSON.stringify({
ok: false,
error: {
code: 'UNSUPPORTED_OPERATION',
message: 'Unable to dismiss the iOS keyboard without a safe native dismiss control',
code: 'KEYBOARD_DISMISS_UNAVAILABLE',
message:
'Unable to dismiss the keyboard: it exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)',
hint,
},
}),
Expand All @@ -54,9 +60,6 @@ test('parseRunnerResponse surfaces the keyboard-dismiss hint naming the occlusio
assert.ok(error instanceof AppError);
assert.equal(error.code, 'UNSUPPORTED_OPERATION');
assert.equal(error.details?.hint, hint);
assert.match(String(error.details?.hint), /tap_keyboard_occludes_target/);
assert.match(String(error.details?.hint), /center stays above the keys/i);
assert.match(String(error.details?.hint), /keyboard enter/i);
return true;
},
);
Expand Down
Loading
Loading