From 8b439ec4644d53663717add4c061ce7078b442de Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 14:54:36 +0900 Subject: [PATCH 1/3] fix(mobile): fail an action agent-device refuses for an unknown screen origin as an action failure --- .../mobile-window-origin-not-actionable.md | 5 +++++ docs/mobile.mdx | 4 ++++ packages/mobile/src/errors.ts | 21 ++++++++++++++++--- packages/mobile/tests/unit/errors.test.ts | 14 +++++++++++++ plugins/e2e/skills/e2e/references/setup.md | 3 +++ skills/e2e/references/setup.md | 3 +++ 6 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 .changeset/mobile-window-origin-not-actionable.md diff --git a/.changeset/mobile-window-origin-not-actionable.md b/.changeset/mobile-window-origin-not-actionable.md new file mode 100644 index 000000000..a3c5aff43 --- /dev/null +++ b/.changeset/mobile-window-origin-not-actionable.md @@ -0,0 +1,5 @@ +--- +'@e2e-dev/mobile': patch +--- + +An iOS action on a node that agent-device refuses with `window-origin-unknown` now fails as `ACTION_FAILED` (`NOT_ACTIONABLE`) instead of an infrastructure `ENGINE_FAILURE`. agent-device sent nothing, and the message names `screen.tapAt()`, which agent-device still takes. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index b29ff2aa0..6c823f898 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -808,6 +808,10 @@ export function cloudDevices(options: { provider: string; device: string; osVers nodes, and a tap on one fails with `ACTION_FAILED`. Call `device.dismissKeyboard()` or scroll the node clear first. See [Visibility](/reference/mobile#visibility). +- On iOS, an action on a node can fail with `ACTION_FAILED` and `has no + established screen-coordinate origin` on a heavy screen, such as right after + a large web view loads. Nothing was sent. Tap the point with + `screen.tapAt()` instead. - `doubleTap()` and `screen.doubleTapAt()` on iOS are two taps about 285 to 335 ms apart. Give a double-tap detector a window of 350 ms or more. - On iOS, `device.foregroundApp()` still names the session's app after diff --git a/packages/mobile/src/errors.ts b/packages/mobile/src/errors.ts index cc37936d2..d94f16c6f 100644 --- a/packages/mobile/src/errors.ts +++ b/packages/mobile/src/errors.ts @@ -11,6 +11,7 @@ * | a stale-ref refusal by `details.reason` (action paths only) | NODE_STALE (retryable) | * | target_still_moving with `details.dispatched: no` | NODE_STALE (retryable) | * | a covered or keyboard-occluded target, `dispatched: no` | NOT_ACTIONABLE, with the hint | + * | a target with no screen origin to aim at, `dispatched: no` | NOT_ACTIONABLE, with the hint | * | the iOS runner busy, wedged, or past its watchdog | ENGINE_FAILURE, naming the | * | a snapshot the iOS runner acquired but could not present | runner and the recovery | * | anything else, agent-device's own timeouts included | ENGINE_FAILURE with the text | @@ -102,6 +103,16 @@ const STALE_REF_REASONS: ReadonlySet = new Set([ */ const OCCLUDED_REASONS: ReadonlySet = new Set(['target_covered', 'tap_keyboard_occludes_target']); +/** + * The `details.reason` agent-device 0.21.37 refuses a ref action with when the + * capture the ref came from could not place its window on the screen. Nothing + * was sent, and the runner is healthy. agent-device's own hint names its CLI + * (`snapshot`, `screenshot`), so the failure carries the test's move instead: + * a point tap is not aimed from the capture, and agent-device takes it. + */ +const NO_SCREEN_ORIGIN_REASON = 'window-origin-unknown'; +const NO_SCREEN_ORIGIN_HINT = 'Nothing was sent. Tap the point instead with screen.tapAt().'; + /** * What the iOS automation runner reported about itself, by the code * agent-device emits for it: `RUNNER_WEDGED` at the top level, `RUNNER_BUSY` @@ -295,9 +306,10 @@ export async function runCommand(label: string, work: () => Promise, signa * stale-ref reasons are sufficient on their own: Android dialog recovery can * mark a refused drag as `dispatched: unknown`. A `target_still_moving` refusal * additionally requires `dispatched: no`; missing or unknown dispatch is not retryable. - * A target agent-device refused as covered (`OCCLUDED_REASONS`) with nothing - * dispatched is `NOT_ACTIONABLE`: the test's own failure, with the hint that - * names the cover, never an infrastructure one. + * A target agent-device refused as covered (`OCCLUDED_REASONS`) or as having + * no screen origin (`NO_SCREEN_ORIGIN_REASON`) with nothing dispatched is + * `NOT_ACTIONABLE`, with the move that gets past it: an action failure on a + * healthy device, never an infrastructure one. */ export function staleOr(cause: unknown, operation: string, where?: string): Error { if (isClassified(cause)) return cause; @@ -309,5 +321,8 @@ export function staleOr(cause: unknown, operation: string, where?: string): Erro if (typeof reason === 'string' && OCCLUDED_REASONS.has(reason) && dispatched === 'no') { return new EngineError('NOT_ACTIONABLE', `${operation} failed: ${withHint(cause, message(cause))}`, { retryable: false, cause }); } + if (reason === NO_SCREEN_ORIGIN_REASON && dispatched === 'no') { + return new EngineError('NOT_ACTIONABLE', `${operation} failed: ${message(cause)} Hint: ${NO_SCREEN_ORIGIN_HINT}`, { retryable: false, cause }); + } return translateError(cause, operation, where); } diff --git a/packages/mobile/tests/unit/errors.test.ts b/packages/mobile/tests/unit/errors.test.ts index 1e5fb9c1f..6fd011f5f 100644 --- a/packages/mobile/tests/unit/errors.test.ts +++ b/packages/mobile/tests/unit/errors.test.ts @@ -121,6 +121,20 @@ describe('error translation', () => { }); }); + // As agent-device 0.21.37 sends it for a ref aimed from a capture without the window's screen origin. + it('fails an action agent-device refused as window-origin-unknown with NOT_ACTIONABLE and the tapAt move', () => { + const text = 'Ref @e2 has no established screen-coordinate origin'; + const hint = 'Take a fresh snapshot after the accessibility channel recovers, or use screenshot to choose screen coordinates.'; + const refused = new AppError('COMMAND_FAILED', text, { reason: 'window-origin-unknown', hint, dispatched: 'no' }); + const failure = staleOr(refused, 'perform tap'); + expect(failure).toMatchObject({ code: 'NOT_ACTIONABLE', retryable: false, cause: refused }); + // agent-device's hint names its CLI; the failure names the test's move instead. + expect(failure.message).toBe(`perform tap failed: ${text} Hint: Nothing was sent. Tap the point instead with screen.tapAt().`); + expect( + staleOr(new AppError('COMMAND_FAILED', text, { reason: 'window-origin-unknown', hint, dispatched: 'unknown' }), 'perform tap'), + ).toMatchObject({ code: 'ENGINE_FAILURE' }); + }); + it('maps the rest to ENGINE_FAILURE with the text, a ref named in a message without a stale reason included', () => { expect(staleOr(new Error('ref @e12 not found in the current snapshot'), 'perform tap')).toMatchObject({ code: 'ENGINE_FAILURE', diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index 0afc20fb9..d1e008cd5 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -422,6 +422,9 @@ export default { child still showing stay visible. A tap on a node agent-device finds covered (by an overlay or the keyboard) fails `ACTION_FAILED` naming the cover; dismiss it or scroll the target clear. +- On iOS, an action on a node can fail `ACTION_FAILED` with `has no + established screen-coordinate origin` on a heavy screen (a large web view + just loaded). Nothing was sent; tap the point with `screen.tapAt()`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 0afc20fb9..d1e008cd5 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -422,6 +422,9 @@ export default { child still showing stay visible. A tap on a node agent-device finds covered (by an overlay or the keyboard) fails `ACTION_FAILED` naming the cover; dismiss it or scroll the target clear. +- On iOS, an action on a node can fail `ACTION_FAILED` with `has no + established screen-coordinate origin` on a heavy screen (a large web view + just loaded). Nothing was sent; tap the point with `screen.tapAt()`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` From bf53278288e986c97fd7d76863518a112fbefc9c Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 15:12:51 +0900 Subject: [PATCH 2/3] fix(mobile): name screen.tapAt only for a refused tap --- .changeset/mobile-window-origin-not-actionable.md | 2 +- docs/mobile.mdx | 2 +- packages/mobile/src/errors.ts | 5 +++-- packages/mobile/tests/unit/errors.test.ts | 2 +- plugins/e2e/skills/e2e/references/setup.md | 3 ++- skills/e2e/references/setup.md | 3 ++- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.changeset/mobile-window-origin-not-actionable.md b/.changeset/mobile-window-origin-not-actionable.md index a3c5aff43..57bda687e 100644 --- a/.changeset/mobile-window-origin-not-actionable.md +++ b/.changeset/mobile-window-origin-not-actionable.md @@ -2,4 +2,4 @@ '@e2e-dev/mobile': patch --- -An iOS action on a node that agent-device refuses with `window-origin-unknown` now fails as `ACTION_FAILED` (`NOT_ACTIONABLE`) instead of an infrastructure `ENGINE_FAILURE`. agent-device sent nothing, and the message names `screen.tapAt()`, which agent-device still takes. +An iOS action on a node that agent-device refuses with `window-origin-unknown` now fails as `ACTION_FAILED` (`NOT_ACTIONABLE`) instead of an infrastructure `ENGINE_FAILURE`. agent-device sent nothing, and for a tap the message names `screen.tapAt()`, which agent-device still takes. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index 6c823f898..9951b20ef 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -810,7 +810,7 @@ export function cloudDevices(options: { provider: string; device: string; osVers [Visibility](/reference/mobile#visibility). - On iOS, an action on a node can fail with `ACTION_FAILED` and `has no established screen-coordinate origin` on a heavy screen, such as right after - a large web view loads. Nothing was sent. Tap the point with + a large web view loads. Nothing was sent. For a tap, tap the point with `screen.tapAt()` instead. - `doubleTap()` and `screen.doubleTapAt()` on iOS are two taps about 285 to 335 ms apart. Give a double-tap detector a window of 350 ms or more. diff --git a/packages/mobile/src/errors.ts b/packages/mobile/src/errors.ts index d94f16c6f..d0f323bc3 100644 --- a/packages/mobile/src/errors.ts +++ b/packages/mobile/src/errors.ts @@ -108,10 +108,11 @@ const OCCLUDED_REASONS: ReadonlySet = new Set(['target_covered', 'tap_ke * capture the ref came from could not place its window on the screen. Nothing * was sent, and the runner is healthy. agent-device's own hint names its CLI * (`snapshot`, `screenshot`), so the failure carries the test's move instead: - * a point tap is not aimed from the capture, and agent-device takes it. + * for a tap, a point tap is not aimed from the capture, and agent-device + * takes it. Other actions have no point form, so the hint stays on the tap. */ const NO_SCREEN_ORIGIN_REASON = 'window-origin-unknown'; -const NO_SCREEN_ORIGIN_HINT = 'Nothing was sent. Tap the point instead with screen.tapAt().'; +const NO_SCREEN_ORIGIN_HINT = 'Nothing was sent. For a tap, tap the point instead with screen.tapAt().'; /** * What the iOS automation runner reported about itself, by the code diff --git a/packages/mobile/tests/unit/errors.test.ts b/packages/mobile/tests/unit/errors.test.ts index 6fd011f5f..20e02b325 100644 --- a/packages/mobile/tests/unit/errors.test.ts +++ b/packages/mobile/tests/unit/errors.test.ts @@ -129,7 +129,7 @@ describe('error translation', () => { const failure = staleOr(refused, 'perform tap'); expect(failure).toMatchObject({ code: 'NOT_ACTIONABLE', retryable: false, cause: refused }); // agent-device's hint names its CLI; the failure names the test's move instead. - expect(failure.message).toBe(`perform tap failed: ${text} Hint: Nothing was sent. Tap the point instead with screen.tapAt().`); + expect(failure.message).toBe(`perform tap failed: ${text} Hint: Nothing was sent. For a tap, tap the point instead with screen.tapAt().`); expect( staleOr(new AppError('COMMAND_FAILED', text, { reason: 'window-origin-unknown', hint, dispatched: 'unknown' }), 'perform tap'), ).toMatchObject({ code: 'ENGINE_FAILURE' }); diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index d1e008cd5..e4947ddfe 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -424,7 +424,8 @@ export default { `ACTION_FAILED` naming the cover; dismiss it or scroll the target clear. - On iOS, an action on a node can fail `ACTION_FAILED` with `has no established screen-coordinate origin` on a heavy screen (a large web view - just loaded). Nothing was sent; tap the point with `screen.tapAt()`. + just loaded). Nothing was sent; for a tap, tap the point with + `screen.tapAt()`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index d1e008cd5..e4947ddfe 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -424,7 +424,8 @@ export default { `ACTION_FAILED` naming the cover; dismiss it or scroll the target clear. - On iOS, an action on a node can fail `ACTION_FAILED` with `has no established screen-coordinate origin` on a heavy screen (a large web view - just loaded). Nothing was sent; tap the point with `screen.tapAt()`. + just loaded). Nothing was sent; for a tap, tap the point with + `screen.tapAt()`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` From e469d2811f2423e7357a81ff357f19aa740c3a56 Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 19:17:13 +0900 Subject: [PATCH 3/3] refactor(mobile): one map of not-actionable refusal reasons and their hints; drop the web view note --- docs/mobile.mdx | 4 -- packages/mobile/src/errors.ts | 50 +++++++++++----------- plugins/e2e/skills/e2e/references/setup.md | 4 -- skills/e2e/references/setup.md | 4 -- 4 files changed, 24 insertions(+), 38 deletions(-) diff --git a/docs/mobile.mdx b/docs/mobile.mdx index 9951b20ef..b29ff2aa0 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -808,10 +808,6 @@ export function cloudDevices(options: { provider: string; device: string; osVers nodes, and a tap on one fails with `ACTION_FAILED`. Call `device.dismissKeyboard()` or scroll the node clear first. See [Visibility](/reference/mobile#visibility). -- On iOS, an action on a node can fail with `ACTION_FAILED` and `has no - established screen-coordinate origin` on a heavy screen, such as right after - a large web view loads. Nothing was sent. For a tap, tap the point with - `screen.tapAt()` instead. - `doubleTap()` and `screen.doubleTapAt()` on iOS are two taps about 285 to 335 ms apart. Give a double-tap detector a window of 350 ms or more. - On iOS, `device.foregroundApp()` still names the session's app after diff --git a/packages/mobile/src/errors.ts b/packages/mobile/src/errors.ts index d0f323bc3..637e7c539 100644 --- a/packages/mobile/src/errors.ts +++ b/packages/mobile/src/errors.ts @@ -95,24 +95,24 @@ const STALE_REF_REASONS: ReadonlySet = new Set([ ]); /** - * The `details.reason` values agent-device refuses an action with when - * something else is drawn where the touch would land: an overlay its - * occlusion scan found over the target (`coveredBy` names it), or the soft - * keyboard over the target's touch point. Nothing was sent, and the app is - * fine; the test has to move the cover or the target first. - */ -const OCCLUDED_REASONS: ReadonlySet = new Set(['target_covered', 'tap_keyboard_occludes_target']); - -/** - * The `details.reason` agent-device 0.21.37 refuses a ref action with when the - * capture the ref came from could not place its window on the screen. Nothing - * was sent, and the runner is healthy. agent-device's own hint names its CLI - * (`snapshot`, `screenshot`), so the failure carries the test's move instead: - * for a tap, a point tap is not aimed from the capture, and agent-device - * takes it. Other actions have no point form, so the hint stays on the tap. + * The `details.reason` values agent-device refuses an action with, nothing + * sent, on a healthy device, each with the hint the failure carries: + * `undefined` keeps agent-device's own, and a string replaces it. + * + * - `target_covered`, `tap_keyboard_occludes_target`: an overlay its + * occlusion scan found over the target (`coveredBy` names it), or the soft + * keyboard over the target's touch point. The test has to move the cover + * or the target first, and agent-device's hint says how. + * - `window-origin-unknown` (0.21.37): the capture the ref came from could not + * place its window on the screen. agent-device's hint names its CLI + * (`snapshot`, `screenshot`); a point tap is not aimed from the capture, and + * agent-device takes it. */ -const NO_SCREEN_ORIGIN_REASON = 'window-origin-unknown'; -const NO_SCREEN_ORIGIN_HINT = 'Nothing was sent. For a tap, tap the point instead with screen.tapAt().'; +const NOT_ACTIONABLE_REASONS: ReadonlyMap = new Map([ + ['target_covered', undefined], + ['tap_keyboard_occludes_target', undefined], + ['window-origin-unknown', 'Nothing was sent. For a tap, tap the point instead with screen.tapAt().'], +]); /** * What the iOS automation runner reported about itself, by the code @@ -307,10 +307,9 @@ export async function runCommand(label: string, work: () => Promise, signa * stale-ref reasons are sufficient on their own: Android dialog recovery can * mark a refused drag as `dispatched: unknown`. A `target_still_moving` refusal * additionally requires `dispatched: no`; missing or unknown dispatch is not retryable. - * A target agent-device refused as covered (`OCCLUDED_REASONS`) or as having - * no screen origin (`NO_SCREEN_ORIGIN_REASON`) with nothing dispatched is - * `NOT_ACTIONABLE`, with the move that gets past it: an action failure on a - * healthy device, never an infrastructure one. + * A target agent-device refused for a reason in `NOT_ACTIONABLE_REASONS` with + * nothing dispatched is `NOT_ACTIONABLE`, with the move that gets past it: an + * action failure on a healthy device, never an infrastructure one. */ export function staleOr(cause: unknown, operation: string, where?: string): Error { if (isClassified(cause)) return cause; @@ -319,11 +318,10 @@ export function staleOr(cause: unknown, operation: string, where?: string): Erro (reason === 'target_still_moving' && dispatched === 'no')) { return new EngineError('NODE_STALE', `${operation}: ${message(cause)}`, { retryable: true, cause }); } - if (typeof reason === 'string' && OCCLUDED_REASONS.has(reason) && dispatched === 'no') { - return new EngineError('NOT_ACTIONABLE', `${operation} failed: ${withHint(cause, message(cause))}`, { retryable: false, cause }); - } - if (reason === NO_SCREEN_ORIGIN_REASON && dispatched === 'no') { - return new EngineError('NOT_ACTIONABLE', `${operation} failed: ${message(cause)} Hint: ${NO_SCREEN_ORIGIN_HINT}`, { retryable: false, cause }); + if (typeof reason === 'string' && NOT_ACTIONABLE_REASONS.has(reason) && dispatched === 'no') { + const hint = NOT_ACTIONABLE_REASONS.get(reason); + const text = hint === undefined ? withHint(cause, message(cause)) : `${message(cause)} Hint: ${hint}`; + return new EngineError('NOT_ACTIONABLE', `${operation} failed: ${text}`, { retryable: false, cause }); } return translateError(cause, operation, where); } diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index e4947ddfe..0afc20fb9 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -422,10 +422,6 @@ export default { child still showing stay visible. A tap on a node agent-device finds covered (by an overlay or the keyboard) fails `ACTION_FAILED` naming the cover; dismiss it or scroll the target clear. -- On iOS, an action on a node can fail `ACTION_FAILED` with `has no - established screen-coordinate origin` on a heavy screen (a large web view - just loaded). Nothing was sent; for a tap, tap the point with - `screen.tapAt()`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index e4947ddfe..0afc20fb9 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -422,10 +422,6 @@ export default { child still showing stay visible. A tap on a node agent-device finds covered (by an overlay or the keyboard) fails `ACTION_FAILED` naming the cover; dismiss it or scroll the target clear. -- On iOS, an action on a node can fail `ACTION_FAILED` with `has no - established screen-coordinate origin` on a heavy screen (a large web view - just loaded). Nothing was sent; for a tap, tap the point with - `screen.tapAt()`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold`