From 44e514eceb9a21fe315faa1bff016be67b451ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 16:46:40 +0100 Subject: [PATCH 1/4] feat(open): let a polling caller skip the first-screen wait An app whose first screen labels nothing, such as a game canvas, held the first capture after open for the full 10 s first-screen budget. open now accepts the existing postGestureStabilization opt-out (--no-post-gesture-stabilization, the user config key, or AGENT_DEVICE_POST_GESTURE_STABILIZATION=0). With it, open marks no wait and still drops the record the replaced screen left pending. Co-Authored-By: Claude Opus 5.5 --- .../src/flag-definitions-action.ts | 4 ++-- packages/command-surface/src/management/app.ts | 4 ++++ packages/contracts/src/client-app.ts | 4 +++- packages/contracts/src/client-gesture.ts | 7 ++++--- packages/contracts/src/interaction-guarantees.ts | 2 +- .../deferred-interaction-outcome.test.ts | 15 +++++++++++++++ .../src/core/deferred-interaction-outcome.ts | 9 +++++++++ .../internal/session-open-execution.ts | 2 +- website/docs/docs/commands.md | 4 ++-- website/docs/docs/configuration.md | 2 +- 10 files changed, 42 insertions(+), 11 deletions(-) diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 0f15125d2e..e36333c619 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -236,9 +236,9 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ // An opt-out for callers that verify by polling: a synopsis would offer it on every scroll and swipe. usageHidden: true, usageDescription: - 'Scroll, swipe, gesture swipe: skip the movement check and the stabilization wait that follow a gesture; use it only when your own polling verifies the screen', + 'Scroll, swipe, gesture swipe, open: skip the movement check and the stabilization wait that follow a gesture, and the first-screen wait after an app launch; use it only when your own polling verifies the screen', inputDescription: - 'Set false to skip the movement check on a directional scroll and the stabilization wait before the next read. Set it only when your own polling verifies the screen: the next read can return a tree that the gesture is still moving.', + 'Set false to skip the movement check on a directional scroll and the stabilization wait before the next read, or the first-screen wait after an app launch. Set it only when your own polling verifies the screen: the next read can return a tree that the gesture is still moving or the launch screen.', // A repository must not switch off the check that keeps reads after a gesture from a stale tree. projectConfig: false, // A per-request policy of the caller, not part of the step: a replay keeps the default. diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index 659b12319c..77bba7d068 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -11,6 +11,7 @@ import { jsonSchemaField, leaseDeviceSpecFields, optionField, + postGestureStabilizationFields, stringArrayField, stringField, stringSchema, @@ -87,6 +88,7 @@ const openCommandMetadata = defineFieldCommandMetadata( launchUrl: stringField( 'Launch URL for the opened app; does not select the server used by metro reload.', ), + ...postGestureStabilizationFields(), ...leaseDeviceSpecFields(), }, ); @@ -139,6 +141,7 @@ const openCliSchema = { 'surface', ...METRO_RELOAD_FLAGS, 'launchUrl', + 'postGestureStabilization', ...LEASE_DEVICE_SPEC_FLAGS, ], } as const satisfies CommandSchemaOverride; @@ -173,6 +176,7 @@ const openCliReader: CliReader = (positionals, flags) => ({ metroPort: flags.metroPort, bundleUrl: flags.bundleUrl, launchUrl: flags.launchUrl, + postGestureStabilization: flags.postGestureStabilization, ...leaseDeviceSpecInputFromFlags(flags), }); diff --git a/packages/contracts/src/client-app.ts b/packages/contracts/src/client-app.ts index 9a2ca879d2..24aa401df4 100644 --- a/packages/contracts/src/client-app.ts +++ b/packages/contracts/src/client-app.ts @@ -23,6 +23,7 @@ import type { } from './client-connection.ts'; import type { AgentDeviceSessionDevice, StartupPerfSample } from './client-device-view.ts'; import type { LeaseDeviceSpec } from './client-lease.ts'; +import type { PostGestureStabilizationOptions } from './client-gesture.ts'; export type DeviceSelectionReason = | 'explicit-selector' @@ -72,7 +73,8 @@ export type AppDeployResult = { export type AppOpenOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions & - LeaseDeviceSpec & { + LeaseDeviceSpec & + PostGestureStabilizationOptions & { app?: string; url?: string; surface?: SessionSurface; diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index c053d52158..3ac47f2bd9 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -41,12 +41,13 @@ export type SettleCommandOptions = { timeoutMs?: number; }; -/** For a caller that verifies the screen by polling it: opts out of the checks after a gesture. */ +/** For a caller that verifies the screen by polling it: opts out of the checks after a gesture or an app launch. */ export type PostGestureStabilizationOptions = { /** * Set false to skip the movement check on a directional scroll and the stabilization wait - * before the next read. Set it only when your own polling verifies the screen: the next read can - * return a tree that the gesture is still moving. + * before the next read, or the first-screen wait after an app launch. Set it only when your own + * polling verifies the screen: the next read can return a tree that the gesture is still moving + * or the launch screen. */ postGestureStabilization?: boolean; }; diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index b6010dbc8f..fd0b8cad97 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -186,7 +186,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization), unless the request sets postGestureStabilization false; that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index af9127e4f2..b35c60c5f7 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -158,6 +158,21 @@ test('every Apple open that launched an app marks a first-screen wait, and an An ); }); +test('an Apple open with postGestureStabilization false marks no wait and drops the replaced record', () => { + const session = makeSession('ios'); + session.deferredSurface = { + kind: 'stabilize', + action: 'scroll', + positionals: ['down'], + markedAt: Date.now(), + }; + + markPostOpenStabilization(session, 'observable', { postGestureStabilization: false }); + + assert.equal(session.deferredSurface, undefined); + assert.equal(isPostGestureStabilizationPending(session), false); +}); + test('the first capture after an open waits through a loading tree for the first screen', async () => { vi.useFakeTimers(); const session = makeSession('ios'); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 0b8723602e..9633a0bf63 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -235,16 +235,25 @@ export function movingTargetBaseline( * previous action's targets were. The open launched or foregrounded an app, so both describe a screen * the open replaced: a verdict against them would charge the open's effect to the earlier action. * + * A caller that verifies the screen by polling it opts out with `postGestureStabilization: false`, + * as on a scroll or swipe: a game or canvas app that labels nothing would otherwise hold the next + * capture for the whole budget. The opt-out still clears the replaced screen's record. + * * Apple only: {@link showsFirstScreen} reads the Application and Window roots of an Apple tree, and * an Android tree has no root node to tell them apart. */ export function markPostOpenStabilization( session: SessionState, observation: PostOpenObservation | undefined, + flags?: CommandFlags, ): void { if (!isApplePlatform(session.device.platform)) return; if (!supportsPostGestureStabilization(session.device)) return; if (observation === undefined) return; + if (flags?.postGestureStabilization === false) { + session.deferredSurface = undefined; + return; + } session.deferredSurface = { kind: 'stabilize', action: 'open', diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts index a6d2b9eee0..14016896c7 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts @@ -403,7 +403,7 @@ export async function completeOpenCommand(params: { }); sessionStore.clearCloseEffectsUnconfirmed(nextRef); const nextSession = sessionStore.requireCurrent(nextRef); - markPostOpenStabilization(nextSession, timing.postOpenObservation); + markPostOpenStabilization(nextSession, timing.postOpenObservation, req.flags); if (req.runtime !== undefined) setSessionRuntimeHintsForOpen(sessionStore, sessionName, runtimeHints); applyOrdinaryScriptRecordingOpenOutcome({ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index da4461f90d..bc7c8f320f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -99,7 +99,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; other devices and backends skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree counts as the first screen when, besides the app root, it shows a labeled element that is not text, such as a button, or two different texts, and two reads in a row agree. A splash image, an empty view, a busy indicator, or a single status line such as Expo's "Bundling 91%…" does not count, however long it stays unchanged. An app whose first screen has no labeled element, or a single line of text, such as a game canvas, waits the full 10 seconds. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. +- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; other devices and backends skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree counts as the first screen when, besides the app root, it shows a labeled element that is not text, such as a button, or two different texts, and two reads in a row agree. An unlabeled splash image, an empty view, a busy indicator, or a single status line such as Expo's "Bundling 91%…" does not count, however long it stays unchanged. An app whose first screen is a single line of text, or has no labeled element at all, such as a game canvas, waits the full 10 seconds; to skip the wait, pass `--no-post-gesture-stabilization` to `open` (see [Scroll](#scroll) for the other ways to set it) and poll the screen yourself. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. @@ -566,7 +566,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. - A directional scroll reports what it observed as `movement`: `moved`, `at-edge` (no change and no more content that way), `unchanged` (no change, no edge signal), or `unobserved` (no reliable comparison). If nothing changed, the swipe ran inside a container, and that container still has hidden content in that direction, it refuses with `scroll_no_progress`; the hint names the usual causes: a focused keyboard, a nested scroller, or a list that needs a raw `swipe`. See `agent-device help scroll`. - `--no-post-gesture-stabilization` is for a caller that verifies the screen by polling it, for example a test runner that re-reads the tree every 100ms. It skips the two checks that follow a gesture: the `movement` check above (`scroll ` only), and the wait that makes the first read after the gesture agree across two captures (see [Assertions](#assertions)). The scroll returns right after its gesture with no `movement` field and no `scroll_no_progress` refusal, and the next read is a plain capture without `postGestureOutcome`. Without your own polling, that read can return a tree the gesture is still moving. `swipe` and `gesture swipe` accept it too; other gestures never stabilize, so the CLI refuses the flag on them and the other routes ignore it. None of the checks run on web and desktop targets. With `--settle`, the settle observation already reports the outcome, so the option only removes the wait before settle's first capture. The per-pass checks of `scroll top`, `scroll bottom`, and `scroll --until` stay. -- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. To apply it to every `scroll`, `swipe`, and `gesture swipe` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. +- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. `open` accepts it too and then skips the first-screen wait after an app launch (see [Open apps and links](#open-apps-and-links)). To apply it to every `scroll`, `swipe`, `gesture swipe`, and `open` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. ## Retry after a failed command diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index 48ac21b96d..a41fa6e252 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -92,7 +92,7 @@ explicit config: - Metro endpoint/token fields (`metro*`, `bundleUrl`) - request headers and structured install sources - local code and write destinations (`reporter`, `reportJunit`, `saveScript`, `launchConsole`) -- `postGestureStabilization: false`, which skips the directional scroll `movement` check and the wait that keeps the first read after a scroll or swipe from a tree still in motion; see [Scroll](/docs/commands#scroll) +- `postGestureStabilization: false`, which skips the directional scroll `movement` check and the wait that keeps the first read after a scroll or swipe from a tree still in motion, and the first-screen wait after `open`; see [Scroll](/docs/commands#scroll) Project config can also set `snapshotDepth`, `snapshotScope`, `screenshotScale`, `activity`, `relaunch`, `shutdown`, `fps`, and `quality`. Local paths and executable modules, such as `stepsFile` and `reporter`, are allowed only in user or explicit config. The `record` config key and `AGENT_DEVICE_RECORD` were removed. `--record` is per-invocation only: pass it on the command, or set `record: true` in an SDK/MCP call, to keep a corrective read in a healed script. From 8c29d1ba896dbf3d0f5197ce5a058971735cf9e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 16:52:14 +0100 Subject: [PATCH 2/4] fix(open): clear a pending gesture wait on every platform when the opt-out is set Co-Authored-By: Claude Opus 5.5 --- .../src/flag-definitions-action.ts | 4 +-- packages/contracts/src/client-gesture.ts | 6 ++-- .../deferred-interaction-outcome.test.ts | 28 ++++++++++--------- .../src/core/deferred-interaction-outcome.ts | 9 +++--- website/docs/docs/commands.md | 2 +- website/docs/docs/configuration.md | 2 +- 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index e36333c619..959a6030ba 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -236,9 +236,9 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ // An opt-out for callers that verify by polling: a synopsis would offer it on every scroll and swipe. usageHidden: true, usageDescription: - 'Scroll, swipe, gesture swipe, open: skip the movement check and the stabilization wait that follow a gesture, and the first-screen wait after an app launch; use it only when your own polling verifies the screen', + 'Scroll, swipe, gesture swipe, open: skip the movement check and the stabilization wait that follow a gesture, and on a local iOS simulator the first-screen wait after an app launch; use it only when your own polling verifies the screen', inputDescription: - 'Set false to skip the movement check on a directional scroll and the stabilization wait before the next read, or the first-screen wait after an app launch. Set it only when your own polling verifies the screen: the next read can return a tree that the gesture is still moving or the launch screen.', + 'Set false to skip the movement check on a directional scroll and the stabilization wait before the next read, or, on a local iOS simulator, the first-screen wait after an app launch. Set it only when your own polling verifies the screen: the next read can return a tree that the gesture is still moving or the launch screen.', // A repository must not switch off the check that keeps reads after a gesture from a stale tree. projectConfig: false, // A per-request policy of the caller, not part of the step: a replay keeps the default. diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 3ac47f2bd9..65c7eb9cb9 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -45,9 +45,9 @@ export type SettleCommandOptions = { export type PostGestureStabilizationOptions = { /** * Set false to skip the movement check on a directional scroll and the stabilization wait - * before the next read, or the first-screen wait after an app launch. Set it only when your own - * polling verifies the screen: the next read can return a tree that the gesture is still moving - * or the launch screen. + * before the next read, or, on a local iOS simulator, the first-screen wait after an app launch. + * Set it only when your own polling verifies the screen: the next read can return a tree that the + * gesture is still moving or the launch screen. */ postGestureStabilization?: boolean; }; diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index b35c60c5f7..7034ab4f0b 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -158,19 +158,21 @@ test('every Apple open that launched an app marks a first-screen wait, and an An ); }); -test('an Apple open with postGestureStabilization false marks no wait and drops the replaced record', () => { - const session = makeSession('ios'); - session.deferredSurface = { - kind: 'stabilize', - action: 'scroll', - positionals: ['down'], - markedAt: Date.now(), - }; - - markPostOpenStabilization(session, 'observable', { postGestureStabilization: false }); - - assert.equal(session.deferredSurface, undefined); - assert.equal(isPostGestureStabilizationPending(session), false); +test('an open with postGestureStabilization false marks no wait and drops the replaced record', () => { + for (const platform of ['ios', 'android'] as const) { + const session = makeSession(platform); + session.deferredSurface = { + kind: 'stabilize', + action: 'scroll', + positionals: ['down'], + markedAt: Date.now(), + }; + + markPostOpenStabilization(session, 'observable', { postGestureStabilization: false }); + + assert.equal(session.deferredSurface, undefined, platform); + assert.equal(isPostGestureStabilizationPending(session), false, platform); + } }); test('the first capture after an open waits through a loading tree for the first screen', async () => { diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 9633a0bf63..feb0a207db 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -237,7 +237,8 @@ export function movingTargetBaseline( * * A caller that verifies the screen by polling it opts out with `postGestureStabilization: false`, * as on a scroll or swipe: a game or canvas app that labels nothing would otherwise hold the next - * capture for the whole budget. The opt-out still clears the replaced screen's record. + * capture for the whole budget. On every platform the opt-out also clears the record an earlier + * gesture left pending, so the next capture pays no wait at all. * * Apple only: {@link showsFirstScreen} reads the Application and Window roots of an Apple tree, and * an Android tree has no root node to tell them apart. @@ -247,13 +248,13 @@ export function markPostOpenStabilization( observation: PostOpenObservation | undefined, flags?: CommandFlags, ): void { - if (!isApplePlatform(session.device.platform)) return; - if (!supportsPostGestureStabilization(session.device)) return; - if (observation === undefined) return; if (flags?.postGestureStabilization === false) { session.deferredSurface = undefined; return; } + if (!isApplePlatform(session.device.platform)) return; + if (!supportsPostGestureStabilization(session.device)) return; + if (observation === undefined) return; session.deferredSurface = { kind: 'stabilize', action: 'open', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index bc7c8f320f..8efb7c8119 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -112,7 +112,7 @@ Open apps and links: iOS Simulator launch behavior: -- An `open` that launches an app reports `timing.postOpenObservation`. Whenever it is present, the next command that reads the screen waits for the app's first screen, up to the 10-second polling budget described under `open`, so a `click` lands on the real first screen where its element comes to rest. If the wait runs out, the command proceeds and warns with `postGestureOutcome.kind` `"loading"` (no first screen yet) or `"unsettled"` (a first screen that kept changing). +- An `open` that launches an app reports `timing.postOpenObservation`. Whenever it is present, and unless the `open` passed `--no-post-gesture-stabilization`, the next command that reads the screen waits for the app's first screen, up to the 10-second polling budget described under `open`, so a `click` lands on the real first screen where its element comes to rest. If the wait runs out, the command proceeds and warns with `postGestureOutcome.kind` `"loading"` (no first screen yet) or `"unsettled"` (a first screen that kept changing). - If CoreSimulator never answers a URL hand-off, `open` fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout`. The Simulator may still have received the URL; retry the open or answer the prompt with `alert accept`. - When `open --launch-url ` raises an `Open in ""?` prompt and the URL's scheme belongs to the session app, `open` taps **Open** and returns `launchConfirmation: "accepted"`. If the app still never starts, `open` hands over the URL once more. - On a local iOS Simulator, `open` recognizes the prompt in the Simulator's preferred languages, through the OS's own wording of its title and buttons, so a French Simulator's `Ouvrir dans « » ?` is answered like the English one. `alert accept` and `alert dismiss` answer it by the same wording. If the runner cannot read that wording, the prompt is not recognized. diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index a41fa6e252..9f39351ed0 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -92,7 +92,7 @@ explicit config: - Metro endpoint/token fields (`metro*`, `bundleUrl`) - request headers and structured install sources - local code and write destinations (`reporter`, `reportJunit`, `saveScript`, `launchConsole`) -- `postGestureStabilization: false`, which skips the directional scroll `movement` check and the wait that keeps the first read after a scroll or swipe from a tree still in motion, and the first-screen wait after `open`; see [Scroll](/docs/commands#scroll) +- `postGestureStabilization: false`, which skips the directional scroll `movement` check and the wait that keeps the first read after a scroll or swipe from a tree still in motion, and the first-screen wait after `open` on a local iOS simulator; see [Scroll](/docs/commands#scroll) Project config can also set `snapshotDepth`, `snapshotScope`, `screenshotScale`, `activity`, `relaunch`, `shutdown`, `fps`, and `quality`. Local paths and executable modules, such as `stepsFile` and `reporter`, are allowed only in user or explicit config. The `record` config key and `AGENT_DEVICE_RECORD` were removed. `--record` is per-invocation only: pass it on the command, or set `record: true` in an SDK/MCP call, to keep a corrective read in a healed script. From 9e81f332ee7026d0adcdc4ca71ed5518ab37adc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 16:53:29 +0100 Subject: [PATCH 3/4] docs(open): link the opt-out to the Navigation section Co-Authored-By: Claude Opus 5.5 --- website/docs/docs/commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8efb7c8119..af04009ccd 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -566,7 +566,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. - A directional scroll reports what it observed as `movement`: `moved`, `at-edge` (no change and no more content that way), `unchanged` (no change, no edge signal), or `unobserved` (no reliable comparison). If nothing changed, the swipe ran inside a container, and that container still has hidden content in that direction, it refuses with `scroll_no_progress`; the hint names the usual causes: a focused keyboard, a nested scroller, or a list that needs a raw `swipe`. See `agent-device help scroll`. - `--no-post-gesture-stabilization` is for a caller that verifies the screen by polling it, for example a test runner that re-reads the tree every 100ms. It skips the two checks that follow a gesture: the `movement` check above (`scroll ` only), and the wait that makes the first read after the gesture agree across two captures (see [Assertions](#assertions)). The scroll returns right after its gesture with no `movement` field and no `scroll_no_progress` refusal, and the next read is a plain capture without `postGestureOutcome`. Without your own polling, that read can return a tree the gesture is still moving. `swipe` and `gesture swipe` accept it too; other gestures never stabilize, so the CLI refuses the flag on them and the other routes ignore it. None of the checks run on web and desktop targets. With `--settle`, the settle observation already reports the outcome, so the option only removes the wait before settle's first capture. The per-pass checks of `scroll top`, `scroll bottom`, and `scroll --until` stay. -- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. `open` accepts it too and then skips the first-screen wait after an app launch (see [Open apps and links](#open-apps-and-links)). To apply it to every `scroll`, `swipe`, `gesture swipe`, and `open` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. +- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. `open` accepts it too and then skips the first-screen wait after an app launch (see [Navigation](#navigation)). To apply it to every `scroll`, `swipe`, `gesture swipe`, and `open` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. ## Retry after a failed command From 65294d36affb91422133b945408b8e5a4b22b225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 17:04:08 +0100 Subject: [PATCH 4/4] fix(open): apply the first-screen opt-out only to an open that launched an app Co-Authored-By: Claude Opus 5.5 --- .../__tests__/deferred-interaction-outcome.test.ts | 14 ++++++++++++++ .../src/core/deferred-interaction-outcome.ts | 6 +++--- website/docs/docs/commands.md | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index 7034ab4f0b..8598b4d1ba 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -175,6 +175,20 @@ test('an open with postGestureStabilization false marks no wait and drops the re } }); +test('an open that launched no app keeps the pending record despite the opt-out', () => { + const session = makeSession('ios'); + session.deferredSurface = { + kind: 'stabilize', + action: 'scroll', + positionals: ['down'], + markedAt: Date.now(), + }; + + markPostOpenStabilization(session, undefined, { postGestureStabilization: false }); + + assert.equal(pendingSurfaceStabilization(session)?.action, 'scroll'); +}); + test('the first capture after an open waits through a loading tree for the first screen', async () => { vi.useFakeTimers(); const session = makeSession('ios'); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index feb0a207db..09cda53f68 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -237,8 +237,8 @@ export function movingTargetBaseline( * * A caller that verifies the screen by polling it opts out with `postGestureStabilization: false`, * as on a scroll or swipe: a game or canvas app that labels nothing would otherwise hold the next - * capture for the whole budget. On every platform the opt-out also clears the record an earlier - * gesture left pending, so the next capture pays no wait at all. + * capture for the whole budget. When the open launched an app, on any platform, the opt-out also + * clears the record an earlier action left pending, so the next capture pays no wait at all. * * Apple only: {@link showsFirstScreen} reads the Application and Window roots of an Apple tree, and * an Android tree has no root node to tell them apart. @@ -248,13 +248,13 @@ export function markPostOpenStabilization( observation: PostOpenObservation | undefined, flags?: CommandFlags, ): void { + if (observation === undefined) return; if (flags?.postGestureStabilization === false) { session.deferredSurface = undefined; return; } if (!isApplePlatform(session.device.platform)) return; if (!supportsPostGestureStabilization(session.device)) return; - if (observation === undefined) return; session.deferredSurface = { kind: 'stabilize', action: 'open', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index af04009ccd..ac9778a2ae 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -566,7 +566,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. - A directional scroll reports what it observed as `movement`: `moved`, `at-edge` (no change and no more content that way), `unchanged` (no change, no edge signal), or `unobserved` (no reliable comparison). If nothing changed, the swipe ran inside a container, and that container still has hidden content in that direction, it refuses with `scroll_no_progress`; the hint names the usual causes: a focused keyboard, a nested scroller, or a list that needs a raw `swipe`. See `agent-device help scroll`. - `--no-post-gesture-stabilization` is for a caller that verifies the screen by polling it, for example a test runner that re-reads the tree every 100ms. It skips the two checks that follow a gesture: the `movement` check above (`scroll ` only), and the wait that makes the first read after the gesture agree across two captures (see [Assertions](#assertions)). The scroll returns right after its gesture with no `movement` field and no `scroll_no_progress` refusal, and the next read is a plain capture without `postGestureOutcome`. Without your own polling, that read can return a tree the gesture is still moving. `swipe` and `gesture swipe` accept it too; other gestures never stabilize, so the CLI refuses the flag on them and the other routes ignore it. None of the checks run on web and desktop targets. With `--settle`, the settle observation already reports the outcome, so the option only removes the wait before settle's first capture. The per-pass checks of `scroll top`, `scroll bottom`, and `scroll --until` stay. -- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. `open` accepts it too and then skips the first-screen wait after an app launch (see [Navigation](#navigation)). To apply it to every `scroll`, `swipe`, `gesture swipe`, and `open` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. +- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. `open` accepts it too: when the open launches an app, it skips the first-screen wait and drops any wait an earlier scroll or swipe left pending (see [Navigation](#navigation)). To apply it to every `scroll`, `swipe`, `gesture swipe`, and `open` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. ## Retry after a failed command