diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 0f15125d2e..959a6030ba 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -236,9 +236,9 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ // An opt-out for callers that verify by polling: a synopsis would offer it on every scroll and swipe. usageHidden: true, usageDescription: - 'Scroll, swipe, gesture swipe: skip the movement check and the stabilization wait that follow a gesture; use it only when your own polling verifies the screen', + 'Scroll, swipe, gesture swipe, open: skip the movement check and the stabilization wait that follow a gesture, and on a local iOS simulator the first-screen wait after an app launch; use it only when your own polling verifies the screen', inputDescription: - 'Set false to skip the movement check on a directional scroll and the stabilization wait before the next read. Set it only when your own polling verifies the screen: the next read can return a tree that the gesture is still moving.', + 'Set false to skip the movement check on a directional scroll and the stabilization wait before the next read, or, on a local iOS simulator, the first-screen wait after an app launch. Set it only when your own polling verifies the screen: the next read can return a tree that the gesture is still moving or the launch screen.', // A repository must not switch off the check that keeps reads after a gesture from a stale tree. projectConfig: false, // A per-request policy of the caller, not part of the step: a replay keeps the default. diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index 659b12319c..77bba7d068 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -11,6 +11,7 @@ import { jsonSchemaField, leaseDeviceSpecFields, optionField, + postGestureStabilizationFields, stringArrayField, stringField, stringSchema, @@ -87,6 +88,7 @@ const openCommandMetadata = defineFieldCommandMetadata( launchUrl: stringField( 'Launch URL for the opened app; does not select the server used by metro reload.', ), + ...postGestureStabilizationFields(), ...leaseDeviceSpecFields(), }, ); @@ -139,6 +141,7 @@ const openCliSchema = { 'surface', ...METRO_RELOAD_FLAGS, 'launchUrl', + 'postGestureStabilization', ...LEASE_DEVICE_SPEC_FLAGS, ], } as const satisfies CommandSchemaOverride; @@ -173,6 +176,7 @@ const openCliReader: CliReader = (positionals, flags) => ({ metroPort: flags.metroPort, bundleUrl: flags.bundleUrl, launchUrl: flags.launchUrl, + postGestureStabilization: flags.postGestureStabilization, ...leaseDeviceSpecInputFromFlags(flags), }); diff --git a/packages/contracts/src/client-app.ts b/packages/contracts/src/client-app.ts index 9a2ca879d2..24aa401df4 100644 --- a/packages/contracts/src/client-app.ts +++ b/packages/contracts/src/client-app.ts @@ -23,6 +23,7 @@ import type { } from './client-connection.ts'; import type { AgentDeviceSessionDevice, StartupPerfSample } from './client-device-view.ts'; import type { LeaseDeviceSpec } from './client-lease.ts'; +import type { PostGestureStabilizationOptions } from './client-gesture.ts'; export type DeviceSelectionReason = | 'explicit-selector' @@ -72,7 +73,8 @@ export type AppDeployResult = { export type AppOpenOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions & - LeaseDeviceSpec & { + LeaseDeviceSpec & + PostGestureStabilizationOptions & { app?: string; url?: string; surface?: SessionSurface; diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index c053d52158..65c7eb9cb9 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -41,12 +41,13 @@ export type SettleCommandOptions = { timeoutMs?: number; }; -/** For a caller that verifies the screen by polling it: opts out of the checks after a gesture. */ +/** For a caller that verifies the screen by polling it: opts out of the checks after a gesture or an app launch. */ export type PostGestureStabilizationOptions = { /** * Set false to skip the movement check on a directional scroll and the stabilization wait - * before the next read. Set it only when your own polling verifies the screen: the next read can - * return a tree that the gesture is still moving. + * before the next read, or, on a local iOS simulator, the first-screen wait after an app launch. + * Set it only when your own polling verifies the screen: the next read can return a tree that the + * gesture is still moving or the launch screen. */ postGestureStabilization?: boolean; }; diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index b6010dbc8f..fd0b8cad97 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -186,7 +186,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization), unless the request sets postGestureStabilization false; that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index af9127e4f2..8598b4d1ba 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -158,6 +158,37 @@ test('every Apple open that launched an app marks a first-screen wait, and an An ); }); +test('an open with postGestureStabilization false marks no wait and drops the replaced record', () => { + for (const platform of ['ios', 'android'] as const) { + const session = makeSession(platform); + session.deferredSurface = { + kind: 'stabilize', + action: 'scroll', + positionals: ['down'], + markedAt: Date.now(), + }; + + markPostOpenStabilization(session, 'observable', { postGestureStabilization: false }); + + assert.equal(session.deferredSurface, undefined, platform); + assert.equal(isPostGestureStabilizationPending(session), false, platform); + } +}); + +test('an open that launched no app keeps the pending record despite the opt-out', () => { + const session = makeSession('ios'); + session.deferredSurface = { + kind: 'stabilize', + action: 'scroll', + positionals: ['down'], + markedAt: Date.now(), + }; + + markPostOpenStabilization(session, undefined, { postGestureStabilization: false }); + + assert.equal(pendingSurfaceStabilization(session)?.action, 'scroll'); +}); + test('the first capture after an open waits through a loading tree for the first screen', async () => { vi.useFakeTimers(); const session = makeSession('ios'); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 0b8723602e..09cda53f68 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -235,16 +235,26 @@ export function movingTargetBaseline( * previous action's targets were. The open launched or foregrounded an app, so both describe a screen * the open replaced: a verdict against them would charge the open's effect to the earlier action. * + * A caller that verifies the screen by polling it opts out with `postGestureStabilization: false`, + * as on a scroll or swipe: a game or canvas app that labels nothing would otherwise hold the next + * capture for the whole budget. When the open launched an app, on any platform, the opt-out also + * clears the record an earlier action left pending, so the next capture pays no wait at all. + * * Apple only: {@link showsFirstScreen} reads the Application and Window roots of an Apple tree, and * an Android tree has no root node to tell them apart. */ export function markPostOpenStabilization( session: SessionState, observation: PostOpenObservation | undefined, + flags?: CommandFlags, ): void { + if (observation === undefined) return; + if (flags?.postGestureStabilization === false) { + session.deferredSurface = undefined; + return; + } if (!isApplePlatform(session.device.platform)) return; if (!supportsPostGestureStabilization(session.device)) return; - if (observation === undefined) return; session.deferredSurface = { kind: 'stabilize', action: 'open', diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts index a6d2b9eee0..14016896c7 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-open-execution.ts @@ -403,7 +403,7 @@ export async function completeOpenCommand(params: { }); sessionStore.clearCloseEffectsUnconfirmed(nextRef); const nextSession = sessionStore.requireCurrent(nextRef); - markPostOpenStabilization(nextSession, timing.postOpenObservation); + markPostOpenStabilization(nextSession, timing.postOpenObservation, req.flags); if (req.runtime !== undefined) setSessionRuntimeHintsForOpen(sessionStore, sessionName, runtimeHints); applyOrdinaryScriptRecordingOpenOutcome({ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index da4461f90d..ac9778a2ae 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -99,7 +99,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; other devices and backends skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree counts as the first screen when, besides the app root, it shows a labeled element that is not text, such as a button, or two different texts, and two reads in a row agree. A splash image, an empty view, a busy indicator, or a single status line such as Expo's "Bundling 91%…" does not count, however long it stays unchanged. An app whose first screen has no labeled element, or a single line of text, such as a game canvas, waits the full 10 seconds. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. +- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; other devices and backends skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree counts as the first screen when, besides the app root, it shows a labeled element that is not text, such as a button, or two different texts, and two reads in a row agree. An unlabeled splash image, an empty view, a busy indicator, or a single status line such as Expo's "Bundling 91%…" does not count, however long it stays unchanged. An app whose first screen is a single line of text, or has no labeled element at all, such as a game canvas, waits the full 10 seconds; to skip the wait, pass `--no-post-gesture-stabilization` to `open` (see [Scroll](#scroll) for the other ways to set it) and poll the screen yourself. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. @@ -112,7 +112,7 @@ Open apps and links: iOS Simulator launch behavior: -- An `open` that launches an app reports `timing.postOpenObservation`. Whenever it is present, the next command that reads the screen waits for the app's first screen, up to the 10-second polling budget described under `open`, so a `click` lands on the real first screen where its element comes to rest. If the wait runs out, the command proceeds and warns with `postGestureOutcome.kind` `"loading"` (no first screen yet) or `"unsettled"` (a first screen that kept changing). +- An `open` that launches an app reports `timing.postOpenObservation`. Whenever it is present, and unless the `open` passed `--no-post-gesture-stabilization`, the next command that reads the screen waits for the app's first screen, up to the 10-second polling budget described under `open`, so a `click` lands on the real first screen where its element comes to rest. If the wait runs out, the command proceeds and warns with `postGestureOutcome.kind` `"loading"` (no first screen yet) or `"unsettled"` (a first screen that kept changing). - If CoreSimulator never answers a URL hand-off, `open` fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout`. The Simulator may still have received the URL; retry the open or answer the prompt with `alert accept`. - When `open --launch-url ` raises an `Open in ""?` prompt and the URL's scheme belongs to the session app, `open` taps **Open** and returns `launchConfirmation: "accepted"`. If the app still never starts, `open` hands over the URL once more. - On a local iOS Simulator, `open` recognizes the prompt in the Simulator's preferred languages, through the OS's own wording of its title and buttons, so a French Simulator's `Ouvrir dans « » ?` is answered like the English one. `alert accept` and `alert dismiss` answer it by the same wording. If the runner cannot read that wording, the prompt is not recognized. @@ -566,7 +566,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. - A directional scroll reports what it observed as `movement`: `moved`, `at-edge` (no change and no more content that way), `unchanged` (no change, no edge signal), or `unobserved` (no reliable comparison). If nothing changed, the swipe ran inside a container, and that container still has hidden content in that direction, it refuses with `scroll_no_progress`; the hint names the usual causes: a focused keyboard, a nested scroller, or a list that needs a raw `swipe`. See `agent-device help scroll`. - `--no-post-gesture-stabilization` is for a caller that verifies the screen by polling it, for example a test runner that re-reads the tree every 100ms. It skips the two checks that follow a gesture: the `movement` check above (`scroll ` only), and the wait that makes the first read after the gesture agree across two captures (see [Assertions](#assertions)). The scroll returns right after its gesture with no `movement` field and no `scroll_no_progress` refusal, and the next read is a plain capture without `postGestureOutcome`. Without your own polling, that read can return a tree the gesture is still moving. `swipe` and `gesture swipe` accept it too; other gestures never stabilize, so the CLI refuses the flag on them and the other routes ignore it. None of the checks run on web and desktop targets. With `--settle`, the settle observation already reports the outcome, so the option only removes the wait before settle's first capture. The per-pass checks of `scroll top`, `scroll bottom`, and `scroll --until` stay. -- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. To apply it to every `scroll`, `swipe`, and `gesture swipe` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. +- The opt-out belongs to the caller. Pass `--no-post-gesture-stabilization` on the command, or `postGestureStabilization: false` in the Node client or a batch step. `open` accepts it too: when the open launches an app, it skips the first-screen wait and drops any wait an earlier scroll or swipe left pending (see [Navigation](#navigation)). To apply it to every `scroll`, `swipe`, `gesture swipe`, and `open` command, set `postGestureStabilization` to `false` in your user config, or set `AGENT_DEVICE_POST_GESTURE_STABILIZATION=0`. `./agent-device.json` rejects the key, so a repository cannot turn the check off for whoever runs it, and a recorded `.ad` script does not carry it, so a replay keeps the wait. MCP and AI SDK tools do not offer it; an operator sets the variable or the user config key for the process that serves them. ## Retry after a failed command diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index 48ac21b96d..9f39351ed0 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -92,7 +92,7 @@ explicit config: - Metro endpoint/token fields (`metro*`, `bundleUrl`) - request headers and structured install sources - local code and write destinations (`reporter`, `reportJunit`, `saveScript`, `launchConsole`) -- `postGestureStabilization: false`, which skips the directional scroll `movement` check and the wait that keeps the first read after a scroll or swipe from a tree still in motion; see [Scroll](/docs/commands#scroll) +- `postGestureStabilization: false`, which skips the directional scroll `movement` check and the wait that keeps the first read after a scroll or swipe from a tree still in motion, and the first-screen wait after `open` on a local iOS simulator; see [Scroll](/docs/commands#scroll) Project config can also set `snapshotDepth`, `snapshotScope`, `screenshotScale`, `activity`, `relaunch`, `shutdown`, `fps`, and `quality`. Local paths and executable modules, such as `stepsFile` and `reporter`, are allowed only in user or explicit config. The `record` config key and `AGENT_DEVICE_RECORD` were removed. `--record` is per-invocation only: pass it on the command, or set `record: true` in an SDK/MCP call, to keep a corrective read in a healed script.