From 6f4b568b9748b5c7f2f7e368d25c36a18b2eff6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 21:32:23 +0100 Subject: [PATCH 01/15] fix(open): wait for the launched app's first screen before the first capture The post-open stabilization accepted any quiet pair, so a React Native launch or loading tree (only the Application root) settled at once and open --foreground returned a 1-node snapshot. Every iOS and Android app launch now marks a first-screen wait: a content-free tree never settles, a sparse tree settles after 1.5s unchanged, and a 10s budget ends in a typed postGestureOutcome.kind 'loading' with a warning. Co-Authored-By: Claude Opus 5.5 --- .../src/post-gesture-stability.test.ts | 178 +++++++++++++++++- .../capture-kit/src/post-gesture-stability.ts | 93 ++++++++- packages/client/src/index.ts | 4 + .../src/output/result-serialization.ts | 1 + .../src/flag-definitions-action.ts | 4 +- .../src/commands/capture/runtime/snapshot.ts | 10 +- .../command-surface/src/management/app.ts | 2 +- packages/contracts/src/client-app.ts | 8 +- packages/contracts/src/client-capture.ts | 7 + .../contracts/src/daemon-command-surface.ts | 2 + packages/contracts/src/facades/snapshot.ts | 2 + .../contracts/src/interaction-guarantees.ts | 2 +- packages/contracts/src/snapshot-model.ts | 30 ++- .../deferred-interaction-outcome.test.ts | 104 +++++++--- .../src/core/deferred-interaction-outcome.ts | 18 +- packages/daemon/src/core/session-state.ts | 2 + .../src/execution/selector-capture-runtime.ts | 8 +- packages/daemon/src/server/response-views.ts | 1 + .../session-open-execution-runtime.test.ts | 7 +- packages/selectors/src/absence-observation.ts | 8 +- website/docs/docs/commands.md | 1 + 21 files changed, 441 insertions(+), 51 deletions(-) diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts index 04c0db917d..2e5d0a1fb4 100644 --- a/packages/capture-kit/src/post-gesture-stability.test.ts +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -3,7 +3,10 @@ import { beforeEach, test, vi } from 'vitest'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { ObservationClock } from './observe-until.ts'; import { + formatPostGestureOutcomeWarning, + readFirstScreenContent, runPostGestureStabilityLoop, + type FirstScreenContent, type PostGestureBaseline, type PostGesturePreActionBaseline, type PostGestureStabilityDecision, @@ -20,7 +23,12 @@ beforeEach(() => { vi.mocked(emitDiagnostic).mockClear(); }); -type Surface = Readonly<{ signature: readonly string[]; costMs: number; backend?: string }>; +type Surface = Readonly<{ + signature: readonly string[]; + costMs: number; + backend?: string; + firstScreen?: FirstScreenContent; +}>; // The negative half of these lines is type-level: each @ts-expect-error fails `pnpm typecheck` // the moment the baseline union would accept a post-action reading as pre-action evidence again @@ -93,6 +101,7 @@ function hooksFor( same(baseline.signature, quiet.signature) ? 'unchanged' : 'changed', surfacesIdentical: same, summarizeDivergence: () => ({}), + readFirstScreen: (value) => value.firstScreen ?? { kind: 'measured', contentNodes: 20 }, }; } @@ -218,3 +227,170 @@ test('an effect that lands after a rebase keeps the loop polling until the moved assert.equal(diagnostics('post_gesture_snapshot_baseline_rebased').length, 1); assert.equal(diagnostics('post_gesture_snapshot_stale_accept').length, 0); }); + +// --- first-screen readiness after an app launch --- + +const LAUNCH = { action: 'open', positionals: [], readiness: 'first-screen' as const }; + +function launchSurface(name: string, contentNodes: number, costMs = 50): Surface { + return { + signature: [name, String(contentNodes)], + costMs, + firstScreen: { kind: 'measured', contentNodes }, + }; +} + +test('a launch keeps polling through a quiet loading tree until the first screen appears', async () => { + const clock = fakeClock(); + const loading = launchSurface('loading', 0); + const home = launchSurface('home', 20); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, [...Array.from({ length: 12 }, () => loading), home, home]), + clock, + }); + + assert.equal(outcome.value, home); + assert.equal(outcome.postGestureOutcome, undefined); + assert.ok(clock.now() > 1_500, `a quiet loading tree settled after ${clock.now()}ms`); +}); + +test('a launch that reads a populated first screen returns on its first quiet pair', async () => { + const clock = fakeClock(); + const home = launchSurface('home', 20); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, [home, home]), + clock, + }); + + assert.equal(outcome.value, home); + assert.equal(outcome.postGestureOutcome, undefined); + assert.ok(clock.now() < 500, `settled after ${clock.now()}ms`); +}); + +test('a sparse first screen settles once it stays unchanged, far inside the launch budget', async () => { + const clock = fakeClock(); + const oneButton = launchSurface('one-button', 1); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor( + clock, + Array.from({ length: 20 }, () => oneButton), + ), + clock, + }); + + assert.equal(outcome.value, oneButton); + assert.equal(outcome.postGestureOutcome, undefined); + assert.ok(clock.now() >= 1_500, `a sparse tree settled after only ${clock.now()}ms`); + assert.ok(clock.now() < 2_500, `a sparse tree waited ${clock.now()}ms`); +}); + +test('a sparse tree that keeps changing does not settle on a quiet pair', async () => { + const clock = fakeClock(); + const progress = [10, 10, 40, 40, 70, 70, 100, 100].map((percent) => + launchSurface(`downloading ${percent}%`, 1), + ); + const home = launchSurface('home', 20); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, [...progress, home, home]), + clock, + }); + + assert.equal(outcome.value, home); +}); + +test('a launch whose budget expires on a loading tree reports the app as loading', async () => { + const clock = fakeClock(); + const loading = launchSurface('loading', 0); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor( + clock, + Array.from({ length: 60 }, () => loading), + ), + clock, + }); + + assert.equal(outcome.value, loading); + assert.deepEqual(outcome.postGestureOutcome, { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }); + assert.ok(clock.now() >= 10_000 && clock.now() < 10_500, `gave up after ${clock.now()}ms`); + assert.match(formatPostGestureOutcomeWarning(outcome.postGestureOutcome!), /still loading/); +}); + +test('a launch whose populated tree never quiets reports it unsettled, not loading', async () => { + const clock = fakeClock(); + const frames = Array.from({ length: 60 }, (_, frame) => launchSurface(`frame ${frame}`, 20)); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, frames), + clock, + }); + + assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); +}); + +test('an unverifiable capture settles a launch on its quiet pair instead of waiting as loading', async () => { + const clock = fakeClock(); + const unreadable: Surface = { + signature: ['root'], + costMs: 50, + firstScreen: { kind: 'unverifiable' }, + }; + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, [unreadable, unreadable]), + clock, + }); + + assert.equal(outcome.value, unreadable); + assert.equal(outcome.postGestureOutcome, undefined); +}); + +test('a gesture record never applies the first-screen rule to an empty tree', async () => { + const clock = fakeClock(); + const empty = launchSurface('empty', 0); + + const outcome = await runPostGestureStabilityLoop({ + pending: PENDING, + hooks: hooksFor(clock, [empty, empty]), + clock, + }); + + assert.equal(outcome.value, empty); + assert.equal(outcome.postGestureOutcome, undefined); +}); + +test('first-screen content excludes the Application and Window roots and distrusts sparse reads', () => { + const root = { index: 0, ref: 'e1', type: 'Application' }; + const window = { index: 1, ref: 'e2', type: 'Window', parentIndex: 0 }; + const button = { index: 2, ref: 'e3', type: 'Button', label: 'Continue', parentIndex: 1 }; + + assert.deepEqual(readFirstScreenContent({ nodes: [root, window] }), { + kind: 'measured', + contentNodes: 0, + }); + assert.deepEqual(readFirstScreenContent({ nodes: [root, window, button] }), { + kind: 'measured', + contentNodes: 1, + }); + assert.deepEqual( + readFirstScreenContent({ + nodes: [root], + snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' }, + }), + { kind: 'unverifiable' }, + ); +}); diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 8bde5bb8cd..f1ee0535e2 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -1,6 +1,11 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SnapshotKeyboardBandFact } from '@agent-device/kernel/snapshot'; -import type { PostGestureAction, PostGestureOutcome } from '@agent-device/contracts/snapshot-model'; +import type { + PostGestureAction, + PostGestureOutcome, + SnapshotState, +} from '@agent-device/contracts/snapshot-model'; +import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import { observeUntil, type ObservationClock, type ObservationSchedule } from './observe-until.ts'; /** @@ -41,6 +46,60 @@ const POST_GESTURE_STABILITY_SCHEDULE: ObservationSchedule = { */ const STABILIZATION_DISTRUST_DEADLINE_MS = POST_GESTURE_STABILITY_SCHEDULE.budgetMs + 2_000; +/** + * Cadence and budget for a launched app's first screen. A launch shows a splash or a loading + * surface before its first screen, and a JavaScript bundle can take seconds to arrive, so the + * budget is wider than a gesture's. It bounds when a poll may start; a slow remote capture that + * starts in time is still judged. + */ +const FIRST_SCREEN_SCHEDULE: ObservationSchedule = { + intervalMs: POST_GESTURE_STABILITY_SCHEDULE.intervalMs, + budgetMs: 10_000, + minPolls: 2, +}; + +/** + * At most this many nodes besides the Application and Window roots is a sparse first screen. With + * its root that is the five-node tree the settle loop already calls nearly empty. + */ +const FIRST_SCREEN_SPARSE_CONTENT_NODES = 4; + +/** + * How long a sparse first screen must stay unchanged before it counts as the real first screen. A + * screen with one button is ready after this window; a loading surface that changes within it is + * not. + */ +const FIRST_SCREEN_SPARSE_QUIET_MS = 1_500; + +/** + * What one capture shows of a launched app's first screen: the nodes it holds besides the + * Application and Window roots, or `unverifiable` when the producer reported a sparse read. An + * accessibility failure does not prove an empty UI, so it cannot hold the loop as a loading screen. + */ +export type FirstScreenContent = + | Readonly<{ kind: 'measured'; contentNodes: number }> + | Readonly<{ kind: 'unverifiable' }>; + +export function readFirstScreenContent( + snapshot: Pick, +): FirstScreenContent { + if (snapshot.snapshotQuality?.state === 'sparse') return { kind: 'unverifiable' }; + const contentNodes = snapshot.nodes.filter((node) => !isViewportRootNode(node)).length; + return { kind: 'measured', contentNodes }; +} + +/** + * Whether a quiet pair can stand as a launched app's first screen. A tree with no content is a + * launch or loading surface and never qualifies. A sparse tree qualifies once it has stayed + * unchanged for {@link FIRST_SCREEN_SPARSE_QUIET_MS}. Any other tree qualifies at once. + */ +export function isFirstScreenReady(content: FirstScreenContent, quietForMs: number): boolean { + if (content.kind === 'unverifiable') return true; + if (content.contentNodes === 0) return false; + if (content.contentNodes > FIRST_SCREEN_SPARSE_CONTENT_NODES) return true; + return quietForMs >= FIRST_SCREEN_SPARSE_QUIET_MS; +} + export type BaselineSurfaceEvidence = 'changed' | 'unchanged' | 'ambiguous'; /** @@ -86,6 +145,11 @@ export type PostGestureStabilityPending = { positionals: string[]; /** Absent when the writer stored no baseline (non-Apple, or nothing usable pre-gesture). */ baseline?: PostGesturePreActionBaseline; + /** + * `first-screen`: an app launch marked this record. A quiet pair settles the loop only when + * {@link isFirstScreenReady} accepts it, under {@link FIRST_SCREEN_SCHEDULE}. + */ + readiness?: 'first-screen'; }; export type PostGestureStabilityHooks = { @@ -106,6 +170,8 @@ export type PostGestureStabilityHooks = { surfacesIdentical: (baseline: S, current: S) => boolean; /** Why `surfacesIdentical` said no, for the veto diagnostic. */ summarizeDivergence: (baseline: S, current: S) => Record; + /** What one attempt shows of a launched app's first screen. Read only for a launch record. */ + readFirstScreen: (value: T) => FirstScreenContent; }; export type PostGestureStabilityOutcome = { @@ -129,6 +195,9 @@ export type PostGestureStabilityOutcome = { */ export function formatPostGestureOutcomeWarning({ kind, gesture }: PostGestureOutcome): string { const named = [gesture.action, ...gesture.positionals].join(' ').trim(); + if (kind === 'loading') { + return `The app was still loading after ${named} when this tree was read: it may show a launch or loading screen. Run snapshot -i again, or wait for expected content (wait text ...) before acting.`; + } return kind === 'unsettled' ? `The surface was still changing after ${named} when this tree was read, so it may not match where the surface comes to rest: an element missing from it is not proof of absence.` : `${named} produced no visible change: the tree still matches its pre-gesture state. ` + @@ -193,6 +262,11 @@ export async function runPostGestureStabilityLoop + isFirstScreenReady(hooks.readFirstScreen(value), lastPairAgreed ? now() - quietSinceMs : 0); let surfaceCache: CapturedSurface | undefined; const surfaceOf = (value: T): CapturedSurface => { @@ -204,18 +278,25 @@ export async function runPostGestureStabilityLoop>({ ...(params.initial !== undefined ? { initial: params.initial } : {}), capture: () => hooks.capture(), - schedule: POST_GESTURE_STABILITY_SCHEDULE, + schedule: firstScreen ? FIRST_SCREEN_SCHEDULE : POST_GESTURE_STABILITY_SCHEDULE, ...(clock ? { clock } : {}), verdict: (latest, previousValue) => { attempts += 1; const current = surfaceOf(latest); - if (previousValue === undefined) return { kind: 'continue' }; + if (previousValue === undefined) { + quietSinceMs = now(); + return { kind: 'continue' }; + } const previous = surfaceOf(previousValue); lastPairAgreed = hooks.signaturesStable(previous.signature, current.signature); - if (!lastPairAgreed) return { kind: 'continue' }; + if (!lastPairAgreed) { + quietSinceMs = now(); + return { kind: 'continue' }; + } const elapsedMs = now() - startedAt; + if (firstScreen && !firstScreenReady(latest)) return { kind: 'continue' }; // A capture plan may fall back or be pre-empted by the XCTest-channel penalty at any time, // so the serving surface can change mid-poll. This quiet pair says nothing about the // gesture: adopt it as a post-action baseline and keep going (#1569). The pair that rebases @@ -273,6 +354,10 @@ export async function runPostGestureStabilityLoop; diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index bb7434b860..cb1922bc69 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -11,6 +11,8 @@ export { isSnapshotBackend, usesMobileSnapshotPresentation, inheritPostGestureOutcome, + isPostGestureRestUnproven, + readPostGestureOutcome, snapshotRefMinting, buildSnapshotNodeMap, findSnapshotAncestor, diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 30dc392063..7c87b3519c 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -186,7 +186,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS Simulator open that did not read the launched app (any postOpenObservation other than observable) marks the next capture in that session to wait for a quiet surface (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS or Android open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; diff --git a/packages/contracts/src/snapshot-model.ts b/packages/contracts/src/snapshot-model.ts index 3af3071940..e585e17a10 100644 --- a/packages/contracts/src/snapshot-model.ts +++ b/packages/contracts/src/snapshot-model.ts @@ -147,12 +147,40 @@ export type PostGestureAction = { action: string; positionals: string[] }; /** * `unsettled`: the surface was still changing when the stabilization deadline expired. * `no-effect`: the settled surface still matches the pre-gesture tree (#1600). + * `loading`: an app launch's budget expired before its tree showed a first screen; the tree may be + * a launch or loading surface. */ export type PostGestureOutcome = { - kind: 'unsettled' | 'no-effect'; + kind: 'unsettled' | 'no-effect' | 'loading'; gesture: PostGestureAction; }; +const POST_GESTURE_OUTCOME_KINDS: ReadonlySet = new Set([ + 'unsettled', + 'no-effect', + 'loading', +]); + +/** Reads a published post-gesture outcome off the wire; anything malformed is no outcome. */ +export function readPostGestureOutcome(value: unknown): PostGestureOutcome | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const { kind, gesture } = value as Record; + if (typeof kind !== 'string' || !POST_GESTURE_OUTCOME_KINDS.has(kind)) return undefined; + if (typeof gesture !== 'object' || gesture === null) return undefined; + const { action, positionals } = gesture as Record; + if (typeof action !== 'string' || !Array.isArray(positionals)) return undefined; + if (!positionals.every((positional) => typeof positional === 'string')) return undefined; + return { kind: kind as PostGestureOutcome['kind'], gesture: { action, positionals } }; +} + +/** + * Whether a tree with this outcome may not show the surface at rest: it was still changing, or the + * launched app was still loading. Such a tree proves no absence and serves no cache. + */ +export function isPostGestureRestUnproven(outcome: PostGestureOutcome | undefined): boolean { + return outcome?.kind === 'unsettled' || outcome?.kind === 'loading'; +} + /** * A capture taken at once to recover or widen `previous` reads the same moment after the same * gesture, so it carries that capture's outcome. diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index 6fc31fe45b..303ba1fb34 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -10,12 +10,34 @@ import { type DeferredOutcomeSnapshotAttempt, } from '../deferred-interaction-outcome.ts'; import type { SessionState } from '../session-state.ts'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { deliverySnapshot, makeSession, pickupSnapshot, } from './post-gesture-stabilization-fixtures.ts'; +/** A first screen with more than a sparse tree's content: a header and five rows. */ +function homeScreenSnapshot(y = 500) { + return makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Home', + rect: { x: 20, y: 60, width: 200, height: 30 }, + }, + ...Array.from({ length: 5 }, (_, row) => ({ + index: row + 2, + parentIndex: 0, + type: 'Button', + label: `Row ${row}`, + rect: { x: 20, y: y + row * 50, width: 300, height: 44 }, + })), + ]); +} + afterEach(() => { vi.useRealTimers(); }); @@ -110,26 +132,62 @@ test('an Apple open that could not observe its app marks stabilization without a assert.equal(pendingSurfaceStabilization(session)?.baseline?.signature, undefined); }); -test('every open verdict other than observable marks stabilization', () => { - const pending = ( - [ - 'unobservable', - 'probe-failed', - 'app-unidentified', - 'not-eligible', - 'observable', - undefined, - ] as const - ).map((observation) => { - const session = makeSession('ios'); - markPostOpenStabilization(session, observation); - return isPostGestureStabilizationPending(session); - }); - const android = makeSession('android'); - markPostOpenStabilization(android, 'probe-failed'); +test('every open that launched an app marks a first-screen wait, on iOS and Android alike', () => { + const readiness = (platform: 'ios' | 'android') => + ( + [ + 'unobservable', + 'probe-failed', + 'app-unidentified', + 'not-eligible', + 'observable', + undefined, + ] as const + ).map((observation) => { + const session = makeSession(platform); + markPostOpenStabilization(session, observation); + return pendingSurfaceStabilization(session)?.readiness; + }); + const launched = Array.from({ length: 5 }, () => 'first-screen'); + + assert.deepEqual(readiness('ios'), [...launched, undefined]); + assert.deepEqual(readiness('android'), [...launched, undefined]); +}); + +test('the first capture after an open waits through a loading tree for the first screen', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const loading = makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ]); + const home = homeScreenSnapshot(); + const { capture } = scriptedCapture([loading, loading, loading, loading, loading, home, home]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(3_000); + const result = await pendingResult; - assert.deepEqual(pending, [true, true, true, true, false, false]); - assert.equal(isPostGestureStabilizationPending(android), false); + assert.equal(result?.snapshot, home); + assert.equal(result?.snapshot.postGestureOutcome, undefined); + assert.equal(isPostGestureStabilizationPending(session), false); +}); + +test('the first capture after an open marks a tree that never showed a first screen as loading', async () => { + vi.useFakeTimers(); + const session = makeSession('android'); + markPostOpenStabilization(session, 'observable'); + const { capture } = scriptedCapture([makeSnapshotState([])]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(11_000); + const result = await pendingResult; + + assert.equal(result?.snapshot.nodes.length, 0); + assert.deepEqual(result?.snapshot.postGestureOutcome, { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }); }); test('the first capture after an unobserved open reads the list where it came to rest', async () => { @@ -137,9 +195,9 @@ test('the first capture after an unobserved open reads the list where it came to const session = makeSession('ios'); markPostOpenStabilization(session, 'unobservable'); const { capture, calls } = scriptedCapture([ - pickupSnapshot(319), - pickupSnapshot(406), - pickupSnapshot(406), + homeScreenSnapshot(319), + homeScreenSnapshot(406), + homeScreenSnapshot(406), ]); const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); @@ -149,7 +207,7 @@ test('the first capture after an unobserved open reads the list where it came to const result = await pendingResult; assert.equal(calls(), 3); - assert.equal(result?.snapshot.nodes[1]?.rect?.y, 406); + assert.equal(result?.snapshot.nodes[2]?.rect?.y, 406); assert.equal(result?.snapshot.postGestureOutcome, undefined); assert.equal(isPostGestureStabilizationPending(session), false); }); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index b286dde98b..0adb627ed6 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -23,6 +23,7 @@ import { snapshotSurfaceComparisonKey, } from './interaction-surface-signature.ts'; import { + readFirstScreenContent, runPostGestureStabilityLoop, type PostGestureStabilityOutcome, } from '@agent-device/capture-kit/post-gesture-stability'; @@ -222,25 +223,24 @@ export function movingTargetBaseline( } /** - * Called once an app open has published its session. A verdict other than `observable` means the - * open never read the launched app: the probe failed or could not identify it, the device has no - * observation path, or the app stayed unreadable. Nothing proved its layout at rest, so the next - * capture runs the quiet-window loop (#3354, #3367). An open that reports no verdict does not mark. - * No baseline: the pre-open tree is a different surface, so the open is never reported as having no - * effect. + * Called once an app open has published its session. Every verdict means the open launched an app, + * and none proves its first screen: an `observable` app can still show its launch or loading + * surface. The next capture therefore waits for the first screen (#3354, #3367). An open that + * reports no verdict launched no app, such as a URL or deep link, and does not mark. No baseline: the + * pre-open tree is a different surface, so the open is never reported as having no effect. */ export function markPostOpenStabilization( session: SessionState, observation: PostOpenObservation | undefined, ): void { - if (!isApplePlatform(session.device.platform)) return; if (!supportsPostGestureStabilization(session.device)) return; - if (observation === undefined || observation === 'observable') return; + if (observation === undefined) return; session.deferredSurface = { kind: 'stabilize', action: 'open', positionals: [], markedAt: Date.now(), + readiness: 'first-screen', }; } @@ -366,6 +366,7 @@ export async function capturePostGestureStabilizedResult(params: { action: pending.action, positionals: pending.positionals ?? [], baseline: pending.baseline, + ...(pending.readiness ? { readiness: pending.readiness } : {}), }, initial: params.initial, hooks: { @@ -382,6 +383,7 @@ export async function capturePostGestureStabilizedResult(params: { classifyBaselineEvidence: classifyPostGestureBaselineEvidence, surfacesIdentical: haveIdenticalDiscriminatingSurfaces, summarizeDivergence: summarizeDiscriminatingSurfaceDivergence, + readFirstScreen: (value) => readFirstScreenContent(readSnapshot(value)), }, }); } finally { diff --git a/packages/daemon/src/core/session-state.ts b/packages/daemon/src/core/session-state.ts index 439ddaf3d4..b93b1cded9 100644 --- a/packages/daemon/src/core/session-state.ts +++ b/packages/daemon/src/core/session-state.ts @@ -107,6 +107,8 @@ export type DeferredSurface = * marker carries no baseline at all. */ baseline?: PostGesturePreActionBaseline; + /** `first-screen`: an app launch marked this record; the loop waits for its first screen. */ + readiness?: 'first-screen'; } | { kind: 'recheck'; diff --git a/packages/daemon/src/execution/selector-capture-runtime.ts b/packages/daemon/src/execution/selector-capture-runtime.ts index 51da158f5a..550e57a78d 100644 --- a/packages/daemon/src/execution/selector-capture-runtime.ts +++ b/packages/daemon/src/execution/selector-capture-runtime.ts @@ -1,7 +1,11 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { BackendSnapshotResult } from '@agent-device/contracts/backend'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; -import { inheritPostGestureOutcome, type SnapshotState } from '@agent-device/contracts/snapshot'; +import { + inheritPostGestureOutcome, + isPostGestureRestUnproven, + type SnapshotState, +} from '@agent-device/contracts/snapshot'; import { snapshotPresentationOptionsFromFlags } from '@agent-device/capture-kit/snapshot-state'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import type { DaemonRequest } from '../core/daemon-request.ts'; @@ -279,7 +283,7 @@ function canUseSessionSnapshotCache( if (request.cache?.useSessionSnapshot !== true) return false; if (getActiveAndroidSnapshotFreshness(session)) return false; if (shouldBypassForPostGestureStabilization(session, request)) return false; - return session.snapshot?.postGestureOutcome?.kind !== 'unsettled'; + return !isPostGestureRestUnproven(session.snapshot?.postGestureOutcome); } function isFreshSelectorSnapshot(snapshot: SnapshotState, timestamp: number): boolean { diff --git a/packages/daemon/src/server/response-views.ts b/packages/daemon/src/server/response-views.ts index 117b818b19..7ba6660842 100644 --- a/packages/daemon/src/server/response-views.ts +++ b/packages/daemon/src/server/response-views.ts @@ -35,6 +35,7 @@ function snapshotView(data: DaemonResponseData, level: ResponseLevel): DaemonRes 'visibility', 'viewport', 'presentedSurface', + 'postGestureOutcome', 'snapshotQuality', 'snapshotQualityNotes', 'targetActivation', diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts b/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts index a4ca574f3a..1e219b5116 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts @@ -932,8 +932,11 @@ test('a fresh iOS open whose observation probe failed leaves the next capture to expect(pendingSurfaceStabilization(session)?.action).toBe('open'); }); -test('a fresh iOS open that observed its app leaves nothing pending', async () => { +test('a fresh iOS open that observed its app still leaves the next capture to wait for its first screen', async () => { const session = await openReportingObservation('observed-open', 'observable'); - expect(pendingSurfaceStabilization(session)).toBeUndefined(); + expect(pendingSurfaceStabilization(session)).toMatchObject({ + action: 'open', + readiness: 'first-screen', + }); }); diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index fbccf713bf..6e346261b5 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -1,4 +1,8 @@ -import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; +import { + extractNodeText, + isPostGestureRestUnproven, + normalizeType, +} from '@agent-device/contracts/snapshot'; import { readNodeLocalIdentity, TARGET_ANNOTATION_MAX_FIELD_BYTES, @@ -98,7 +102,7 @@ export function classifyAbsenceObservation( } if (matchCount === 0) { return { - kind: snapshot.postGestureOutcome?.kind === 'unsettled' ? 'unsettled' : 'absent', + kind: isPostGestureRestUnproven(snapshot.postGestureOutcome) ? 'unsettled' : 'absent', matches: 0, }; } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a93e539340..435e85b8eb 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -98,6 +98,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. +- iOS and Android: after `open` launches an app, the first capture in the session waits up to 10 seconds for the app's first screen. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count. A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. From 0d6b36de5a3798bef66f603307e3bb70916d5d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 21:40:54 +0100 Subject: [PATCH 02/15] refactor(capture-kit): keep the first-screen readiness rule module-private Co-Authored-By: Claude Opus 5.5 --- packages/capture-kit/src/post-gesture-stability.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index f1ee0535e2..3c0720acd1 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -93,7 +93,7 @@ export function readFirstScreenContent( * launch or loading surface and never qualifies. A sparse tree qualifies once it has stayed * unchanged for {@link FIRST_SCREEN_SPARSE_QUIET_MS}. Any other tree qualifies at once. */ -export function isFirstScreenReady(content: FirstScreenContent, quietForMs: number): boolean { +function isFirstScreenReady(content: FirstScreenContent, quietForMs: number): boolean { if (content.kind === 'unverifiable') return true; if (content.contentNodes === 0) return false; if (content.contentNodes > FIRST_SCREEN_SPARSE_CONTENT_NODES) return true; From 61c230a081e5a8ca0078153664fd703ffa457389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 22:06:24 +0100 Subject: [PATCH 03/15] fix(open): keep the first-screen wait on iOS and settle fixtures on a first screen Restore the Apple-only gate on markPostOpenStabilization: Android trees carry no Application/Window root, so the content count cannot tell a launch surface from a first screen there. Move the content count into the daemon so capture-kit's import closure stays unchanged. Classify a miss on a loading tree as its own absence observation, tie the outcome-kind table to the union, state the sparse-read exception and the polling budget in help and docs, and give the post-open reads in the provider and contract scenarios a populated first screen. Co-Authored-By: Claude Opus 5.5 --- .../src/post-gesture-stability.test.ts | 23 --------- .../capture-kit/src/post-gesture-stability.ts | 15 +----- .../src/output/snapshot-serialization.test.ts | 26 ++++++++++ .../src/flag-definitions-action.ts | 2 +- .../commands/capture/runtime/snapshot.test.ts | 33 ++++++++++++ .../command-surface/src/management/app.ts | 2 +- packages/contracts/src/client-app.ts | 9 ++-- .../contracts/src/interaction-guarantees.ts | 2 +- packages/contracts/src/snapshot-model.test.ts | 14 +++++ packages/contracts/src/snapshot-model.ts | 13 ++--- .../deferred-interaction-outcome.test.ts | 51 +++++++++++++++++-- .../src/core/deferred-interaction-outcome.ts | 23 +++++++-- .../server/__tests__/response-views.test.ts | 9 ++++ .../selectors/src/absence-observation.test.ts | 19 +++++++ packages/selectors/src/absence-observation.ts | 17 +++---- .../interaction-contract/daemon-harness.ts | 15 ++---- .../provider-scenarios/device-pin.test.ts | 10 ++-- .../provider-scenarios/fixtures.ts | 21 ++++++++ ...teraction-direct-selector-fallback.test.ts | 12 +++-- .../interaction-verify.test.ts | 9 ++-- .../ios-wait-selector.test.ts | 11 +++- .../provider-scenarios/ios-world.ts | 19 +++++++ .../press-selector-resolution.test.ts | 6 +-- .../remote-proxy-parity.test.ts | 5 +- .../scroll-movement-observation.test.ts | 12 +++-- .../settle-observation.test.ts | 23 ++++++--- .../stale-ref-warning.test.ts | 19 ++++--- .../provider-scenarios/versioned-refs.test.ts | 14 ++--- website/docs/docs/commands.md | 5 +- 29 files changed, 319 insertions(+), 120 deletions(-) diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts index 2e5d0a1fb4..57a136db40 100644 --- a/packages/capture-kit/src/post-gesture-stability.test.ts +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -4,7 +4,6 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { ObservationClock } from './observe-until.ts'; import { formatPostGestureOutcomeWarning, - readFirstScreenContent, runPostGestureStabilityLoop, type FirstScreenContent, type PostGestureBaseline, @@ -372,25 +371,3 @@ test('a gesture record never applies the first-screen rule to an empty tree', as assert.equal(outcome.value, empty); assert.equal(outcome.postGestureOutcome, undefined); }); - -test('first-screen content excludes the Application and Window roots and distrusts sparse reads', () => { - const root = { index: 0, ref: 'e1', type: 'Application' }; - const window = { index: 1, ref: 'e2', type: 'Window', parentIndex: 0 }; - const button = { index: 2, ref: 'e3', type: 'Button', label: 'Continue', parentIndex: 1 }; - - assert.deepEqual(readFirstScreenContent({ nodes: [root, window] }), { - kind: 'measured', - contentNodes: 0, - }); - assert.deepEqual(readFirstScreenContent({ nodes: [root, window, button] }), { - kind: 'measured', - contentNodes: 1, - }); - assert.deepEqual( - readFirstScreenContent({ - nodes: [root], - snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' }, - }), - { kind: 'unverifiable' }, - ); -}); diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 3c0720acd1..9d1df592d3 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -1,11 +1,6 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SnapshotKeyboardBandFact } from '@agent-device/kernel/snapshot'; -import type { - PostGestureAction, - PostGestureOutcome, - SnapshotState, -} from '@agent-device/contracts/snapshot-model'; -import { isViewportRootNode } from '@agent-device/contracts/snapshot'; +import type { PostGestureAction, PostGestureOutcome } from '@agent-device/contracts/snapshot-model'; import { observeUntil, type ObservationClock, type ObservationSchedule } from './observe-until.ts'; /** @@ -80,14 +75,6 @@ export type FirstScreenContent = | Readonly<{ kind: 'measured'; contentNodes: number }> | Readonly<{ kind: 'unverifiable' }>; -export function readFirstScreenContent( - snapshot: Pick, -): FirstScreenContent { - if (snapshot.snapshotQuality?.state === 'sparse') return { kind: 'unverifiable' }; - const contentNodes = snapshot.nodes.filter((node) => !isViewportRootNode(node)).length; - return { kind: 'measured', contentNodes }; -} - /** * Whether a quiet pair can stand as a launched app's first screen. A tree with no content is a * launch or loading surface and never qualifies. A sparse tree qualifies once it has stayed diff --git a/packages/command-output/src/output/snapshot-serialization.test.ts b/packages/command-output/src/output/snapshot-serialization.test.ts index a85614475c..d4d8b61e76 100644 --- a/packages/command-output/src/output/snapshot-serialization.test.ts +++ b/packages/command-output/src/output/snapshot-serialization.test.ts @@ -1,5 +1,6 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; +import type { PostGestureOutcome } from '@agent-device/contracts/snapshot-model'; import { serializeSnapshotResult } from './result-serialization.ts'; import { snapshotViewportSizeFrom } from '@agent-device/kernel/rect'; @@ -144,6 +145,31 @@ test('serializeSnapshotResult omits a presented surface the producer did not rea assert.equal('presentedSurface' in data, false); }); +test('serializeSnapshotResult publishes the post-gesture outcome beside the tree', () => { + const postGestureOutcome: PostGestureOutcome = { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }; + const data = serializeSnapshotResult({ + nodes: [], + truncated: false, + postGestureOutcome, + identifiers: { session: 'qa' }, + }); + + assert.deepEqual(data.postGestureOutcome, postGestureOutcome); +}); + +test('serializeSnapshotResult omits the post-gesture outcome of a tree read at rest', () => { + const data = serializeSnapshotResult({ + nodes: [], + truncated: false, + identifiers: { session: 'qa' }, + }); + + assert.equal('postGestureOutcome' in data, false); +}); + test('serializeSnapshotResult retains clipping notes with or without a capture verdict', () => { const snapshotQualityNotes = [ { code: 'regular-geometry-clipped' as const, clippedNodeCount: 1, droppedNodeCount: 1 }, diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 24b3433b32..ff3bbdbeeb 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -413,7 +413,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageDescription: 'open: keep normal app/device selection and return an initial snapshot of the first screen; without an app, resolve the sole running app on the sole booted iOS simulator', inputDescription: - 'Include an initial interactive snapshot in a fresh open response. On iOS and Android, a launched app\'s snapshot waits up to 10s for its first screen; when it is still loading, the snapshot carries postGestureOutcome.kind "loading" and a warning. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', + 'Include an initial interactive snapshot in a fresh open response. On iOS, a launched app\'s snapshot polls for its first screen for up to 10s (a capture started before then still completes and is judged); when it is still loading, the snapshot carries postGestureOutcome.kind "loading" and a warning. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', projectConfig: false, recorded: false, }, diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts index a2320dc896..0b8b362742 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts @@ -851,6 +851,39 @@ test('runtime snapshot warns when its tree was read on a surface still moving af const result = await device.capture.snapshot({ session: 'default' }); assert.deepEqual(result.warnings, [formatPostGestureOutcomeWarning(outcome)]); + assert.deepEqual(result.postGestureOutcome, outcome); +}); + +test('runtime snapshot publishes a launched app that was still loading as its outcome', async () => { + const outcome: PostGestureOutcome = { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }; + const device = createSnapshotOnlyDevice({ + snapshot: { + ...makeSnapshotState([{ index: 0, depth: 0, type: 'Application', label: 'Demo' }], { + backend: 'xctest', + }), + postGestureOutcome: outcome, + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.deepEqual(result.postGestureOutcome, outcome); + assert.deepEqual(result.warnings, [formatPostGestureOutcomeWarning(outcome)]); +}); + +test('runtime snapshot publishes no outcome for a tree read at rest', async () => { + const device = createSnapshotOnlyDevice({ + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'xctest', + }), + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal('postGestureOutcome' in result, false); }); test('runtime snapshot warns when a gesture produced no visible change', async () => { diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index d6e00c03af..051c3f5870 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -43,7 +43,7 @@ const appsCommandMetadata = defineFieldCommandMetadata( const openCommandMetadata = defineFieldCommandMetadata( 'open', - 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On iOS and Android, the first capture after an app launch, such as the foreground initial snapshot, waits up to 10s for the first screen: a tree with only the app root never qualifies, and a tree with at most four other nodes qualifies after 1.5s unchanged. If the budget runs out, the capture returns what it read with postGestureOutcome.kind "loading" and a warning. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', + 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On iOS, the first capture after an app launch, such as the foreground initial snapshot, polls for the first screen for up to 10s; a capture started before then still completes and is judged. A tree with only the app root never qualifies, and a tree with at most four other nodes qualifies after 1.5s unchanged; a capture the producer marks sparse qualifies when two reads agree, whatever it holds. If the budget runs out, the capture returns what it read with postGestureOutcome.kind "loading" and a warning. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', { app: stringField('App name, bundle id, package, or URL.'), url: stringField('Optional URL passed with an app shell.'), diff --git a/packages/contracts/src/client-app.ts b/packages/contracts/src/client-app.ts index 2e48b1b839..e6bbe36ad7 100644 --- a/packages/contracts/src/client-app.ts +++ b/packages/contracts/src/client-app.ts @@ -90,10 +90,11 @@ export type AppOpenOptions = AgentDeviceRequestOverrides & // `--foreground` FlagDefinition's `inputDescription`) verbatim, pinned to it by // `commands/command-input-option-field.test.ts`. /** - * Include an initial interactive snapshot in a fresh open response. On iOS - * and Android, a launched app's snapshot waits up to 10s for its first screen; - * when it is still loading, the snapshot carries postGestureOutcome.kind - * "loading" and a warning. With no app argument, discover the sole running + * Include an initial interactive snapshot in a fresh open response. On iOS, + * a launched app's snapshot polls for its first screen for up to 10s (a + * capture started before then still completes and is judged); when it is + * still loading, the snapshot carries postGestureOutcome.kind "loading" and a + * warning. With no app argument, discover the sole running * app on the sole booted iOS simulator; ambiguous environments fail closed. */ foreground?: boolean; diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 7c87b3519c..b6010dbc8f 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -186,7 +186,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS or Android open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; diff --git a/packages/contracts/src/snapshot-model.test.ts b/packages/contracts/src/snapshot-model.test.ts index bb210d6e6a..b439308ec6 100644 --- a/packages/contracts/src/snapshot-model.test.ts +++ b/packages/contracts/src/snapshot-model.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest'; import { + readPostGestureOutcome, snapshotStateProvenance, type SnapshotProvenance, type SnapshotStateProvenance, @@ -31,3 +32,16 @@ test('snapshotStateProvenance extracts exactly the pair', () => { producer: 'revyl-ios-tree', }); }); + +test('readPostGestureOutcome reads every outcome kind off the wire and drops malformed values', () => { + const gesture = { action: 'open', positionals: [] }; + for (const kind of ['unsettled', 'no-effect', 'loading'] as const) { + expect(readPostGestureOutcome({ kind, gesture })).toEqual({ kind, gesture }); + } + expect(readPostGestureOutcome({ kind: 'toString', gesture })).toBeUndefined(); + expect(readPostGestureOutcome({ kind: 'loading', gesture: { action: 'open' } })).toBeUndefined(); + expect( + readPostGestureOutcome({ kind: 'loading', gesture: { action: 'open', positionals: [1] } }), + ).toBeUndefined(); + expect(readPostGestureOutcome('loading')).toBeUndefined(); +}); diff --git a/packages/contracts/src/snapshot-model.ts b/packages/contracts/src/snapshot-model.ts index e585e17a10..e472018cca 100644 --- a/packages/contracts/src/snapshot-model.ts +++ b/packages/contracts/src/snapshot-model.ts @@ -155,17 +155,18 @@ export type PostGestureOutcome = { gesture: PostGestureAction; }; -const POST_GESTURE_OUTCOME_KINDS: ReadonlySet = new Set([ - 'unsettled', - 'no-effect', - 'loading', -]); +const POST_GESTURE_OUTCOME_KINDS: Readonly> = { + unsettled: true, + 'no-effect': true, + loading: true, +}; /** Reads a published post-gesture outcome off the wire; anything malformed is no outcome. */ export function readPostGestureOutcome(value: unknown): PostGestureOutcome | undefined { if (typeof value !== 'object' || value === null) return undefined; const { kind, gesture } = value as Record; - if (typeof kind !== 'string' || !POST_GESTURE_OUTCOME_KINDS.has(kind)) return undefined; + if (typeof kind !== 'string' || !Object.hasOwn(POST_GESTURE_OUTCOME_KINDS, kind)) + return undefined; if (typeof gesture !== 'object' || gesture === null) return undefined; const { action, positionals } = gesture as Record; if (typeof action !== 'string' || !Array.isArray(positionals)) return undefined; diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index 303ba1fb34..de33b1ba0d 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -132,7 +132,7 @@ test('an Apple open that could not observe its app marks stabilization without a assert.equal(pendingSurfaceStabilization(session)?.baseline?.signature, undefined); }); -test('every open that launched an app marks a first-screen wait, on iOS and Android alike', () => { +test('every Apple open that launched an app marks a first-screen wait, and an Android open none', () => { const readiness = (platform: 'ios' | 'android') => ( [ @@ -151,7 +151,10 @@ test('every open that launched an app marks a first-screen wait, on iOS and Andr const launched = Array.from({ length: 5 }, () => 'first-screen'); assert.deepEqual(readiness('ios'), [...launched, undefined]); - assert.deepEqual(readiness('android'), [...launched, undefined]); + assert.deepEqual( + readiness('android'), + Array.from({ length: 6 }, () => undefined), + ); }); test('the first capture after an open waits through a loading tree for the first screen', async () => { @@ -173,9 +176,51 @@ test('the first capture after an open waits through a loading tree for the first assert.equal(isPostGestureStabilizationPending(session), false); }); +test('the first capture after an open does not count the Window root as first-screen content', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const rootsOnly = makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { index: 1, parentIndex: 0, type: 'Window', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ]); + const { capture } = scriptedCapture([rootsOnly]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(11_000); + const result = await pendingResult; + + assert.equal(result?.snapshot.postGestureOutcome?.kind, 'loading'); +}); + +test('the first capture after an open settles a producer-marked sparse read on its quiet pair', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const sparse = { + ...makeSnapshotState([ + { + index: 0, + type: 'Application', + label: 'App', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ]), + snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' } as const, + }; + const { capture, calls } = scriptedCapture([sparse, sparse]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pendingResult; + + assert.equal(calls(), 2); + assert.equal(result?.snapshot.postGestureOutcome, undefined); +}); + test('the first capture after an open marks a tree that never showed a first screen as loading', async () => { vi.useFakeTimers(); - const session = makeSession('android'); + const session = makeSession('ios'); markPostOpenStabilization(session, 'observable'); const { capture } = scriptedCapture([makeSnapshotState([])]); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 0adb627ed6..8b36b73642 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -23,10 +23,11 @@ import { snapshotSurfaceComparisonKey, } from './interaction-surface-signature.ts'; import { - readFirstScreenContent, runPostGestureStabilityLoop, + type FirstScreenContent, type PostGestureStabilityOutcome, } from '@agent-device/capture-kit/post-gesture-stability'; +import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import type { DeferredSurface, SessionState } from './session-state.ts'; /** @@ -223,16 +224,20 @@ export function movingTargetBaseline( } /** - * Called once an app open has published its session. Every verdict means the open launched an app, - * and none proves its first screen: an `observable` app can still show its launch or loading + * Called once an Apple app open has published its session. Every verdict means the open launched an + * app, and none proves its first screen: an `observable` app can still show its launch or loading * surface. The next capture therefore waits for the first screen (#3354, #3367). An open that * reports no verdict launched no app, such as a URL or deep link, and does not mark. No baseline: the * pre-open tree is a different surface, so the open is never reported as having no effect. + * + * Apple only: {@link readFirstScreenContent} tells the first screen from a launch surface by the + * Application and Window roots, and an Android tree has no root node to tell them apart. */ export function markPostOpenStabilization( session: SessionState, observation: PostOpenObservation | undefined, ): void { + if (!isApplePlatform(session.device.platform)) return; if (!supportsPostGestureStabilization(session.device)) return; if (observation === undefined) return; session.deferredSurface = { @@ -391,6 +396,18 @@ export async function capturePostGestureStabilizedResult(params: { } } +/** + * What one capture shows of a launched app's first screen: the nodes it holds besides the + * Application and Window roots, or `unverifiable` when the producer reported a sparse read. + */ +function readFirstScreenContent( + snapshot: Pick, +): FirstScreenContent { + if (snapshot.snapshotQuality?.state === 'sparse') return { kind: 'unverifiable' }; + const contentNodes = snapshot.nodes.filter((node) => !isViewportRootNode(node)).length; + return { kind: 'measured', contentNodes }; +} + /** The stabilized attempt as a capture result: the tree carries the gesture's outcome as its own fact. */ function resolvedPostGestureCapture( stabilized: PostGestureStabilityOutcome, diff --git a/packages/daemon/src/server/__tests__/response-views.test.ts b/packages/daemon/src/server/__tests__/response-views.test.ts index 61dfd7e185..4abe42733c 100644 --- a/packages/daemon/src/server/__tests__/response-views.test.ts +++ b/packages/daemon/src/server/__tests__/response-views.test.ts @@ -83,6 +83,15 @@ test('digest carries the presented-surface fact of the tree it collapsed (#3565) expect('presentedSurface' in unread).toBe(false); }); +test('digest carries the post-gesture outcome of the tree it collapsed', () => { + const postGestureOutcome = { kind: 'loading', gesture: { action: 'open', positionals: [] } }; + const digest = snapshotView!({ ...SNAPSHOT_DATA, postGestureOutcome }, 'digest'); + expect(digest.postGestureOutcome).toEqual(postGestureOutcome); + + const atRest = snapshotView!(SNAPSHOT_DATA, 'digest'); + expect('postGestureOutcome' in atRest).toBe(false); +}); + test('digest tolerates missing/empty node trees', () => { const digest = snapshotView!({ truncated: true }, 'digest'); expect(digest).toMatchObject({ nodeCount: 0, refs: [], truncated: true }); diff --git a/packages/selectors/src/absence-observation.test.ts b/packages/selectors/src/absence-observation.test.ts index 9e239f5187..cb24e8997a 100644 --- a/packages/selectors/src/absence-observation.test.ts +++ b/packages/selectors/src/absence-observation.test.ts @@ -6,6 +6,7 @@ import { absenceCaptureOptionRefusal, classifyAbsenceObservation, } from './absence-observation.ts'; +import { absenceObservationError } from './absence-observation-errors.ts'; test('classifies a complete capture with no selector matches as absent', () => { const observation = classifyAbsenceObservation(makeSnapshotState([]), []); @@ -13,6 +14,24 @@ test('classifies a complete capture with no selector matches as absent', () => { assert.deepEqual(observation, { kind: 'absent', matches: 0 }); }); +test('classifies a miss on a still-moving or still-loading tree by its own cause, not as absent', () => { + const miss = (kind: 'unsettled' | 'loading' | 'no-effect') => + classifyAbsenceObservation( + { + ...makeSnapshotState([]), + postGestureOutcome: { kind, gesture: { action: 'open', positionals: [] } }, + }, + [], + ); + + assert.deepEqual(miss('unsettled'), { kind: 'unsettled', matches: 0 }); + assert.deepEqual(miss('loading'), { kind: 'loading', matches: 0 }); + assert.deepEqual(miss('no-effect'), { kind: 'absent', matches: 0 }); + const error = absenceObservationError('label=Done', miss('loading')); + assert.match(error.message, /the launched app was still loading its first screen/); + assert.equal(error.details?.observation, 'loading'); +}); + test('classifies matches as present and exposes only stable first-match fields', () => { const first = makeSnapshotState([ { diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index 6e346261b5..02fc268cf2 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -1,8 +1,4 @@ -import { - extractNodeText, - isPostGestureRestUnproven, - normalizeType, -} from '@agent-device/contracts/snapshot'; +import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; import { readNodeLocalIdentity, TARGET_ANNOTATION_MAX_FIELD_BYTES, @@ -34,13 +30,15 @@ export type AbsenceObservation = | { kind: 'present'; matches: number; firstMatch: AbsenceFirstMatch } | { kind: 'sparse'; matches: number; firstMatch?: AbsenceFirstMatch; quality: SparseQuality } | { kind: 'truncated'; matches: number; firstMatch?: AbsenceFirstMatch } - | { kind: 'unsettled'; matches: 0 }; + | { kind: 'unsettled'; matches: 0 } + | { kind: 'loading'; matches: 0 }; /** Why a capture with no match still cannot prove absence. */ export const UNPROVABLE_ABSENCE_CAUSES = { sparse: 'capture was sparse', truncated: 'capture was truncated', unsettled: 'the surface was still changing after a gesture', + loading: 'the launched app was still loading its first screen', } as const; export type UnprovableAbsenceKind = keyof typeof UNPROVABLE_ABSENCE_CAUSES; @@ -101,10 +99,9 @@ export function classifyAbsenceObservation( }; } if (matchCount === 0) { - return { - kind: isPostGestureRestUnproven(snapshot.postGestureOutcome) ? 'unsettled' : 'absent', - matches: 0, - }; + const outcome = snapshot.postGestureOutcome?.kind; + if (outcome === 'unsettled' || outcome === 'loading') return { kind: outcome, matches: 0 }; + return { kind: 'absent', matches: 0 }; } return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; } diff --git a/test/integration/interaction-contract/daemon-harness.ts b/test/integration/interaction-contract/daemon-harness.ts index 240b5a6dcc..826ccc9bee 100644 --- a/test/integration/interaction-contract/daemon-harness.ts +++ b/test/integration/interaction-contract/daemon-harness.ts @@ -1,5 +1,8 @@ import { assertRpcOk } from '../provider-scenarios/assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from '../provider-scenarios/fixtures.ts'; +import { + FIRST_SCREEN_NODES, + PROVIDER_SCENARIO_IOS_SIMULATOR, +} from '../provider-scenarios/fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource, @@ -20,14 +23,6 @@ export const CONTRACT_APP = 'com.example.app'; const CONTRACT_DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; const SETTLING_CAPTURE_READS = 2; -const SETTLING_CAPTURE_NODES = [ - { - index: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, -]; /** * Provider-transcript harness for contract scenarios whose path involves the @@ -47,7 +42,7 @@ export async function withIosContractDaemon( // scenario's own conversation with the runner exactly as scripted. const transcript = createProviderTranscript([ ...Array.from({ length: SETTLING_CAPTURE_READS }, () => - runnerSnapshotEntry(SETTLING_CAPTURE_NODES), + runnerSnapshotEntry(FIRST_SCREEN_NODES), ), ...entries, ]); diff --git a/test/integration/provider-scenarios/device-pin.test.ts b/test/integration/provider-scenarios/device-pin.test.ts index ca5fb3dce4..cc64f284a0 100644 --- a/test/integration/provider-scenarios/device-pin.test.ts +++ b/test/integration/provider-scenarios/device-pin.test.ts @@ -4,6 +4,7 @@ import { createAgentDeviceClient } from '@agent-device/client'; import { runCommand } from '@agent-device/command-surface'; import type { DevicePin } from '@agent-device/contracts/device'; import { + FIRST_SCREEN_NODES, PROVIDER_SCENARIO_ANDROID, PROVIDER_SCENARIO_IOS_SIMULATOR, PROVIDER_SCENARIO_MACOS, @@ -27,13 +28,10 @@ test('Provider-backed integration admits an existing session device through its deviceId: device.id, repeat: true, result: { + // A populated first screen, so the first capture after open settles on one quiet pair. nodes: [ - { - index: 0, - type: 'Application', - label: 'Existing Session App', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, + { ...FIRST_SCREEN_NODES[0], label: 'Existing Session App' }, + ...FIRST_SCREEN_NODES.slice(1), ], truncated: false, }, diff --git a/test/integration/provider-scenarios/fixtures.ts b/test/integration/provider-scenarios/fixtures.ts index f65b058644..68e0d15264 100644 --- a/test/integration/provider-scenarios/fixtures.ts +++ b/test/integration/provider-scenarios/fixtures.ts @@ -97,3 +97,24 @@ export function createDemoIosApp(prefix: string): { tempRoot: string; appPath: s ); return { tempRoot, appPath }; } + +/** + * A populated first screen for the reads that settle the first capture after `open`. The post-open + * wait refuses a tree that holds only the app root and holds a tree of four or fewer nodes for a + * quiet window, so these reads carry five rows and settle on one quiet pair. + */ +export const FIRST_SCREEN_NODES: readonly Record[] = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + ...Array.from({ length: 5 }, (_, row) => ({ + index: row + 1, + parentIndex: 0, + type: 'StaticText', + label: `Row ${row}`, + rect: { x: 20, y: 100 + row * 50, width: 300, height: 44 }, + })), +]; diff --git a/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts index 1a27db946e..54f5802848 100644 --- a/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts +++ b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { assertRpcError, assertRpcOk } from './assertions.ts'; import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from './test-timeouts.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource, @@ -49,8 +49,8 @@ function snapshotEntry(nodes: unknown[]): ProviderScenarioProviderEntry { // The scripted open is not read back, so the first capture after it settles over a quiet pair of // reads and the direct tap path stays paused until that capture has run. const SETTLING_CAPTURE_ENTRIES = [ - snapshotEntry([APPLICATION_NODE]), - snapshotEntry([APPLICATION_NODE]), + snapshotEntry([...FIRST_SCREEN_NODES]), + snapshotEntry([...FIRST_SCREEN_NODES]), ]; async function settleAfterOpen(daemon: ProviderScenarioHarness): Promise { @@ -99,9 +99,10 @@ test('Provider-backed iOS selector wait captures directly and strips selectorCha rect: { x: 100, y: 300, width: 200, height: 44 }, }, ]); - const transcript = createProviderTranscript([continueSnapshot, continueSnapshot]); + const transcript = createProviderTranscript([...SETTLING_CAPTURE_ENTRIES, continueSnapshot]); await withDirectSelectorScenario(transcript, async (daemon) => { + await settleAfterOpen(daemon); const wait = await daemon.callCommand('wait', ['label="Continue"']); const data = assertRpcOk(wait); assert.equal(data.selector, 'label="Continue"'); @@ -122,7 +123,7 @@ test('Provider-backed iOS selector wait polls a fresh capture after a selector m }, ]); const transcript = createProviderTranscript([ - loadingSnapshot, + ...SETTLING_CAPTURE_ENTRIES, loadingSnapshot, snapshotEntry([ APPLICATION_NODE, @@ -138,6 +139,7 @@ test('Provider-backed iOS selector wait polls a fresh capture after a selector m ]); await withDirectSelectorScenario(transcript, async (daemon) => { + await settleAfterOpen(daemon); const wait = await daemon.callCommand('wait', ['label="Continue"']); const data = assertRpcOk(wait); assert.equal(data.selector, 'label="Continue"'); diff --git a/test/integration/provider-scenarios/interaction-verify.test.ts b/test/integration/provider-scenarios/interaction-verify.test.ts index 733fdd74be..090a1efccf 100644 --- a/test/integration/provider-scenarios/interaction-verify.test.ts +++ b/test/integration/provider-scenarios/interaction-verify.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -57,8 +57,10 @@ function snapshotEntry(nodes: unknown[]): ProviderScenarioProviderEntry { test('Provider-backed integration press --verify returns post-action evidence digest', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i to obtain refs; the first capture after open settles: one extra read - snapshotEntry(BEFORE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry([...FIRST_SCREEN_NODES]), + snapshotEntry([...FIRST_SCREEN_NODES]), + // snapshot -i to obtain refs snapshotEntry(BEFORE_NODES), { command: 'ios.runner.tap', @@ -92,6 +94,7 @@ test('Provider-backed integration press --verify returns post-action evidence di udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/test/integration/provider-scenarios/ios-wait-selector.test.ts b/test/integration/provider-scenarios/ios-wait-selector.test.ts index bec41aad4a..d313d119da 100644 --- a/test/integration/provider-scenarios/ios-wait-selector.test.ts +++ b/test/integration/provider-scenarios/ios-wait-selector.test.ts @@ -5,11 +5,18 @@ import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { WAIT_REASONS } from '@agent-device/contracts/wait'; import { mkdtempForTestSync } from '@agent-device/test-kit'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { assertRpcOk } from './assertions.ts'; import { createProviderScenarioHarness } from './harness.ts'; import { createRecordingAppleToolProvider } from './providers.ts'; +// Rows that make each tree a populated first screen, so the first capture after open settles on one +// quiet pair. +const FIRST_SCREEN_ROWS = FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ + ...node, + index: row + 2, +})); + test.each([ ['wait', 'selector'], ['replay', 'selector'], @@ -65,6 +72,7 @@ test.each([ label: 'Ready', rect: { x: 10, y: 10, width: 100, height: 100 }, }, + ...FIRST_SCREEN_ROWS, ], }; }, @@ -153,6 +161,7 @@ test('Apple wait text ignores a text field’s typed value but matches its label value: 'benchad', rect: { x: 16, y: 200, width: 361, height: 44 }, }, + ...FIRST_SCREEN_ROWS, ], }; }, diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index c1cee904e6..fe0a765ea9 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -423,6 +423,23 @@ export async function createIosPhysicalReinstallWorld(): Promise ({ + index: firstIndex + row, + type: 'XCUIElementTypeCell', + label, + identifier: label, + rect: { x: 16, y: 520 + row * 50, width: 360, height: 44 }, + enabled: true, + hittable: true, + })); +} + function runnerSnapshot() { return { command: 'ios.runner.snapshot', @@ -448,6 +465,7 @@ function runnerSnapshot() { enabled: true, hittable: true, }, + ...settingsRows(2), ], truncated: false, }, @@ -479,6 +497,7 @@ function generalCellSnapshot(options: { y: number }) { enabled: true, hittable: true, }, + ...settingsRows(2), ], truncated: false, }, diff --git a/test/integration/provider-scenarios/press-selector-resolution.test.ts b/test/integration/provider-scenarios/press-selector-resolution.test.ts index 0dad270e79..1bb74a3c8b 100644 --- a/test/integration/provider-scenarios/press-selector-resolution.test.ts +++ b/test/integration/provider-scenarios/press-selector-resolution.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -67,8 +67,8 @@ async function withPressSelectorDaemon( // The scripted open is not read back, so the first capture after it settles over a quiet pair of // reads. A snapshot consumes that pair here, keeping the scenario's own captures one-for-one. const runnerTranscript = createProviderTranscript([ - snapshotEntry(APPLICATION_ONLY_NODES), - snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), ...entries, ]); const appleRunnerProvider = createAppleRunnerProviderFromTranscript( diff --git a/test/integration/provider-scenarios/remote-proxy-parity.test.ts b/test/integration/provider-scenarios/remote-proxy-parity.test.ts index c75187839d..53138e4ed8 100644 --- a/test/integration/provider-scenarios/remote-proxy-parity.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-parity.test.ts @@ -8,7 +8,7 @@ import { test, vi } from 'vitest'; import { sendToDaemon } from '@agent-device/daemon-client'; import { AppError, type DaemonError } from '@agent-device/kernel/errors'; import { skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, type ProviderScenarioHarness } from './harness.ts'; import { createProviderScenarioProxy, @@ -89,6 +89,9 @@ function scriptedTree() { enabled: true, hittable: true, }, + // More rows make this a populated first screen, so the first capture after open settles on one + // quiet pair. + ...FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ ...node, index: row + 2 })), ], truncated: false, }; diff --git a/test/integration/provider-scenarios/scroll-movement-observation.test.ts b/test/integration/provider-scenarios/scroll-movement-observation.test.ts index 411ad06fb6..7f1edb4429 100644 --- a/test/integration/provider-scenarios/scroll-movement-observation.test.ts +++ b/test/integration/provider-scenarios/scroll-movement-observation.test.ts @@ -4,7 +4,7 @@ import type { AppleToolProvider } from '@agent-device/platform-apple/tool-provid import type { SimulatorSnapshotSource } from '@agent-device/platform-apple/snapshot-source'; import type { ExecOptions, ExecResult } from '@agent-device/host-kit/command'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -148,8 +148,10 @@ function iosSimulatorTool(): { provider: AppleToolProvider } { test('Provider-backed integration scroll down answers moved on the first post-gesture capture', async () => { const runnerTranscript = createProviderTranscript([ + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), snapshotEntry(screen(0, true)), // pre-scroll baseline - snapshotEntry(screen(0, true)), // the first capture after open settles: one extra read scrollEntry(), snapshotEntry(screen(-100, true)), // post-gesture: rows shifted into view on the first read ]); @@ -169,6 +171,7 @@ test('Provider-backed integration scroll down answers moved on the first post-ge async (daemon) => { assertRpcOk(await daemon.callCommand('open', [APP], { platform: 'ios', udid: DEVICE_ID })); assertRpcOk(await daemon.callCommand('snapshot')); + assertRpcOk(await daemon.callCommand('snapshot')); const callsBeforeScroll = runnerTranscript.calls.length; const scroll = await daemon.callCommand('scroll', ['down']); @@ -194,8 +197,10 @@ test('Provider-backed integration scroll down answers moved on the first post-ge // unscripted runner call, so either regression fails this scenario. test('Provider-backed integration scroll with postGestureStabilization false skips the movement check and the next read waits for nothing', async () => { const runnerTranscript = createProviderTranscript([ + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), snapshotEntry(screen(0, true)), // pre-scroll baseline - snapshotEntry(screen(0, true)), // the first capture after open settles: one extra read scrollEntry(), snapshotEntry(screen(-100, true)), // the only read after the gesture: the caller's own ]); @@ -216,6 +221,7 @@ test('Provider-backed integration scroll with postGestureStabilization false ski const client = daemon.client(); await client.apps.open({ app: APP, platform: 'ios', udid: DEVICE_ID }); await client.capture.snapshot(); + await client.capture.snapshot(); const callsBeforeScroll = runnerTranscript.calls.length; const scroll = await client.interactions.scroll({ diff --git a/test/integration/provider-scenarios/settle-observation.test.ts b/test/integration/provider-scenarios/settle-observation.test.ts index e37a59c67a..de76eb22f1 100644 --- a/test/integration/provider-scenarios/settle-observation.test.ts +++ b/test/integration/provider-scenarios/settle-observation.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -178,8 +178,10 @@ test( { timeout: PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS }, async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues refs; the first capture after open settles: one extra read - snapshotEntry(BEFORE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues refs snapshotEntry(BEFORE_NODES), // press label=Continue --settle: resolution capture, tap, settle captures snapshotEntry(BEFORE_NODES), @@ -211,6 +213,7 @@ test( udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, @@ -310,10 +313,12 @@ test('Provider-backed integration never-settled press --settle does not issue di }, ]; const runnerTranscript = createProviderTranscript([ - // press label=Continue --settle: resolution capture (the first capture after open settles: one extra read), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // press label=Continue --settle: resolution capture, // tap, then a settle stream that never repeats itself, so the loop can only ever time out. snapshotEntry(BEFORE_NODES), - snapshotEntry(BEFORE_NODES), tapEntry(200, 322), changingSnapshotEntry(loadingNodes), ]); @@ -340,6 +345,7 @@ test('Provider-backed integration never-settled press --settle does not issue di udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const press = await daemon.callCommand('press', ['label=Continue'], { settle: true, @@ -371,8 +377,10 @@ test('Provider-backed integration never-settled press --settle does not issue di test('Provider-backed integration modal-dismiss press --settle attaches the unchanged interactive tail', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues refs; the first capture after open settles: one extra read - snapshotEntry(MODAL_BEFORE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues refs snapshotEntry(MODAL_BEFORE_NODES), // press label=Cancel --settle: resolution capture, tap, settle captures. // The dialog closes leaving Continue in place — a removals-only diff with @@ -406,6 +414,7 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/test/integration/provider-scenarios/stale-ref-warning.test.ts b/test/integration/provider-scenarios/stale-ref-warning.test.ts index 327b0df46e..115989e3d4 100644 --- a/test/integration/provider-scenarios/stale-ref-warning.test.ts +++ b/test/integration/provider-scenarios/stale-ref-warning.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -62,7 +62,7 @@ const MAGIC_CODE_NODES = [ }, ]; -function snapshotEntry(nodes = NODES): ProviderScenarioProviderEntry { +function snapshotEntry(nodes: readonly unknown[] = NODES): ProviderScenarioProviderEntry { return { command: 'ios.runner.snapshot', deviceId: DEVICE_ID, @@ -82,8 +82,10 @@ function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { test('Provider-backed integration iOS @refs reject after a selector press replaces the tree', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues refs to the client; the first capture after open settles: one extra read - snapshotEntry(), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues refs to the client snapshotEntry(), // press label=Continue: selector resolution capture replaces the stored tree snapshotEntry(), @@ -117,6 +119,7 @@ test('Provider-backed integration iOS @refs reject after a selector press replac udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, @@ -150,9 +153,10 @@ test('Provider-backed integration iOS @refs reject after a selector press replac test('Provider-backed iOS press rejects a stale ref after navigation', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: @e3 is Verify on the magic-code screen; the first capture after open settles: - // one extra read. - snapshotEntry(MAGIC_CODE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: @e3 is Verify on the magic-code screen. snapshotEntry(MAGIC_CODE_NODES), // press label=Back: the pre-action capture still sees that screen. snapshotEntry(MAGIC_CODE_NODES), @@ -183,6 +187,7 @@ test('Provider-backed iOS press rejects a stale ref after navigation', async () udid: DEVICE_ID, }), ); + assertRpcOk(await daemon.callCommand('snapshot')); assertRpcOk( await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/test/integration/provider-scenarios/versioned-refs.test.ts b/test/integration/provider-scenarios/versioned-refs.test.ts index 171ae7d074..cf17021ff1 100644 --- a/test/integration/provider-scenarios/versioned-refs.test.ts +++ b/test/integration/provider-scenarios/versioned-refs.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -46,12 +46,12 @@ const NODES = [ }, ]; -function snapshotEntry(): ProviderScenarioProviderEntry { +function snapshotEntry(nodes: readonly unknown[] = NODES): ProviderScenarioProviderEntry { return { command: 'ios.runner.snapshot', deviceId: DEVICE_ID, platform: 'apple', - result: { nodes: NODES, truncated: false }, + result: { nodes, truncated: false }, }; } @@ -66,9 +66,10 @@ function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { test('Provider-backed integration rejects stale pinned @refs and accepts current pins', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues the complete frame at the seeded generation g1; the first capture after - // open settles: one extra read - snapshotEntry(), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues the complete frame at generation g1 snapshotEntry(), // press @e2~s{g1}: pinned to the CURRENT frame — admitted, taps Cancel, // and crosses the seam so the frame expires @@ -103,6 +104,7 @@ test('Provider-backed integration rejects stale pinned @refs and accepts current udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 435e85b8eb..0fbdde766d 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -98,7 +98,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- iOS and Android: after `open` launches an app, the first capture in the session waits up to 10 seconds for the app's first screen. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count. A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. +- iOS: after `open` launches an app, the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count. A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. A capture that its producer marks sparse (`snapshotQuality.state: "sparse"`) counts when two reads agree, whatever it holds, because a failed accessibility read does not prove an empty screen. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. @@ -622,12 +622,13 @@ agent-device is text 'id="greeting"' "Welcome back" - Supported predicates are `visible`, `hidden`, `exists`, `absent`, `editable`, `selected`, `focused`, and `text`. - `is visible` passes when the element is in the visible viewport, or has no rect of its own but sits inside a visible ancestor. `is exists` only checks that the selector matches. - `is text ` compares the element's text with the value. `wait text` checks that text is present, not that it can be tapped. -- `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth capture. It does not mean hidden: `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, or unsettled captures fail. +- `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth capture. It does not mean hidden: `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, unsettled, or loading captures fail. - Predicates that name one element (all except `exists` and `absent`) and `get attrs` fail with `AMBIGUOUS_MATCH` when the selector matches more than one node; that is not a missing element. `error.details.matches` has the count, and up to five candidate `@ref`s are listed, pinned to `error.details.refsGeneration`, so you can act on one directly. Without a stored tree (a sparse capture), only the count is reported. See [Selectors](/docs/selectors#find-an-element). - On iOS, React Native text appears as two nodes with the same label. Selectors that must resolve one element (`is`, `get attrs`, `screenshot --crop-on`, replay) treat the pair as one; `find ... list` and `snapshot --json` show both. - On the web, on-screen text the browser cannot hand an action to (static text, a `status` announcement) is in the tree and readable by `wait text`, `find text`, `get text`, and `text=`, but carries no `@ref` and has no rect: `find ... list` prints it as `(no ref)`, and it is never a tap target. Such text is published on `value`, so `value="X"` can match page text as well as a field holding that text, and a predicate that needs one element then reports `AMBIGUOUS_MATCH`; use `text=` for page text, or add `editable=true` to name the field. - The first read after `scroll`, `swipe`, or `gesture swipe` waits for two captures to agree. When that proves something about the gesture, `is`, `get`, `find`, `wait`, interactions that captured it, and `snapshot` add a warning, and all but `snapshot` report `postGestureOutcome` (`{ "kind", "gesture": { "action", "positionals" } }`) in `data` or `error.details`: - `kind: "unsettled"`: the screen was still changing. A miss is not proof of absence; read again. `is absent` refuses with `observation: "unsettled"`, and `wait absent` keeps polling. + - `kind: "loading"`: the app that `open` launched still showed no first screen when the wait ran out. A miss is not proof of absence. `is absent` refuses with `observation: "loading"`, and `wait absent` keeps polling. - `kind: "no-effect"`: nothing changed. The list may be at its edge or ignore synthesized scrolls; a raw `swipe x1 y1 x2 y2` inside it moves such lists. The soft keyboard appearing or going away counts as a change only when the pre-gesture baseline and a later quiet capture both read the keyboard from their own tree and disagree: the iOS XCTest runner and the iOS Simulator AX bridge (the default Simulator route) do, a WebDriver (Appium) visibility probe does not. The AX bridge reads the keyboard as gone only from a complete tree that holds no keyboard window. Otherwise only the content is compared. - A caller that verifies the screen by polling it can skip that wait, and with it these outcomes, using `--no-post-gesture-stabilization` (see [Scroll](#scroll)). From efe440583b6ec240a734faa2449570454fb34b21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 22:14:10 +0100 Subject: [PATCH 04/15] fix(open): qualify the first-screen wait by the launch observation; parent fixture rows on the app root Co-Authored-By: Claude Opus 5.5 --- test/integration/provider-scenarios/remote-proxy-parity.test.ts | 2 +- website/docs/docs/commands.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/provider-scenarios/remote-proxy-parity.test.ts b/test/integration/provider-scenarios/remote-proxy-parity.test.ts index 53138e4ed8..8391d9f7e4 100644 --- a/test/integration/provider-scenarios/remote-proxy-parity.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-parity.test.ts @@ -91,7 +91,7 @@ function scriptedTree() { }, // More rows make this a populated first screen, so the first capture after open settles on one // quiet pair. - ...FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ ...node, index: row + 2 })), + ...FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ ...node, index: row + 2, parentIndex: 1 })), ], truncated: false, }; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 0fbdde766d..bcf302feff 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -98,7 +98,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- iOS: after `open` launches an app, the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count. A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. A capture that its producer marks sparse (`snapshotQuality.state: "sparse"`) counts when two reads agree, whatever it holds, because a failed accessibility read does not prove an empty screen. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. +- iOS: after `open` launches an app and the backend reports that launch (local simulators and devices do; a backend that reports no post-open observation skips the wait), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count. A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. A capture that its producer marks sparse (`snapshotQuality.state: "sparse"`) counts when two reads agree, whatever it holds, because a failed accessibility read does not prove an empty screen. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. From 1f137b44ab06d3d3bcc93b223112591644c9fe77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 22:17:32 +0100 Subject: [PATCH 05/15] style: format the parity fixture rows Co-Authored-By: Claude Opus 5.5 --- .../provider-scenarios/remote-proxy-parity.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/provider-scenarios/remote-proxy-parity.test.ts b/test/integration/provider-scenarios/remote-proxy-parity.test.ts index 8391d9f7e4..d55dab3563 100644 --- a/test/integration/provider-scenarios/remote-proxy-parity.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-parity.test.ts @@ -91,7 +91,11 @@ function scriptedTree() { }, // More rows make this a populated first screen, so the first capture after open settles on one // quiet pair. - ...FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ ...node, index: row + 2, parentIndex: 1 })), + ...FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ + ...node, + index: row + 2, + parentIndex: 1, + })), ], truncated: false, }; From a4a80367b4ab3f223bb391efeb08dd95abac9f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 22:18:10 +0100 Subject: [PATCH 06/15] fix(open): never accept a root-only tree as the first screen; an open supersedes owed stabilization A root-only tree is a launch or loading surface whatever quality verdict the producer gave it: the iOS runner marks exactly that tree sparse, and a runner that predates verdicts publishes it with none. Count content beyond the roots instead of trusting the sparse verdict. State why an open replaces a scroll's owed stabilization. Co-Authored-By: Claude Opus 5.5 --- .../src/post-gesture-stability.test.ts | 25 +------- .../capture-kit/src/post-gesture-stability.ts | 37 ++++++------ .../command-surface/src/management/app.ts | 2 +- .../deferred-interaction-outcome.test.ts | 59 +++++++++++++++---- .../src/core/deferred-interaction-outcome.ts | 23 +++----- website/docs/docs/commands.md | 2 +- 6 files changed, 80 insertions(+), 68 deletions(-) diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts index 57a136db40..fb7052f557 100644 --- a/packages/capture-kit/src/post-gesture-stability.test.ts +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -5,7 +5,6 @@ import type { ObservationClock } from './observe-until.ts'; import { formatPostGestureOutcomeWarning, runPostGestureStabilityLoop, - type FirstScreenContent, type PostGestureBaseline, type PostGesturePreActionBaseline, type PostGestureStabilityDecision, @@ -26,7 +25,7 @@ type Surface = Readonly<{ signature: readonly string[]; costMs: number; backend?: string; - firstScreen?: FirstScreenContent; + firstScreenContent?: number; }>; // The negative half of these lines is type-level: each @ts-expect-error fails `pnpm typecheck` @@ -100,7 +99,7 @@ function hooksFor( same(baseline.signature, quiet.signature) ? 'unchanged' : 'changed', surfacesIdentical: same, summarizeDivergence: () => ({}), - readFirstScreen: (value) => value.firstScreen ?? { kind: 'measured', contentNodes: 20 }, + countFirstScreenContent: (value) => value.firstScreenContent ?? 20, }; } @@ -235,7 +234,7 @@ function launchSurface(name: string, contentNodes: number, costMs = 50): Surface return { signature: [name, String(contentNodes)], costMs, - firstScreen: { kind: 'measured', contentNodes }, + firstScreenContent: contentNodes, }; } @@ -340,24 +339,6 @@ test('a launch whose populated tree never quiets reports it unsettled, not loadi assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); }); -test('an unverifiable capture settles a launch on its quiet pair instead of waiting as loading', async () => { - const clock = fakeClock(); - const unreadable: Surface = { - signature: ['root'], - costMs: 50, - firstScreen: { kind: 'unverifiable' }, - }; - - const outcome = await runPostGestureStabilityLoop({ - pending: LAUNCH, - hooks: hooksFor(clock, [unreadable, unreadable]), - clock, - }); - - assert.equal(outcome.value, unreadable); - assert.equal(outcome.postGestureOutcome, undefined); -}); - test('a gesture record never applies the first-screen rule to an empty tree', async () => { const clock = fakeClock(); const empty = launchSurface('empty', 0); diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 9d1df592d3..45b7814eeb 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -67,23 +67,16 @@ const FIRST_SCREEN_SPARSE_CONTENT_NODES = 4; const FIRST_SCREEN_SPARSE_QUIET_MS = 1_500; /** - * What one capture shows of a launched app's first screen: the nodes it holds besides the - * Application and Window roots, or `unverifiable` when the producer reported a sparse read. An - * accessibility failure does not prove an empty UI, so it cannot hold the loop as a loading screen. + * Whether a quiet pair can stand as a launched app's first screen, from the nodes it holds besides + * the Application and Window roots. A tree with no such node is a launch or loading surface and never + * qualifies, whatever quality verdict the producer gave it: the iOS runner marks exactly that + * root-only tree sparse, and a runner that predates verdicts publishes it with none. A sparse tree + * qualifies once it has stayed unchanged for {@link FIRST_SCREEN_SPARSE_QUIET_MS}. Any other tree + * qualifies at once. */ -export type FirstScreenContent = - | Readonly<{ kind: 'measured'; contentNodes: number }> - | Readonly<{ kind: 'unverifiable' }>; - -/** - * Whether a quiet pair can stand as a launched app's first screen. A tree with no content is a - * launch or loading surface and never qualifies. A sparse tree qualifies once it has stayed - * unchanged for {@link FIRST_SCREEN_SPARSE_QUIET_MS}. Any other tree qualifies at once. - */ -function isFirstScreenReady(content: FirstScreenContent, quietForMs: number): boolean { - if (content.kind === 'unverifiable') return true; - if (content.contentNodes === 0) return false; - if (content.contentNodes > FIRST_SCREEN_SPARSE_CONTENT_NODES) return true; +function isFirstScreenReady(contentNodes: number, quietForMs: number): boolean { + if (contentNodes === 0) return false; + if (contentNodes > FIRST_SCREEN_SPARSE_CONTENT_NODES) return true; return quietForMs >= FIRST_SCREEN_SPARSE_QUIET_MS; } @@ -157,8 +150,11 @@ export type PostGestureStabilityHooks = { surfacesIdentical: (baseline: S, current: S) => boolean; /** Why `surfacesIdentical` said no, for the veto diagnostic. */ summarizeDivergence: (baseline: S, current: S) => Record; - /** What one attempt shows of a launched app's first screen. Read only for a launch record. */ - readFirstScreen: (value: T) => FirstScreenContent; + /** + * How many nodes one attempt holds besides the Application and Window roots. Read only for a + * launch record. + */ + countFirstScreenContent: (value: T) => number; }; export type PostGestureStabilityOutcome = { @@ -253,7 +249,10 @@ export async function runPostGestureStabilityLoop - isFirstScreenReady(hooks.readFirstScreen(value), lastPairAgreed ? now() - quietSinceMs : 0); + isFirstScreenReady( + hooks.countFirstScreenContent(value), + lastPairAgreed ? now() - quietSinceMs : 0, + ); let surfaceCache: CapturedSurface | undefined; const surfaceOf = (value: T): CapturedSurface => { diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index 051c3f5870..6ec1dba0fe 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -43,7 +43,7 @@ const appsCommandMetadata = defineFieldCommandMetadata( const openCommandMetadata = defineFieldCommandMetadata( 'open', - 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On iOS, the first capture after an app launch, such as the foreground initial snapshot, polls for the first screen for up to 10s; a capture started before then still completes and is judged. A tree with only the app root never qualifies, and a tree with at most four other nodes qualifies after 1.5s unchanged; a capture the producer marks sparse qualifies when two reads agree, whatever it holds. If the budget runs out, the capture returns what it read with postGestureOutcome.kind "loading" and a warning. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', + 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On iOS, the first capture after an app launch, such as the foreground initial snapshot, polls for the first screen for up to 10s; a capture started before then still completes and is judged. A tree with only the app root never qualifies, even when the producer marks it sparse; a tree with at most four other nodes qualifies after 1.5s unchanged, and a larger tree when two reads agree. If the budget runs out, the capture returns what it read with postGestureOutcome.kind "loading" and a warning. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', { app: stringField('App name, bundle id, package, or URL.'), url: stringField('Optional URL passed with an app shell.'), diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index de33b1ba0d..e4dc0b89df 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -10,6 +10,7 @@ import { type DeferredOutcomeSnapshotAttempt, } from '../deferred-interaction-outcome.ts'; import type { SessionState } from '../session-state.ts'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { deliverySnapshot, @@ -193,20 +194,39 @@ test('the first capture after an open does not count the Window root as first-sc assert.equal(result?.snapshot.postGestureOutcome?.kind, 'loading'); }); -test('the first capture after an open settles a producer-marked sparse read on its quiet pair', async () => { +test('the first capture after an open waits through a root-only tree whatever its quality verdict', async () => { + const root = makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ]); + const rootOnlyReads: SnapshotState[] = [ + // The runner's verdict on a tree holding only its roots. + { ...root, snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' } }, + // A runner that predates verdicts publishes the same tree with none. + { nodes: root.nodes, createdAt: root.createdAt, backend: 'xctest' }, + ]; + for (const rootOnly of rootOnlyReads) { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const home = homeScreenSnapshot(); + const { capture } = scriptedCapture([rootOnly, rootOnly, rootOnly, rootOnly, home, home]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(3_000); + const result = await pendingResult; + + assert.equal(result?.snapshot, home); + assert.equal(result?.snapshot.postGestureOutcome, undefined); + } +}); + +test('the first capture after an open settles a sparse read that holds content beyond its roots', async () => { vi.useFakeTimers(); const session = makeSession('ios'); markPostOpenStabilization(session, 'observable'); const sparse = { - ...makeSnapshotState([ - { - index: 0, - type: 'Application', - label: 'App', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - ]), - snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' } as const, + ...homeScreenSnapshot(), + snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'budget' } as const, }; const { capture, calls } = scriptedCapture([sparse, sparse]); @@ -218,6 +238,25 @@ test('the first capture after an open settles a producer-marked sparse read on i assert.equal(result?.snapshot.postGestureOutcome, undefined); }); +test('an open supersedes the stabilization a scroll still owed on the screen it replaced', () => { + const session = makeSession('ios'); + session.snapshot = homeScreenSnapshot(); + markDeferredInteractionOutcome({ + session, + command: 'scroll', + positionals: ['down'], + flags: undefined, + }); + assert.ok(pendingSurfaceStabilization(session)?.baseline); + + markPostOpenStabilization(session, 'observable'); + + const pending = pendingSurfaceStabilization(session); + assert.equal(pending?.action, 'open'); + assert.equal(pending?.readiness, 'first-screen'); + assert.equal(pending?.baseline, undefined); +}); + test('the first capture after an open marks a tree that never showed a first screen as loading', async () => { vi.useFakeTimers(); const session = makeSession('ios'); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 8b36b73642..c641f96aac 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -24,7 +24,6 @@ import { } from './interaction-surface-signature.ts'; import { runPostGestureStabilityLoop, - type FirstScreenContent, type PostGestureStabilityOutcome, } from '@agent-device/capture-kit/post-gesture-stability'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; @@ -230,7 +229,12 @@ export function movingTargetBaseline( * reports no verdict launched no app, such as a URL or deep link, and does not mark. No baseline: the * pre-open tree is a different surface, so the open is never reported as having no effect. * - * Apple only: {@link readFirstScreenContent} tells the first screen from a launch surface by the + * The record replaces whatever the session's deferred surface held. A scroll's owed stabilization + * judged the scroll against its pre-action tree, and a moving-target baseline describes where the + * previous action's targets were. The open launched or foregrounded an app, so both describe a screen + * the open replaced: a verdict against them would charge the open's effect to the earlier action. + * + * Apple only: the first-screen rule tells the first screen from a launch surface by the * Application and Window roots, and an Android tree has no root node to tell them apart. */ export function markPostOpenStabilization( @@ -388,7 +392,8 @@ export async function capturePostGestureStabilizedResult(params: { classifyBaselineEvidence: classifyPostGestureBaselineEvidence, surfacesIdentical: haveIdenticalDiscriminatingSurfaces, summarizeDivergence: summarizeDiscriminatingSurfaceDivergence, - readFirstScreen: (value) => readFirstScreenContent(readSnapshot(value)), + countFirstScreenContent: (value) => + readSnapshot(value).nodes.filter((node) => !isViewportRootNode(node)).length, }, }); } finally { @@ -396,18 +401,6 @@ export async function capturePostGestureStabilizedResult(params: { } } -/** - * What one capture shows of a launched app's first screen: the nodes it holds besides the - * Application and Window roots, or `unverifiable` when the producer reported a sparse read. - */ -function readFirstScreenContent( - snapshot: Pick, -): FirstScreenContent { - if (snapshot.snapshotQuality?.state === 'sparse') return { kind: 'unverifiable' }; - const contentNodes = snapshot.nodes.filter((node) => !isViewportRootNode(node)).length; - return { kind: 'measured', contentNodes }; -} - /** The stabilized attempt as a capture result: the tree carries the gesture's outcome as its own fact. */ function resolvedPostGestureCapture( stabilized: PostGestureStabilityOutcome, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index bcf302feff..59ee3c48d3 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -98,7 +98,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- iOS: after `open` launches an app and the backend reports that launch (local simulators and devices do; a backend that reports no post-open observation skips the wait), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count. A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. A capture that its producer marks sparse (`snapshotQuality.state: "sparse"`) counts when two reads agree, whatever it holds, because a failed accessibility read does not prove an empty screen. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. +- iOS: after `open` launches an app and the backend reports that launch (local simulators and devices do; a backend that reports no post-open observation skips the wait), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count, even when its producer marks it sparse (`snapshotQuality.state: "sparse"`). A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. From 093d6ee54f3ced4e6dd00c7230bd3744b2b943ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 22:32:07 +0100 Subject: [PATCH 07/15] docs(open): name both timeout outcomes of the first-screen wait Co-Authored-By: Claude Opus 5.5 --- packages/command-registry/src/flag-definitions-action.ts | 2 +- packages/command-surface/src/management/app.ts | 2 +- packages/contracts/src/client-app.ts | 7 ++++--- website/docs/docs/commands.md | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index ff3bbdbeeb..e3eb16fb15 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -413,7 +413,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageDescription: 'open: keep normal app/device selection and return an initial snapshot of the first screen; without an app, resolve the sole running app on the sole booted iOS simulator', inputDescription: - 'Include an initial interactive snapshot in a fresh open response. On iOS, a launched app\'s snapshot polls for its first screen for up to 10s (a capture started before then still completes and is judged); when it is still loading, the snapshot carries postGestureOutcome.kind "loading" and a warning. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', + 'Include an initial interactive snapshot in a fresh open response. On iOS, a launched app\'s snapshot polls for its first screen for up to 10s (a capture started before then still completes and is judged); if it runs out, the snapshot carries postGestureOutcome.kind "loading" (no first screen yet) or "unsettled" (a first screen that kept changing) and a warning. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', projectConfig: false, recorded: false, }, diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index 6ec1dba0fe..da9aa40ed1 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -43,7 +43,7 @@ const appsCommandMetadata = defineFieldCommandMetadata( const openCommandMetadata = defineFieldCommandMetadata( 'open', - 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On iOS, the first capture after an app launch, such as the foreground initial snapshot, polls for the first screen for up to 10s; a capture started before then still completes and is judged. A tree with only the app root never qualifies, even when the producer marks it sparse; a tree with at most four other nodes qualifies after 1.5s unchanged, and a larger tree when two reads agree. If the budget runs out, the capture returns what it read with postGestureOutcome.kind "loading" and a warning. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', + 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On iOS, the first capture after an app launch, such as the foreground initial snapshot, polls for the first screen for up to 10s; a capture started before then still completes and is judged. A tree with only the app root never qualifies, even when the producer marks it sparse; a tree with at most four other nodes qualifies after 1.5s unchanged, and a larger tree when two reads agree. If the budget runs out, the capture returns what it read with a warning and postGestureOutcome.kind "loading" (no first screen yet) or "unsettled" (a first screen that kept changing). Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', { app: stringField('App name, bundle id, package, or URL.'), url: stringField('Optional URL passed with an app shell.'), diff --git a/packages/contracts/src/client-app.ts b/packages/contracts/src/client-app.ts index e6bbe36ad7..7a005da5a2 100644 --- a/packages/contracts/src/client-app.ts +++ b/packages/contracts/src/client-app.ts @@ -92,9 +92,10 @@ export type AppOpenOptions = AgentDeviceRequestOverrides & /** * Include an initial interactive snapshot in a fresh open response. On iOS, * a launched app's snapshot polls for its first screen for up to 10s (a - * capture started before then still completes and is judged); when it is - * still loading, the snapshot carries postGestureOutcome.kind "loading" and a - * warning. With no app argument, discover the sole running + * capture started before then still completes and is judged); if it runs out, + * the snapshot carries postGestureOutcome.kind "loading" (no first screen yet) + * or "unsettled" (a first screen that kept changing) and a warning. With no + * app argument, discover the sole running * app on the sole booted iOS simulator; ambiguous environments fail closed. */ foreground?: boolean; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 59ee3c48d3..a42257be11 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -98,7 +98,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- iOS: after `open` launches an app and the backend reports that launch (local simulators and devices do; a backend that reports no post-open observation skips the wait), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count, even when its producer marks it sparse (`snapshotQuality.state: "sparse"`). A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read, with `postGestureOutcome.kind: "loading"` and a warning. Run `snapshot -i` again, or `wait text `, before you act on it. +- iOS: after `open` launches an app and the backend reports that launch (local simulators and devices do; a backend that reports no post-open observation skips the wait), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count, even when its producer marks it sparse (`snapshotQuality.state: "sparse"`). A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. @@ -627,7 +627,7 @@ agent-device is text 'id="greeting"' "Welcome back" - On iOS, React Native text appears as two nodes with the same label. Selectors that must resolve one element (`is`, `get attrs`, `screenshot --crop-on`, replay) treat the pair as one; `find ... list` and `snapshot --json` show both. - On the web, on-screen text the browser cannot hand an action to (static text, a `status` announcement) is in the tree and readable by `wait text`, `find text`, `get text`, and `text=`, but carries no `@ref` and has no rect: `find ... list` prints it as `(no ref)`, and it is never a tap target. Such text is published on `value`, so `value="X"` can match page text as well as a field holding that text, and a predicate that needs one element then reports `AMBIGUOUS_MATCH`; use `text=` for page text, or add `editable=true` to name the field. - The first read after `scroll`, `swipe`, or `gesture swipe` waits for two captures to agree. When that proves something about the gesture, `is`, `get`, `find`, `wait`, interactions that captured it, and `snapshot` add a warning, and all but `snapshot` report `postGestureOutcome` (`{ "kind", "gesture": { "action", "positionals" } }`) in `data` or `error.details`: - - `kind: "unsettled"`: the screen was still changing. A miss is not proof of absence; read again. `is absent` refuses with `observation: "unsettled"`, and `wait absent` keeps polling. + - `kind: "unsettled"`: the screen was still changing (after a gesture, or a first screen after `open` that never quieted). A miss is not proof of absence; read again. `is absent` refuses with `observation: "unsettled"`, and `wait absent` keeps polling. - `kind: "loading"`: the app that `open` launched still showed no first screen when the wait ran out. A miss is not proof of absence. `is absent` refuses with `observation: "loading"`, and `wait absent` keeps polling. - `kind: "no-effect"`: nothing changed. The list may be at its edge or ignore synthesized scrolls; a raw `swipe x1 y1 x2 y2` inside it moves such lists. The soft keyboard appearing or going away counts as a change only when the pre-gesture baseline and a later quiet capture both read the keyboard from their own tree and disagree: the iOS XCTest runner and the iOS Simulator AX bridge (the default Simulator route) do, a WebDriver (Appium) visibility probe does not. The AX bridge reads the keyboard as gone only from a complete tree that holds no keyboard window. Otherwise only the content is compared. - A caller that verifies the screen by polling it can skip that wait, and with it these outcomes, using `--no-post-gesture-stabilization` (see [Scroll](#scroll)). From eea1539b6236c4ea59df1df0f1c3fdb521189d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 22:36:09 +0100 Subject: [PATCH 08/15] fix(open): report loading only for a launch tree with no content A small first screen that keeps changing through the budget is unsettled, as the docs say; loading means nothing beyond the app roots. Co-Authored-By: Claude Opus 5.5 --- .../capture-kit/src/post-gesture-stability.test.ts | 14 ++++++++++++++ packages/capture-kit/src/post-gesture-stability.ts | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts index fb7052f557..d0deda7a73 100644 --- a/packages/capture-kit/src/post-gesture-stability.test.ts +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -339,6 +339,20 @@ test('a launch whose populated tree never quiets reports it unsettled, not loadi assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); }); +test('a launch whose small first screen keeps changing reports it unsettled, not loading', async () => { + const clock = fakeClock(); + const frames = Array.from({ length: 60 }, (_, frame) => launchSurface(`tick ${frame}`, 2)); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, frames), + clock, + }); + + assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); + assert.ok(clock.now() >= 10_000, `gave up after ${clock.now()}ms`); +}); + test('a gesture record never applies the first-screen rule to an empty tree', async () => { const clock = fakeClock(); const empty = launchSurface('empty', 0); diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 45b7814eeb..39fecc2ac9 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -340,8 +340,9 @@ export async function runPostGestureStabilityLoop Date: Fri, 13 Nov 2026 22:44:21 +0100 Subject: [PATCH 09/15] fix(open): judge a first screen at the deadline by the same readiness rule A small tree whose last pair agrees just before the budget ends has not held the 1.5 s quiet window, so it is unsettled, not settled. Co-Authored-By: Claude Opus 5.5 --- .../src/post-gesture-stability.test.ts | 19 +++++++++++++++++++ .../capture-kit/src/post-gesture-stability.ts | 13 +++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts index d0deda7a73..b11fddf2f6 100644 --- a/packages/capture-kit/src/post-gesture-stability.test.ts +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -353,6 +353,25 @@ test('a launch whose small first screen keeps changing reports it unsettled, not assert.ok(clock.now() >= 10_000, `gave up after ${clock.now()}ms`); }); +test('a small first screen that quiets just before the deadline is unsettled, not settled', async () => { + const clock = fakeClock(); + const hooks = { + ...hooksFor(clock, []), + // Changes on every read until 9.6 s, then holds: its last pair agrees, but it has been quiet + // for well under the 1.5 s window a small tree needs. + capture: async () => { + clock.advance(50); + const at = clock.now(); + return at < 9_600 ? launchSurface(`tick ${at}`, 2, 0) : launchSurface('held', 2, 0); + }, + }; + + const outcome = await runPostGestureStabilityLoop({ pending: LAUNCH, hooks, clock }); + + assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); + assert.ok(clock.now() >= 10_000, `gave up after ${clock.now()}ms`); +}); + test('a gesture record never applies the first-screen rule to an empty tree', async () => { const clock = fakeClock(); const empty = launchSurface('empty', 0); diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 39fecc2ac9..cbb70c04ce 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -340,10 +340,15 @@ export async function runPostGestureStabilityLoop Date: Fri, 13 Nov 2026 23:58:23 +0100 Subject: [PATCH 10/15] docs(open): limit the first-screen wait promise to local iOS simulators Only a local iOS simulator reports the post-open observation that arms the wait; physical devices and other backends skip it. Absence classification now maps every post-gesture outcome kind, so a new rest-unproven kind is a compile error rather than a silent absent. Co-Authored-By: Claude Opus 5.5 --- .../src/flag-definitions-action.ts | 2 +- packages/command-surface/src/management/app.ts | 2 +- packages/contracts/src/client-app.ts | 12 +++++------- packages/selectors/src/absence-observation.ts | 16 +++++++++++++--- website/docs/docs/commands.md | 2 +- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index e3eb16fb15..fadfeb4b69 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -413,7 +413,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageDescription: 'open: keep normal app/device selection and return an initial snapshot of the first screen; without an app, resolve the sole running app on the sole booted iOS simulator', inputDescription: - 'Include an initial interactive snapshot in a fresh open response. On iOS, a launched app\'s snapshot polls for its first screen for up to 10s (a capture started before then still completes and is judged); if it runs out, the snapshot carries postGestureOutcome.kind "loading" (no first screen yet) or "unsettled" (a first screen that kept changing) and a warning. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', + 'Include an initial interactive snapshot in a fresh open response. On a local iOS simulator it waits up to 10s for the app\'s first screen (physical devices and other backends skip the wait); on timeout it carries postGestureOutcome.kind "loading" or "unsettled" and a warning. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', projectConfig: false, recorded: false, }, diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index da9aa40ed1..e1a7f06fe9 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -43,7 +43,7 @@ const appsCommandMetadata = defineFieldCommandMetadata( const openCommandMetadata = defineFieldCommandMetadata( 'open', - 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On iOS, the first capture after an app launch, such as the foreground initial snapshot, polls for the first screen for up to 10s; a capture started before then still completes and is judged. A tree with only the app root never qualifies, even when the producer marks it sparse; a tree with at most four other nodes qualifies after 1.5s unchanged, and a larger tree when two reads agree. If the budget runs out, the capture returns what it read with a warning and postGestureOutcome.kind "loading" (no first screen yet) or "unsettled" (a first screen that kept changing). Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', + 'Boot the selected device when needed, then open an app, deep link, or URL in a session. The app or URL inputs select the foreground automation target. On a local iOS simulator, the first capture after an app launch, such as the foreground initial snapshot, polls for the first screen for up to 10s (physical devices and backends that report no post-open observation skip the wait); a capture started before then still completes and is judged. A tree with only the app root never qualifies, even when the producer marks it sparse; a tree with at most four other nodes qualifies after 1.5s unchanged, and a larger tree when two reads agree. If the budget runs out, the capture returns what it read with a warning and postGestureOutcome.kind "loading" (no first screen yet) or "unsettled" (a first screen that kept changing). Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', { app: stringField('App name, bundle id, package, or URL.'), url: stringField('Optional URL passed with an app shell.'), diff --git a/packages/contracts/src/client-app.ts b/packages/contracts/src/client-app.ts index 7a005da5a2..a63a22963c 100644 --- a/packages/contracts/src/client-app.ts +++ b/packages/contracts/src/client-app.ts @@ -90,13 +90,11 @@ export type AppOpenOptions = AgentDeviceRequestOverrides & // `--foreground` FlagDefinition's `inputDescription`) verbatim, pinned to it by // `commands/command-input-option-field.test.ts`. /** - * Include an initial interactive snapshot in a fresh open response. On iOS, - * a launched app's snapshot polls for its first screen for up to 10s (a - * capture started before then still completes and is judged); if it runs out, - * the snapshot carries postGestureOutcome.kind "loading" (no first screen yet) - * or "unsettled" (a first screen that kept changing) and a warning. With no - * app argument, discover the sole running - * app on the sole booted iOS simulator; ambiguous environments fail closed. + * Include an initial interactive snapshot in a fresh open response. On a local iOS simulator it + * waits up to 10s for the app's first screen (physical devices and other backends skip the + * wait); on timeout it carries postGestureOutcome.kind "loading" or "unsettled" and a warning. + * With no app argument, discover the sole running app on the sole booted iOS simulator; + * ambiguous environments fail closed. */ foreground?: boolean; saveScript?: boolean | string; diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index 02fc268cf2..25d919e357 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -5,7 +5,7 @@ import { truncateToUtf8Bytes, } from '@agent-device/ad-script'; import type { SnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { PostGestureOutcome, SnapshotState } from '@agent-device/contracts/snapshot-model'; export type AbsenceFirstMatch = { id?: string; @@ -43,6 +43,15 @@ export const UNPROVABLE_ABSENCE_CAUSES = { export type UnprovableAbsenceKind = keyof typeof UNPROVABLE_ABSENCE_CAUSES; +/** Which post-gesture outcomes leave a zero-match capture unable to prove absence; every kind must be listed. */ +const ABSENCE_UNPROVEN_BY_OUTCOME: Readonly< + Record +> = { + unsettled: 'unsettled', + loading: 'loading', + 'no-effect': undefined, +}; + export function isUnprovableAbsence(kind: unknown): kind is UnprovableAbsenceKind { return typeof kind === 'string' && Object.hasOwn(UNPROVABLE_ABSENCE_CAUSES, kind); } @@ -99,8 +108,9 @@ export function classifyAbsenceObservation( }; } if (matchCount === 0) { - const outcome = snapshot.postGestureOutcome?.kind; - if (outcome === 'unsettled' || outcome === 'loading') return { kind: outcome, matches: 0 }; + const outcome = snapshot.postGestureOutcome; + const unproven = outcome ? ABSENCE_UNPROVEN_BY_OUTCOME[outcome.kind] : undefined; + if (unproven) return { kind: unproven, matches: 0 }; return { kind: 'absent', matches: 0 }; } return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a42257be11..78249a53e7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -98,7 +98,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- iOS: after `open` launches an app and the backend reports that launch (local simulators and devices do; a backend that reports no post-open observation skips the wait), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count, even when its producer marks it sparse (`snapshotQuality.state: "sparse"`). A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. +- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; physical devices and backends that report no post-open observation skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count, even when its producer marks it sparse (`snapshotQuality.state: "sparse"`). A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. From 61f5873ca3966c07af6c59d6b54d4f93600ee4d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 00:03:36 +0100 Subject: [PATCH 11/15] refactor(selectors): split the zero-match absence branch out of classifyAbsenceObservation Co-Authored-By: Claude Opus 5.5 --- packages/selectors/src/absence-observation.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index 25d919e357..2dd0ef0d4e 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -107,15 +107,16 @@ export function classifyAbsenceObservation( }, }; } - if (matchCount === 0) { - const outcome = snapshot.postGestureOutcome; - const unproven = outcome ? ABSENCE_UNPROVEN_BY_OUTCOME[outcome.kind] : undefined; - if (unproven) return { kind: unproven, matches: 0 }; - return { kind: 'absent', matches: 0 }; - } + if (matchCount === 0) return zeroMatchObservation(snapshot.postGestureOutcome); return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; } +/** A zero-match capture proves absence unless its post-gesture outcome left the screen unproven. */ +function zeroMatchObservation(outcome: PostGestureOutcome | undefined): AbsenceObservation { + const unproven = outcome ? ABSENCE_UNPROVEN_BY_OUTCOME[outcome.kind] : undefined; + return unproven ? { kind: unproven, matches: 0 } : { kind: 'absent', matches: 0 }; +} + /** * The one definition of "this capture is too sparse to trust": the backend's own verdict, then the * legacy iOS shape that predates verdicts. Shared with `scroll --until`, which must not stop on a From 7872ffdb083ab3756ce6190a52960f73e5d3ce20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 14 Nov 2026 00:12:58 +0100 Subject: [PATCH 12/15] docs(open): describe the post-open wait as the first-screen wait on every reported observation Co-Authored-By: Claude Opus 5.5 --- website/docs/docs/commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 78249a53e7..af6b166583 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -111,7 +111,7 @@ Open apps and links: iOS Simulator launch behavior: -- An `open` that launches an app reports `timing.postOpenObservation`. When it is present and not `"observable"`, the next command that reads the screen waits briefly until two reads agree, so a `click` lands where the element comes to rest. If the screen is still moving after that wait, the command proceeds and warns that the surface was still changing. +- An `open` that launches an app reports `timing.postOpenObservation`. Whenever it is present, the next command that reads the screen waits for the app's first screen, up to the 10-second polling budget described under `open`, so a `click` lands on the real first screen where its element comes to rest. If the wait runs out, the command proceeds and warns with `postGestureOutcome.kind` `"loading"` (no first screen yet) or `"unsettled"` (a first screen that kept changing). - If CoreSimulator never answers a URL hand-off, `open` fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout`. The Simulator may still have received the URL; retry the open or answer the prompt with `alert accept`. - When `open --launch-url ` raises an `Open in ""?` prompt and the URL's scheme belongs to the session app, `open` taps **Open** and returns `launchConfirmation: "accepted"`. If the app still never starts, `open` hands over the URL once more. - On a local iOS Simulator, `open` recognizes the prompt in the Simulator's preferred languages, through the OS's own wording of its title and buttons, so a French Simulator's `Ouvrir dans « » ?` is answered like the English one. `alert accept` and `alert dismiss` answer it by the same wording. If the runner cannot read that wording, the prompt is not recognized. From 896ab4324acebd5cc392151d06bebd26f9cf2a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 10:55:38 +0100 Subject: [PATCH 13/15] fix(capture): tell a first screen from a splash by its readable content Measured on Bluesky, the storyboard splash is one unlabeled full-screen image that held unchanged for 1.2-1.9 s after open returned, so the "sparse tree quiet for 1.5 s" rule could return the splash as the first screen. A first screen now needs an element besides the app roots with a label or value that is not a busy indicator. The node threshold and the quiet window are gone: a one-button screen is ready on its first quiet pair, and a splash never is. Co-Authored-By: Claude Opus 5.5 --- .../src/post-gesture-stability.test.ts | 103 +++--------------- .../capture-kit/src/post-gesture-stability.ts | 67 ++---------- .../command-surface/src/management/app.ts | 2 +- .../deferred-interaction-outcome.test.ts | 82 +++++++++++++- .../src/core/deferred-interaction-outcome.ts | 30 ++++- website/docs/docs/commands.md | 2 +- 6 files changed, 137 insertions(+), 149 deletions(-) diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts index b11fddf2f6..d0f30d73fe 100644 --- a/packages/capture-kit/src/post-gesture-stability.test.ts +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -25,7 +25,8 @@ type Surface = Readonly<{ signature: readonly string[]; costMs: number; backend?: string; - firstScreenContent?: number; + /** Whether this read shows a launched app's first screen; absent reads count as one. */ + firstScreen?: boolean; }>; // The negative half of these lines is type-level: each @ts-expect-error fails `pnpm typecheck` @@ -99,7 +100,7 @@ function hooksFor( same(baseline.signature, quiet.signature) ? 'unchanged' : 'changed', surfacesIdentical: same, summarizeDivergence: () => ({}), - countFirstScreenContent: (value) => value.firstScreenContent ?? 20, + showsFirstScreen: (value) => value.firstScreen ?? true, }; } @@ -230,33 +231,30 @@ test('an effect that lands after a rebase keeps the loop polling until the moved const LAUNCH = { action: 'open', positionals: [], readiness: 'first-screen' as const }; -function launchSurface(name: string, contentNodes: number, costMs = 50): Surface { - return { - signature: [name, String(contentNodes)], - costMs, - firstScreenContent: contentNodes, - }; +function launchSurface(name: string, firstScreen: boolean, costMs = 50): Surface { + return { signature: [name], costMs, firstScreen }; } -test('a launch keeps polling through a quiet loading tree until the first screen appears', async () => { +test('a launch keeps polling through a splash that holds unchanged until the first screen appears', async () => { const clock = fakeClock(); - const loading = launchSurface('loading', 0); - const home = launchSurface('home', 20); + const splash = launchSurface('splash', false); + const home = launchSurface('home', true); const outcome = await runPostGestureStabilityLoop({ pending: LAUNCH, - hooks: hooksFor(clock, [...Array.from({ length: 12 }, () => loading), home, home]), + // Ten reads a 200 ms poll apart: the splash holds unchanged for over 2 s. + hooks: hooksFor(clock, [...Array.from({ length: 10 }, () => splash), home, home]), clock, }); assert.equal(outcome.value, home); assert.equal(outcome.postGestureOutcome, undefined); - assert.ok(clock.now() > 1_500, `a quiet loading tree settled after ${clock.now()}ms`); + assert.ok(clock.now() > 2_000, `a quiet splash settled after ${clock.now()}ms`); }); -test('a launch that reads a populated first screen returns on its first quiet pair', async () => { +test('a launch returns on the first quiet pair that shows the first screen', async () => { const clock = fakeClock(); - const home = launchSurface('home', 20); + const home = launchSurface('home', true); const outcome = await runPostGestureStabilityLoop({ pending: LAUNCH, @@ -269,44 +267,9 @@ test('a launch that reads a populated first screen returns on its first quiet pa assert.ok(clock.now() < 500, `settled after ${clock.now()}ms`); }); -test('a sparse first screen settles once it stays unchanged, far inside the launch budget', async () => { - const clock = fakeClock(); - const oneButton = launchSurface('one-button', 1); - - const outcome = await runPostGestureStabilityLoop({ - pending: LAUNCH, - hooks: hooksFor( - clock, - Array.from({ length: 20 }, () => oneButton), - ), - clock, - }); - - assert.equal(outcome.value, oneButton); - assert.equal(outcome.postGestureOutcome, undefined); - assert.ok(clock.now() >= 1_500, `a sparse tree settled after only ${clock.now()}ms`); - assert.ok(clock.now() < 2_500, `a sparse tree waited ${clock.now()}ms`); -}); - -test('a sparse tree that keeps changing does not settle on a quiet pair', async () => { - const clock = fakeClock(); - const progress = [10, 10, 40, 40, 70, 70, 100, 100].map((percent) => - launchSurface(`downloading ${percent}%`, 1), - ); - const home = launchSurface('home', 20); - - const outcome = await runPostGestureStabilityLoop({ - pending: LAUNCH, - hooks: hooksFor(clock, [...progress, home, home]), - clock, - }); - - assert.equal(outcome.value, home); -}); - test('a launch whose budget expires on a loading tree reports the app as loading', async () => { const clock = fakeClock(); - const loading = launchSurface('loading', 0); + const loading = launchSurface('loading', false); const outcome = await runPostGestureStabilityLoop({ pending: LAUNCH, @@ -326,9 +289,9 @@ test('a launch whose budget expires on a loading tree reports the app as loading assert.match(formatPostGestureOutcomeWarning(outcome.postGestureOutcome!), /still loading/); }); -test('a launch whose populated tree never quiets reports it unsettled, not loading', async () => { +test('a launch whose first screen never quiets reports it unsettled, not loading', async () => { const clock = fakeClock(); - const frames = Array.from({ length: 60 }, (_, frame) => launchSurface(`frame ${frame}`, 20)); + const frames = Array.from({ length: 60 }, (_, frame) => launchSurface(`frame ${frame}`, true)); const outcome = await runPostGestureStabilityLoop({ pending: LAUNCH, @@ -336,45 +299,13 @@ test('a launch whose populated tree never quiets reports it unsettled, not loadi clock, }); - assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); -}); - -test('a launch whose small first screen keeps changing reports it unsettled, not loading', async () => { - const clock = fakeClock(); - const frames = Array.from({ length: 60 }, (_, frame) => launchSurface(`tick ${frame}`, 2)); - - const outcome = await runPostGestureStabilityLoop({ - pending: LAUNCH, - hooks: hooksFor(clock, frames), - clock, - }); - - assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); - assert.ok(clock.now() >= 10_000, `gave up after ${clock.now()}ms`); -}); - -test('a small first screen that quiets just before the deadline is unsettled, not settled', async () => { - const clock = fakeClock(); - const hooks = { - ...hooksFor(clock, []), - // Changes on every read until 9.6 s, then holds: its last pair agrees, but it has been quiet - // for well under the 1.5 s window a small tree needs. - capture: async () => { - clock.advance(50); - const at = clock.now(); - return at < 9_600 ? launchSurface(`tick ${at}`, 2, 0) : launchSurface('held', 2, 0); - }, - }; - - const outcome = await runPostGestureStabilityLoop({ pending: LAUNCH, hooks, clock }); - assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); assert.ok(clock.now() >= 10_000, `gave up after ${clock.now()}ms`); }); test('a gesture record never applies the first-screen rule to an empty tree', async () => { const clock = fakeClock(); - const empty = launchSurface('empty', 0); + const empty = launchSurface('empty', false); const outcome = await runPostGestureStabilityLoop({ pending: PENDING, diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index cbb70c04ce..b46bc7b9b9 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -53,33 +53,6 @@ const FIRST_SCREEN_SCHEDULE: ObservationSchedule = { minPolls: 2, }; -/** - * At most this many nodes besides the Application and Window roots is a sparse first screen. With - * its root that is the five-node tree the settle loop already calls nearly empty. - */ -const FIRST_SCREEN_SPARSE_CONTENT_NODES = 4; - -/** - * How long a sparse first screen must stay unchanged before it counts as the real first screen. A - * screen with one button is ready after this window; a loading surface that changes within it is - * not. - */ -const FIRST_SCREEN_SPARSE_QUIET_MS = 1_500; - -/** - * Whether a quiet pair can stand as a launched app's first screen, from the nodes it holds besides - * the Application and Window roots. A tree with no such node is a launch or loading surface and never - * qualifies, whatever quality verdict the producer gave it: the iOS runner marks exactly that - * root-only tree sparse, and a runner that predates verdicts publishes it with none. A sparse tree - * qualifies once it has stayed unchanged for {@link FIRST_SCREEN_SPARSE_QUIET_MS}. Any other tree - * qualifies at once. - */ -function isFirstScreenReady(contentNodes: number, quietForMs: number): boolean { - if (contentNodes === 0) return false; - if (contentNodes > FIRST_SCREEN_SPARSE_CONTENT_NODES) return true; - return quietForMs >= FIRST_SCREEN_SPARSE_QUIET_MS; -} - export type BaselineSurfaceEvidence = 'changed' | 'unchanged' | 'ambiguous'; /** @@ -127,7 +100,7 @@ export type PostGestureStabilityPending = { baseline?: PostGesturePreActionBaseline; /** * `first-screen`: an app launch marked this record. A quiet pair settles the loop only when - * {@link isFirstScreenReady} accepts it, under {@link FIRST_SCREEN_SCHEDULE}. + * `showsFirstScreen` accepts it, under {@link FIRST_SCREEN_SCHEDULE}. */ readiness?: 'first-screen'; }; @@ -151,10 +124,11 @@ export type PostGestureStabilityHooks = { /** Why `surfacesIdentical` said no, for the veto diagnostic. */ summarizeDivergence: (baseline: S, current: S) => Record; /** - * How many nodes one attempt holds besides the Application and Window roots. Read only for a - * launch record. + * Whether one attempt shows a launched app's first screen rather than its launch or loading + * surface. The producer owns the rule because only it can read the tree. Read only for a launch + * record. */ - countFirstScreenContent: (value: T) => number; + showsFirstScreen: (value: T) => boolean; }; export type PostGestureStabilityOutcome = { @@ -245,14 +219,7 @@ export async function runPostGestureStabilityLoop - isFirstScreenReady( - hooks.countFirstScreenContent(value), - lastPairAgreed ? now() - quietSinceMs : 0, - ); let surfaceCache: CapturedSurface | undefined; const surfaceOf = (value: T): CapturedSurface => { @@ -269,20 +236,14 @@ export async function runPostGestureStabilityLoop { attempts += 1; const current = surfaceOf(latest); - if (previousValue === undefined) { - quietSinceMs = now(); - return { kind: 'continue' }; - } + if (previousValue === undefined) return { kind: 'continue' }; const previous = surfaceOf(previousValue); lastPairAgreed = hooks.signaturesStable(previous.signature, current.signature); - if (!lastPairAgreed) { - quietSinceMs = now(); - return { kind: 'continue' }; - } + if (!lastPairAgreed) return { kind: 'continue' }; const elapsedMs = now() - startedAt; - if (firstScreen && !firstScreenReady(latest)) return { kind: 'continue' }; + if (firstScreen && !hooks.showsFirstScreen(latest)) return { kind: 'continue' }; // A capture plan may fall back or be pre-empted by the XCTest-channel penalty at any time, // so the serving surface can change mid-poll. This quiet pair says nothing about the // gesture: adopt it as a post-action baseline and keep going (#1569). The pair that rebases @@ -340,15 +301,9 @@ export async function runPostGestureStabilityLoop { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + // The storyboard splash a launched React Native app shows: one unlabeled full-screen image. + const splash = makeSnapshotState([ + APP_ROOT, + { + index: 1, + parentIndex: 0, + type: 'Image', + identifier: 'SplashScreenLegacy', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ]); + const home = homeScreenSnapshot(); + const { capture } = scriptedCapture([...Array.from({ length: 12 }, () => splash), home, home]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(4_000); + const result = await pendingResult; + + assert.equal(result?.snapshot, home); + assert.equal(result?.snapshot.postGestureOutcome, undefined); +}); + +test('the first capture after an open does not take a busy indicator for the first screen', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const spinner = makeSnapshotState([ + APP_ROOT, + { + index: 1, + parentIndex: 0, + type: 'ActivityIndicator', + label: 'In progress', + rect: { x: 180, y: 400, width: 30, height: 30 }, + }, + ]); + const { capture } = scriptedCapture([spinner]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(11_000); + const result = await pendingResult; + + assert.equal(result?.snapshot.postGestureOutcome?.kind, 'loading'); +}); + +test('the first capture after an open settles a one-button first screen on its first quiet pair', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const oneButton = makeSnapshotState([ + APP_ROOT, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 20, y: 700, width: 350, height: 50 }, + }, + ]); + const { capture, calls } = scriptedCapture([oneButton, oneButton]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pendingResult; + + assert.equal(calls(), 2); + assert.equal(result?.snapshot.postGestureOutcome, undefined); +}); + test('the first capture after an open waits through a root-only tree whatever its quality verdict', async () => { const root = makeSnapshotState([ { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index c641f96aac..821a60bd93 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -3,6 +3,7 @@ import type { SettleObservation, SurfaceScopedNodes } from '@agent-device/contra import type { PostOpenObservation } from '@agent-device/contracts/application-lifecycle-runtime'; import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device'; +import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract'; import { @@ -234,8 +235,8 @@ export function movingTargetBaseline( * previous action's targets were. The open launched or foregrounded an app, so both describe a screen * the open replaced: a verdict against them would charge the open's effect to the earlier action. * - * Apple only: the first-screen rule tells the first screen from a launch surface by the - * Application and Window roots, and an Android tree has no root node to tell them apart. + * Apple only: {@link showsFirstScreen} reads the Application and Window roots of an Apple tree, and + * an Android tree has no root node to tell them apart. */ export function markPostOpenStabilization( session: SessionState, @@ -253,6 +254,28 @@ export function markPostOpenStabilization( }; } +/** + * Whether a launched app's tree shows its first screen: some node besides the Application and Window + * roots carries a label or value that is not a busy indicator's. Launch surfaces carry none. Measured + * on Bluesky, the storyboard splash is one unlabeled full-screen Image and the empty React Native + * root is one unlabeled view, and each can hold unchanged for longer than a second. A busy indicator + * is labeled ("In progress") but says only that the screen is not ready. No node count or quiet + * window is needed: a one-button screen is ready at once, and a two-image splash never is. + */ +function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean { + return nodes.some( + (node) => + !isViewportRootNode(node) && + Boolean(node.label?.trim() || node.value?.trim()) && + !BUSY_INDICATOR_ROLES.has(formatRole(node.type ?? '')), + ); +} + +const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([ + 'activity-indicator', + 'progress-indicator', +]); + function clearSurfaceStabilization(session: SessionState | undefined): void { if (session?.deferredSurface?.kind !== 'stabilize') return; session.deferredSurface = undefined; @@ -392,8 +415,7 @@ export async function capturePostGestureStabilizedResult(params: { classifyBaselineEvidence: classifyPostGestureBaselineEvidence, surfacesIdentical: haveIdenticalDiscriminatingSurfaces, summarizeDivergence: summarizeDiscriminatingSurfaceDivergence, - countFirstScreenContent: (value) => - readSnapshot(value).nodes.filter((node) => !isViewportRootNode(node)).length, + showsFirstScreen: (value) => showsFirstScreen(readSnapshot(value).nodes), }, }); } finally { diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f513d0b35b..5bd75df864 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -98,7 +98,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. -- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; other devices and backends skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree that holds only the app root, such as a launch or JavaScript loading screen, does not count, even when its producer marks it sparse (`snapshotQuality.state: "sparse"`). A tree with four or fewer other nodes counts after it stays unchanged for 1.5 seconds; a larger tree counts when two reads agree. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. +- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; other devices and backends skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree counts as the first screen when an element other than the app root has a label or value, and two reads in a row agree. A splash image, an empty view, or only a busy indicator does not count, however long it stays unchanged. An app whose first screen has no labeled element, such as a game canvas, waits the full 10 seconds. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. From d67b371702d9661582e9990a5702ae6c609bbf7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 11:04:46 +0100 Subject: [PATCH 14/15] test(daemon): name the splash wait test for its bounded behavior Co-Authored-By: Claude Opus 5.5 --- .../src/core/__tests__/deferred-interaction-outcome.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index 6b01b78652..e25b82700f 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -201,7 +201,7 @@ const APP_ROOT = { rect: { x: 0, y: 0, width: 390, height: 844 }, }; -test('the first capture after an open waits through a splash image however long it holds', async () => { +test('the first capture after an open keeps waiting through a splash image that holds unchanged', async () => { vi.useFakeTimers(); const session = makeSession('ios'); markPostOpenStabilization(session, 'observable'); From 72afa34d522a3586878cedd01b52289b131bc5b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 11:18:51 +0100 Subject: [PATCH 15/15] fix(mcp): keep the first-screen wait out of the always-loaded open schema Merged with main, the always-loaded tools were 433 bytes over the 22 KB budget. The open summary and the foreground input text return to their main wording; the CLI usage line and the docs keep the 10s wait, and a timed-out wait still carries its own warning in the response. Co-Authored-By: Claude Opus 5.5 --- packages/command-registry/src/flag-definitions-action.ts | 4 ++-- packages/command-surface/src/management/app.ts | 2 +- packages/contracts/src/client-app.ts | 7 +++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 612eae2788..0f15125d2e 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -411,9 +411,9 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--foreground', usageDescription: - 'open: keep normal app/device selection and return an initial snapshot of the first screen; without an app, resolve the sole running app on the sole booted iOS simulator', + 'open: keep normal app/device selection and return an initial snapshot of the first screen (a local iOS simulator waits up to 10s for it); without an app, resolve the sole running app on the sole booted iOS simulator', inputDescription: - 'Include an initial interactive snapshot in a fresh open response. On a local iOS simulator it waits up to 10s for the app\'s first screen; on timeout it carries postGestureOutcome.kind "loading" or "unsettled" and a warning. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', + 'Include an initial interactive snapshot in a fresh open response. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', projectConfig: false, recorded: false, }, diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index d7d1d6e438..659b12319c 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -43,7 +43,7 @@ const appsCommandMetadata = defineFieldCommandMetadata( const openCommandMetadata = defineFieldCommandMetadata( 'open', - 'Boot the selected device when needed, then open an app, deep link, or URL in a session. With a known app, start with open and foreground: a fresh open returns the initial interactive snapshot with refs (or initialSnapshotError: then run snapshot). Do not probe with devices or apps first; list them only for an unknown app id, and never guess one. The app or URL inputs select the foreground automation target. On a local iOS simulator, the first capture after a launch waits up to 10s for a first screen with a labeled element, not a splash or loading tree; other devices and backends skip the wait. On timeout it returns what it read, warning with postGestureOutcome.kind "loading" or "unsettled". Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', + 'Boot the selected device when needed, then open an app, deep link, or URL in a session. With a known app, start with open and foreground: a fresh open returns the initial interactive snapshot with refs (or initialSnapshotError: then run snapshot). Do not probe with devices or apps first; list them only for an unknown app id, and never guess one. The app or URL inputs select the foreground automation target. Remote daemons refuse file: URLs in open targets and launch URLs with reason REMOTE_HOST_INPUT_DENIED. Local web content is supported over HTTP(S).', { app: stringField('App name, bundle id, package, or URL.'), url: stringField('Optional URL passed with an app shell.'), diff --git a/packages/contracts/src/client-app.ts b/packages/contracts/src/client-app.ts index c7072bfc36..a462387ed7 100644 --- a/packages/contracts/src/client-app.ts +++ b/packages/contracts/src/client-app.ts @@ -90,10 +90,9 @@ export type AppOpenOptions = AgentDeviceRequestOverrides & // `--foreground` FlagDefinition's `inputDescription`) verbatim, pinned to it by // `commands/command-input-option-field.test.ts`. /** - * Include an initial interactive snapshot in a fresh open response. On a local iOS simulator it - * waits up to 10s for the app's first screen; on timeout it carries postGestureOutcome.kind - * "loading" or "unsettled" and a warning. With no app argument, discover the sole running app - * on the sole booted iOS simulator; ambiguous environments fail closed. + * Include an initial interactive snapshot in a fresh open response. With no + * app argument, discover the sole running app on the sole booted iOS + * simulator; ambiguous environments fail closed. */ foreground?: boolean; saveScript?: boolean | string;