diff --git a/packages/capture-kit/src/post-gesture-stability.test.ts b/packages/capture-kit/src/post-gesture-stability.test.ts index 04c0db917d..d0f30d73fe 100644 --- a/packages/capture-kit/src/post-gesture-stability.test.ts +++ b/packages/capture-kit/src/post-gesture-stability.test.ts @@ -3,6 +3,7 @@ import { beforeEach, test, vi } from 'vitest'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { ObservationClock } from './observe-until.ts'; import { + formatPostGestureOutcomeWarning, runPostGestureStabilityLoop, type PostGestureBaseline, type PostGesturePreActionBaseline, @@ -20,7 +21,13 @@ beforeEach(() => { vi.mocked(emitDiagnostic).mockClear(); }); -type Surface = Readonly<{ signature: readonly string[]; costMs: number; backend?: string }>; +type Surface = Readonly<{ + signature: readonly string[]; + costMs: number; + backend?: string; + /** Whether this read shows a launched app's first screen; absent reads count as one. */ + firstScreen?: boolean; +}>; // The negative half of these lines is type-level: each @ts-expect-error fails `pnpm typecheck` // the moment the baseline union would accept a post-action reading as pre-action evidence again @@ -93,6 +100,7 @@ function hooksFor( same(baseline.signature, quiet.signature) ? 'unchanged' : 'changed', surfacesIdentical: same, summarizeDivergence: () => ({}), + showsFirstScreen: (value) => value.firstScreen ?? true, }; } @@ -218,3 +226,93 @@ test('an effect that lands after a rebase keeps the loop polling until the moved assert.equal(diagnostics('post_gesture_snapshot_baseline_rebased').length, 1); assert.equal(diagnostics('post_gesture_snapshot_stale_accept').length, 0); }); + +// --- first-screen readiness after an app launch --- + +const LAUNCH = { action: 'open', positionals: [], readiness: 'first-screen' as const }; + +function launchSurface(name: string, firstScreen: boolean, costMs = 50): Surface { + return { signature: [name], costMs, firstScreen }; +} + +test('a launch keeps polling through a splash that holds unchanged until the first screen appears', async () => { + const clock = fakeClock(); + const splash = launchSurface('splash', false); + const home = launchSurface('home', true); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + // Ten reads a 200 ms poll apart: the splash holds unchanged for over 2 s. + hooks: hooksFor(clock, [...Array.from({ length: 10 }, () => splash), home, home]), + clock, + }); + + assert.equal(outcome.value, home); + assert.equal(outcome.postGestureOutcome, undefined); + assert.ok(clock.now() > 2_000, `a quiet splash settled after ${clock.now()}ms`); +}); + +test('a launch returns on the first quiet pair that shows the first screen', async () => { + const clock = fakeClock(); + const home = launchSurface('home', true); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, [home, home]), + clock, + }); + + assert.equal(outcome.value, home); + assert.equal(outcome.postGestureOutcome, undefined); + assert.ok(clock.now() < 500, `settled after ${clock.now()}ms`); +}); + +test('a launch whose budget expires on a loading tree reports the app as loading', async () => { + const clock = fakeClock(); + const loading = launchSurface('loading', false); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor( + clock, + Array.from({ length: 60 }, () => loading), + ), + clock, + }); + + assert.equal(outcome.value, loading); + assert.deepEqual(outcome.postGestureOutcome, { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }); + assert.ok(clock.now() >= 10_000 && clock.now() < 10_500, `gave up after ${clock.now()}ms`); + assert.match(formatPostGestureOutcomeWarning(outcome.postGestureOutcome!), /still loading/); +}); + +test('a launch whose first screen never quiets reports it unsettled, not loading', async () => { + const clock = fakeClock(); + const frames = Array.from({ length: 60 }, (_, frame) => launchSurface(`frame ${frame}`, true)); + + const outcome = await runPostGestureStabilityLoop({ + pending: LAUNCH, + hooks: hooksFor(clock, frames), + clock, + }); + + assert.equal(outcome.postGestureOutcome?.kind, 'unsettled'); + assert.ok(clock.now() >= 10_000, `gave up after ${clock.now()}ms`); +}); + +test('a gesture record never applies the first-screen rule to an empty tree', async () => { + const clock = fakeClock(); + const empty = launchSurface('empty', false); + + const outcome = await runPostGestureStabilityLoop({ + pending: PENDING, + hooks: hooksFor(clock, [empty, empty]), + clock, + }); + + assert.equal(outcome.value, empty); + assert.equal(outcome.postGestureOutcome, undefined); +}); diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 8bde5bb8cd..b46bc7b9b9 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -41,6 +41,18 @@ const POST_GESTURE_STABILITY_SCHEDULE: ObservationSchedule = { */ const STABILIZATION_DISTRUST_DEADLINE_MS = POST_GESTURE_STABILITY_SCHEDULE.budgetMs + 2_000; +/** + * Cadence and budget for a launched app's first screen. A launch shows a splash or a loading + * surface before its first screen, and a JavaScript bundle can take seconds to arrive, so the + * budget is wider than a gesture's. It bounds when a poll may start; a slow remote capture that + * starts in time is still judged. + */ +const FIRST_SCREEN_SCHEDULE: ObservationSchedule = { + intervalMs: POST_GESTURE_STABILITY_SCHEDULE.intervalMs, + budgetMs: 10_000, + minPolls: 2, +}; + export type BaselineSurfaceEvidence = 'changed' | 'unchanged' | 'ambiguous'; /** @@ -86,6 +98,11 @@ export type PostGestureStabilityPending = { positionals: string[]; /** Absent when the writer stored no baseline (non-Apple, or nothing usable pre-gesture). */ baseline?: PostGesturePreActionBaseline; + /** + * `first-screen`: an app launch marked this record. A quiet pair settles the loop only when + * `showsFirstScreen` accepts it, under {@link FIRST_SCREEN_SCHEDULE}. + */ + readiness?: 'first-screen'; }; export type PostGestureStabilityHooks = { @@ -106,6 +123,12 @@ export type PostGestureStabilityHooks = { surfacesIdentical: (baseline: S, current: S) => boolean; /** Why `surfacesIdentical` said no, for the veto diagnostic. */ summarizeDivergence: (baseline: S, current: S) => Record; + /** + * Whether one attempt shows a launched app's first screen rather than its launch or loading + * surface. The producer owns the rule because only it can read the tree. Read only for a launch + * record. + */ + showsFirstScreen: (value: T) => boolean; }; export type PostGestureStabilityOutcome = { @@ -129,6 +152,9 @@ export type PostGestureStabilityOutcome = { */ export function formatPostGestureOutcomeWarning({ kind, gesture }: PostGestureOutcome): string { const named = [gesture.action, ...gesture.positionals].join(' ').trim(); + if (kind === 'loading') { + return `The app was still loading after ${named} when this tree was read: it may show a launch or loading screen. Run snapshot -i again, or wait for expected content (wait text ...) before acting.`; + } return kind === 'unsettled' ? `The surface was still changing after ${named} when this tree was read, so it may not match where the surface comes to rest: an element missing from it is not proof of absence.` : `${named} produced no visible change: the tree still matches its pre-gesture state. ` + @@ -193,6 +219,7 @@ export async function runPostGestureStabilityLoop | undefined; const surfaceOf = (value: T): CapturedSurface => { @@ -204,7 +231,7 @@ export async function runPostGestureStabilityLoop>({ ...(params.initial !== undefined ? { initial: params.initial } : {}), capture: () => hooks.capture(), - schedule: POST_GESTURE_STABILITY_SCHEDULE, + schedule: firstScreen ? FIRST_SCREEN_SCHEDULE : POST_GESTURE_STABILITY_SCHEDULE, ...(clock ? { clock } : {}), verdict: (latest, previousValue) => { attempts += 1; @@ -216,6 +243,7 @@ export async function runPostGestureStabilityLoop { + const postGestureOutcome: PostGestureOutcome = { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }; + const data = serializeSnapshotResult({ + nodes: [], + truncated: false, + postGestureOutcome, + identifiers: { session: 'qa' }, + }); + + assert.deepEqual(data.postGestureOutcome, postGestureOutcome); +}); + +test('serializeSnapshotResult omits the post-gesture outcome of a tree read at rest', () => { + const data = serializeSnapshotResult({ + nodes: [], + truncated: false, + identifiers: { session: 'qa' }, + }); + + assert.equal('postGestureOutcome' in data, false); +}); + test('serializeSnapshotResult retains clipping notes with or without a capture verdict', () => { const snapshotQualityNotes = [ { code: 'regular-geometry-clipped' as const, clippedNodeCount: 1, droppedNodeCount: 1 }, diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 284edc3106..0f15125d2e 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -411,7 +411,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--foreground', usageDescription: - 'open: keep normal app/device selection and return an initial snapshot; without an app, resolve the sole running app on the sole booted iOS simulator', + 'open: keep normal app/device selection and return an initial snapshot of the first screen (a local iOS simulator waits up to 10s for it); without an app, resolve the sole running app on the sole booted iOS simulator', inputDescription: 'Include an initial interactive snapshot in a fresh open response. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.', projectConfig: false, diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts index a2320dc896..0b8b362742 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts @@ -851,6 +851,39 @@ test('runtime snapshot warns when its tree was read on a surface still moving af const result = await device.capture.snapshot({ session: 'default' }); assert.deepEqual(result.warnings, [formatPostGestureOutcomeWarning(outcome)]); + assert.deepEqual(result.postGestureOutcome, outcome); +}); + +test('runtime snapshot publishes a launched app that was still loading as its outcome', async () => { + const outcome: PostGestureOutcome = { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }; + const device = createSnapshotOnlyDevice({ + snapshot: { + ...makeSnapshotState([{ index: 0, depth: 0, type: 'Application', label: 'Demo' }], { + backend: 'xctest', + }), + postGestureOutcome: outcome, + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.deepEqual(result.postGestureOutcome, outcome); + assert.deepEqual(result.warnings, [formatPostGestureOutcomeWarning(outcome)]); +}); + +test('runtime snapshot publishes no outcome for a tree read at rest', async () => { + const device = createSnapshotOnlyDevice({ + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'xctest', + }), + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal('postGestureOutcome' in result, false); }); test('runtime snapshot warns when a gesture produced no visible change', async () => { diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 6ecd70a477..73abc8e6b5 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -19,7 +19,7 @@ import type { SnapshotViewportSize, SnapshotVisibility, } from '@agent-device/kernel/snapshot'; -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { PostGestureOutcome, SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { BackendSnapshotResult } from '@agent-device/contracts/backend'; import type { AgentDeviceRuntime, @@ -76,6 +76,11 @@ export type SnapshotCommandResult = { * non-iOS backend — publishes nothing. */ presentedSurface?: PresentedSurface; + /** + * What the post-gesture or post-open wait concluded about this tree, when it found the surface + * not at rest: still changing, unchanged by the gesture, or, after an app launch, still loading. + */ + postGestureOutcome?: PostGestureOutcome; } & PublicSnapshotCaptureAnnotations; type SnapshotCapture = { @@ -123,6 +128,9 @@ export const snapshotCommand: RuntimeCommand< ...(capture.snapshot.presentedSurface ? { presentedSurface: capture.snapshot.presentedSurface } : {}), + ...(capture.snapshot.postGestureOutcome + ? { postGestureOutcome: capture.snapshot.postGestureOutcome } + : {}), ...snapshotAppFields(capture), }); }; diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 7ee8da96b7..1afdf01cf2 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -14,6 +14,7 @@ import type { SnapshotVisibility, } from '@agent-device/kernel/snapshot'; import type { ScreenshotResultData } from './snapshot-types.ts'; +import type { PostGestureOutcome } from './snapshot-model.ts'; import type { AgentDeviceIdentifiers, AgentDeviceRequestOverrides, @@ -83,6 +84,12 @@ export type CaptureSnapshotResult = { * of the field means a non-iOS producer or an older runner published nothing. */ presentedSurface?: PresentedSurface; + /** + * Present when the tree may not show the surface at rest: `unsettled` (still changing after a + * gesture), `no-effect` (a gesture changed nothing), or `loading` (the app launched by `open` + * still showed no first screen when its wait ran out). + */ + postGestureOutcome?: PostGestureOutcome; /** * Screenshot captured automatically when the semantic snapshot was sparse. * Remote clients receive a materialized local path through the daemon artifact channel. diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts index 5e5ab04ea5..85e86560bc 100644 --- a/packages/contracts/src/daemon-command-surface.ts +++ b/packages/contracts/src/daemon-command-surface.ts @@ -8,6 +8,7 @@ import type { ArtifactDescriptor, FileOutputRef } from './artifact-adapter.ts'; import type { DeviceRotation } from './device-rotation.ts'; import type { ClickButton } from './click-button.ts'; import type { DiffSnapshotCommandResult } from './diff.ts'; +import type { PostGestureOutcome } from './snapshot-model.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; import type { PublicSnapshotCaptureAnnotations } from './snapshot-capture-annotations.ts'; import type { ReplayTargetGuardDenotation } from './replay.ts'; @@ -82,6 +83,7 @@ export type DaemonSnapshotResult = Readonly< snapshotDiagnostics?: SnapshotDiagnosticsSummary; keyboard?: SnapshotKeyboardBandFact; viewport?: SnapshotViewportSize; + postGestureOutcome?: PostGestureOutcome; } & PublicSnapshotCaptureAnnotations >; diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index bb7434b860..cb1922bc69 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -11,6 +11,8 @@ export { isSnapshotBackend, usesMobileSnapshotPresentation, inheritPostGestureOutcome, + isPostGestureRestUnproven, + readPostGestureOutcome, snapshotRefMinting, buildSnapshotNodeMap, findSnapshotAncestor, diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 30dc392063..b6010dbc8f 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -186,7 +186,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS Simulator open that did not read the launched app (any postOpenObservation other than observable) marks the next capture in that session to wait for a quiet surface (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; diff --git a/packages/contracts/src/snapshot-model.test.ts b/packages/contracts/src/snapshot-model.test.ts index bb210d6e6a..b439308ec6 100644 --- a/packages/contracts/src/snapshot-model.test.ts +++ b/packages/contracts/src/snapshot-model.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest'; import { + readPostGestureOutcome, snapshotStateProvenance, type SnapshotProvenance, type SnapshotStateProvenance, @@ -31,3 +32,16 @@ test('snapshotStateProvenance extracts exactly the pair', () => { producer: 'revyl-ios-tree', }); }); + +test('readPostGestureOutcome reads every outcome kind off the wire and drops malformed values', () => { + const gesture = { action: 'open', positionals: [] }; + for (const kind of ['unsettled', 'no-effect', 'loading'] as const) { + expect(readPostGestureOutcome({ kind, gesture })).toEqual({ kind, gesture }); + } + expect(readPostGestureOutcome({ kind: 'toString', gesture })).toBeUndefined(); + expect(readPostGestureOutcome({ kind: 'loading', gesture: { action: 'open' } })).toBeUndefined(); + expect( + readPostGestureOutcome({ kind: 'loading', gesture: { action: 'open', positionals: [1] } }), + ).toBeUndefined(); + expect(readPostGestureOutcome('loading')).toBeUndefined(); +}); diff --git a/packages/contracts/src/snapshot-model.ts b/packages/contracts/src/snapshot-model.ts index 3af3071940..e472018cca 100644 --- a/packages/contracts/src/snapshot-model.ts +++ b/packages/contracts/src/snapshot-model.ts @@ -147,12 +147,41 @@ export type PostGestureAction = { action: string; positionals: string[] }; /** * `unsettled`: the surface was still changing when the stabilization deadline expired. * `no-effect`: the settled surface still matches the pre-gesture tree (#1600). + * `loading`: an app launch's budget expired before its tree showed a first screen; the tree may be + * a launch or loading surface. */ export type PostGestureOutcome = { - kind: 'unsettled' | 'no-effect'; + kind: 'unsettled' | 'no-effect' | 'loading'; gesture: PostGestureAction; }; +const POST_GESTURE_OUTCOME_KINDS: Readonly> = { + unsettled: true, + 'no-effect': true, + loading: true, +}; + +/** Reads a published post-gesture outcome off the wire; anything malformed is no outcome. */ +export function readPostGestureOutcome(value: unknown): PostGestureOutcome | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const { kind, gesture } = value as Record; + if (typeof kind !== 'string' || !Object.hasOwn(POST_GESTURE_OUTCOME_KINDS, kind)) + return undefined; + if (typeof gesture !== 'object' || gesture === null) return undefined; + const { action, positionals } = gesture as Record; + if (typeof action !== 'string' || !Array.isArray(positionals)) return undefined; + if (!positionals.every((positional) => typeof positional === 'string')) return undefined; + return { kind: kind as PostGestureOutcome['kind'], gesture: { action, positionals } }; +} + +/** + * Whether a tree with this outcome may not show the surface at rest: it was still changing, or the + * launched app was still loading. Such a tree proves no absence and serves no cache. + */ +export function isPostGestureRestUnproven(outcome: PostGestureOutcome | undefined): boolean { + return outcome?.kind === 'unsettled' || outcome?.kind === 'loading'; +} + /** * A capture taken at once to recover or widen `previous` reads the same moment after the same * gesture, so it carries that capture's outcome. diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index 6fc31fe45b..e25b82700f 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -10,12 +10,35 @@ import { type DeferredOutcomeSnapshotAttempt, } from '../deferred-interaction-outcome.ts'; import type { SessionState } from '../session-state.ts'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { deliverySnapshot, makeSession, pickupSnapshot, } from './post-gesture-stabilization-fixtures.ts'; +/** A first screen: a header and five rows. */ +function homeScreenSnapshot(y = 500) { + return makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Home', + rect: { x: 20, y: 60, width: 200, height: 30 }, + }, + ...Array.from({ length: 5 }, (_, row) => ({ + index: row + 2, + parentIndex: 0, + type: 'Button', + label: `Row ${row}`, + rect: { x: 20, y: y + row * 50, width: 300, height: 44 }, + })), + ]); +} + afterEach(() => { vi.useRealTimers(); }); @@ -110,26 +133,225 @@ test('an Apple open that could not observe its app marks stabilization without a assert.equal(pendingSurfaceStabilization(session)?.baseline?.signature, undefined); }); -test('every open verdict other than observable marks stabilization', () => { - const pending = ( - [ - 'unobservable', - 'probe-failed', - 'app-unidentified', - 'not-eligible', - 'observable', - undefined, - ] as const - ).map((observation) => { +test('every Apple open that launched an app marks a first-screen wait, and an Android open none', () => { + const readiness = (platform: 'ios' | 'android') => + ( + [ + 'unobservable', + 'probe-failed', + 'app-unidentified', + 'not-eligible', + 'observable', + undefined, + ] as const + ).map((observation) => { + const session = makeSession(platform); + markPostOpenStabilization(session, observation); + return pendingSurfaceStabilization(session)?.readiness; + }); + const launched = Array.from({ length: 5 }, () => 'first-screen'); + + assert.deepEqual(readiness('ios'), [...launched, undefined]); + assert.deepEqual( + readiness('android'), + Array.from({ length: 6 }, () => undefined), + ); +}); + +test('the first capture after an open waits through a loading tree for the first screen', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const loading = makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ]); + const home = homeScreenSnapshot(); + const { capture } = scriptedCapture([loading, loading, loading, loading, loading, home, home]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(3_000); + const result = await pendingResult; + + assert.equal(result?.snapshot, home); + assert.equal(result?.snapshot.postGestureOutcome, undefined); + assert.equal(isPostGestureStabilizationPending(session), false); +}); + +test('the first capture after an open does not count the Window root as first-screen content', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const rootsOnly = makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { index: 1, parentIndex: 0, type: 'Window', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ]); + const { capture } = scriptedCapture([rootsOnly]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(11_000); + const result = await pendingResult; + + assert.equal(result?.snapshot.postGestureOutcome?.kind, 'loading'); +}); + +const APP_ROOT = { + index: 0, + type: 'Application', + label: 'App', + rect: { x: 0, y: 0, width: 390, height: 844 }, +}; + +test('the first capture after an open keeps waiting through a splash image that holds unchanged', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + // The storyboard splash a launched React Native app shows: one unlabeled full-screen image. + const splash = makeSnapshotState([ + APP_ROOT, + { + index: 1, + parentIndex: 0, + type: 'Image', + identifier: 'SplashScreenLegacy', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ]); + const home = homeScreenSnapshot(); + const { capture } = scriptedCapture([...Array.from({ length: 12 }, () => splash), home, home]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(4_000); + const result = await pendingResult; + + assert.equal(result?.snapshot, home); + assert.equal(result?.snapshot.postGestureOutcome, undefined); +}); + +test('the first capture after an open does not take a busy indicator for the first screen', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const spinner = makeSnapshotState([ + APP_ROOT, + { + index: 1, + parentIndex: 0, + type: 'ActivityIndicator', + label: 'In progress', + rect: { x: 180, y: 400, width: 30, height: 30 }, + }, + ]); + const { capture } = scriptedCapture([spinner]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(11_000); + const result = await pendingResult; + + assert.equal(result?.snapshot.postGestureOutcome?.kind, 'loading'); +}); + +test('the first capture after an open settles a one-button first screen on its first quiet pair', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const oneButton = makeSnapshotState([ + APP_ROOT, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 20, y: 700, width: 350, height: 50 }, + }, + ]); + const { capture, calls } = scriptedCapture([oneButton, oneButton]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pendingResult; + + assert.equal(calls(), 2); + assert.equal(result?.snapshot.postGestureOutcome, undefined); +}); + +test('the first capture after an open waits through a root-only tree whatever its quality verdict', async () => { + const root = makeSnapshotState([ + { index: 0, type: 'Application', label: 'App', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ]); + const rootOnlyReads: SnapshotState[] = [ + // The runner's verdict on a tree holding only its roots. + { ...root, snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' } }, + // A runner that predates verdicts publishes the same tree with none. + { nodes: root.nodes, createdAt: root.createdAt, backend: 'xctest' }, + ]; + for (const rootOnly of rootOnlyReads) { + vi.useFakeTimers(); const session = makeSession('ios'); - markPostOpenStabilization(session, observation); - return isPostGestureStabilizationPending(session); + markPostOpenStabilization(session, 'observable'); + const home = homeScreenSnapshot(); + const { capture } = scriptedCapture([rootOnly, rootOnly, rootOnly, rootOnly, home, home]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(3_000); + const result = await pendingResult; + + assert.equal(result?.snapshot, home); + assert.equal(result?.snapshot.postGestureOutcome, undefined); + } +}); + +test('the first capture after an open settles a sparse read that holds content beyond its roots', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const sparse = { + ...homeScreenSnapshot(), + snapshotQuality: { state: 'sparse', backend: 'tree', reasonCode: 'budget' } as const, + }; + const { capture, calls } = scriptedCapture([sparse, sparse]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pendingResult; + + assert.equal(calls(), 2); + assert.equal(result?.snapshot.postGestureOutcome, undefined); +}); + +test('an open supersedes the stabilization a scroll still owed on the screen it replaced', () => { + const session = makeSession('ios'); + session.snapshot = homeScreenSnapshot(); + markDeferredInteractionOutcome({ + session, + command: 'scroll', + positionals: ['down'], + flags: undefined, }); - const android = makeSession('android'); - markPostOpenStabilization(android, 'probe-failed'); + assert.ok(pendingSurfaceStabilization(session)?.baseline); + + markPostOpenStabilization(session, 'observable'); + + const pending = pendingSurfaceStabilization(session); + assert.equal(pending?.action, 'open'); + assert.equal(pending?.readiness, 'first-screen'); + assert.equal(pending?.baseline, undefined); +}); + +test('the first capture after an open marks a tree that never showed a first screen as loading', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + markPostOpenStabilization(session, 'observable'); + const { capture } = scriptedCapture([makeSnapshotState([])]); + + const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); + await vi.advanceTimersByTimeAsync(11_000); + const result = await pendingResult; - assert.deepEqual(pending, [true, true, true, true, false, false]); - assert.equal(isPostGestureStabilizationPending(android), false); + assert.equal(result?.snapshot.nodes.length, 0); + assert.deepEqual(result?.snapshot.postGestureOutcome, { + kind: 'loading', + gesture: { action: 'open', positionals: [] }, + }); }); test('the first capture after an unobserved open reads the list where it came to rest', async () => { @@ -137,9 +359,9 @@ test('the first capture after an unobserved open reads the list where it came to const session = makeSession('ios'); markPostOpenStabilization(session, 'unobservable'); const { capture, calls } = scriptedCapture([ - pickupSnapshot(319), - pickupSnapshot(406), - pickupSnapshot(406), + homeScreenSnapshot(319), + homeScreenSnapshot(406), + homeScreenSnapshot(406), ]); const pendingResult = resolveDeferredInteractionOutcome(resolveParams(session, capture)); @@ -149,7 +371,7 @@ test('the first capture after an unobserved open reads the list where it came to const result = await pendingResult; assert.equal(calls(), 3); - assert.equal(result?.snapshot.nodes[1]?.rect?.y, 406); + assert.equal(result?.snapshot.nodes[2]?.rect?.y, 406); assert.equal(result?.snapshot.postGestureOutcome, undefined); assert.equal(isPostGestureStabilizationPending(session), false); }); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index b286dde98b..821a60bd93 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -3,6 +3,7 @@ import type { SettleObservation, SurfaceScopedNodes } from '@agent-device/contra import type { PostOpenObservation } from '@agent-device/contracts/application-lifecycle-runtime'; import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device'; +import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract'; import { @@ -26,6 +27,7 @@ import { runPostGestureStabilityLoop, type PostGestureStabilityOutcome, } from '@agent-device/capture-kit/post-gesture-stability'; +import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import type { DeferredSurface, SessionState } from './session-state.ts'; /** @@ -222,12 +224,19 @@ export function movingTargetBaseline( } /** - * Called once an app open has published its session. A verdict other than `observable` means the - * open never read the launched app: the probe failed or could not identify it, the device has no - * observation path, or the app stayed unreadable. Nothing proved its layout at rest, so the next - * capture runs the quiet-window loop (#3354, #3367). An open that reports no verdict does not mark. - * No baseline: the pre-open tree is a different surface, so the open is never reported as having no - * effect. + * Called once an Apple app open has published its session. Every verdict means the open launched an + * app, and none proves its first screen: an `observable` app can still show its launch or loading + * surface. The next capture therefore waits for the first screen (#3354, #3367). An open that + * reports no verdict launched no app, such as a URL or deep link, and does not mark. No baseline: the + * pre-open tree is a different surface, so the open is never reported as having no effect. + * + * The record replaces whatever the session's deferred surface held. A scroll's owed stabilization + * judged the scroll against its pre-action tree, and a moving-target baseline describes where the + * previous action's targets were. The open launched or foregrounded an app, so both describe a screen + * the open replaced: a verdict against them would charge the open's effect to the earlier action. + * + * Apple only: {@link showsFirstScreen} reads the Application and Window roots of an Apple tree, and + * an Android tree has no root node to tell them apart. */ export function markPostOpenStabilization( session: SessionState, @@ -235,15 +244,38 @@ export function markPostOpenStabilization( ): void { if (!isApplePlatform(session.device.platform)) return; if (!supportsPostGestureStabilization(session.device)) return; - if (observation === undefined || observation === 'observable') return; + if (observation === undefined) return; session.deferredSurface = { kind: 'stabilize', action: 'open', positionals: [], markedAt: Date.now(), + readiness: 'first-screen', }; } +/** + * Whether a launched app's tree shows its first screen: some node besides the Application and Window + * roots carries a label or value that is not a busy indicator's. Launch surfaces carry none. Measured + * on Bluesky, the storyboard splash is one unlabeled full-screen Image and the empty React Native + * root is one unlabeled view, and each can hold unchanged for longer than a second. A busy indicator + * is labeled ("In progress") but says only that the screen is not ready. No node count or quiet + * window is needed: a one-button screen is ready at once, and a two-image splash never is. + */ +function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean { + return nodes.some( + (node) => + !isViewportRootNode(node) && + Boolean(node.label?.trim() || node.value?.trim()) && + !BUSY_INDICATOR_ROLES.has(formatRole(node.type ?? '')), + ); +} + +const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([ + 'activity-indicator', + 'progress-indicator', +]); + function clearSurfaceStabilization(session: SessionState | undefined): void { if (session?.deferredSurface?.kind !== 'stabilize') return; session.deferredSurface = undefined; @@ -366,6 +398,7 @@ export async function capturePostGestureStabilizedResult(params: { action: pending.action, positionals: pending.positionals ?? [], baseline: pending.baseline, + ...(pending.readiness ? { readiness: pending.readiness } : {}), }, initial: params.initial, hooks: { @@ -382,6 +415,7 @@ export async function capturePostGestureStabilizedResult(params: { classifyBaselineEvidence: classifyPostGestureBaselineEvidence, surfacesIdentical: haveIdenticalDiscriminatingSurfaces, summarizeDivergence: summarizeDiscriminatingSurfaceDivergence, + showsFirstScreen: (value) => showsFirstScreen(readSnapshot(value).nodes), }, }); } finally { diff --git a/packages/daemon/src/core/session-state.ts b/packages/daemon/src/core/session-state.ts index 439ddaf3d4..b93b1cded9 100644 --- a/packages/daemon/src/core/session-state.ts +++ b/packages/daemon/src/core/session-state.ts @@ -107,6 +107,8 @@ export type DeferredSurface = * marker carries no baseline at all. */ baseline?: PostGesturePreActionBaseline; + /** `first-screen`: an app launch marked this record; the loop waits for its first screen. */ + readiness?: 'first-screen'; } | { kind: 'recheck'; diff --git a/packages/daemon/src/execution/selector-capture-runtime.ts b/packages/daemon/src/execution/selector-capture-runtime.ts index 51da158f5a..550e57a78d 100644 --- a/packages/daemon/src/execution/selector-capture-runtime.ts +++ b/packages/daemon/src/execution/selector-capture-runtime.ts @@ -1,7 +1,11 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { BackendSnapshotResult } from '@agent-device/contracts/backend'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; -import { inheritPostGestureOutcome, type SnapshotState } from '@agent-device/contracts/snapshot'; +import { + inheritPostGestureOutcome, + isPostGestureRestUnproven, + type SnapshotState, +} from '@agent-device/contracts/snapshot'; import { snapshotPresentationOptionsFromFlags } from '@agent-device/capture-kit/snapshot-state'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import type { DaemonRequest } from '../core/daemon-request.ts'; @@ -279,7 +283,7 @@ function canUseSessionSnapshotCache( if (request.cache?.useSessionSnapshot !== true) return false; if (getActiveAndroidSnapshotFreshness(session)) return false; if (shouldBypassForPostGestureStabilization(session, request)) return false; - return session.snapshot?.postGestureOutcome?.kind !== 'unsettled'; + return !isPostGestureRestUnproven(session.snapshot?.postGestureOutcome); } function isFreshSelectorSnapshot(snapshot: SnapshotState, timestamp: number): boolean { diff --git a/packages/daemon/src/server/__tests__/response-views.test.ts b/packages/daemon/src/server/__tests__/response-views.test.ts index 90eec16bb9..fe71297fb4 100644 --- a/packages/daemon/src/server/__tests__/response-views.test.ts +++ b/packages/daemon/src/server/__tests__/response-views.test.ts @@ -83,6 +83,15 @@ test('digest carries the presented-surface fact of the tree it collapsed (#3565) expect('presentedSurface' in unread).toBe(false); }); +test('digest carries the post-gesture outcome of the tree it collapsed', () => { + const postGestureOutcome = { kind: 'loading', gesture: { action: 'open', positionals: [] } }; + const digest = snapshotView!({ ...SNAPSHOT_DATA, postGestureOutcome }, 'digest'); + expect(digest.postGestureOutcome).toEqual(postGestureOutcome); + + const atRest = snapshotView!(SNAPSHOT_DATA, 'digest'); + expect('postGestureOutcome' in atRest).toBe(false); +}); + test('digest tolerates missing/empty node trees', () => { const digest = snapshotView!({ truncated: true }, 'digest'); expect(digest).toMatchObject({ nodeCount: 0, refs: [], truncated: true }); diff --git a/packages/daemon/src/server/response-views.ts b/packages/daemon/src/server/response-views.ts index 54edab34c7..8f9a9e2319 100644 --- a/packages/daemon/src/server/response-views.ts +++ b/packages/daemon/src/server/response-views.ts @@ -35,6 +35,7 @@ function snapshotView(data: DaemonResponseData, level: ResponseLevel): DaemonRes 'visibility', 'viewport', 'presentedSurface', + 'postGestureOutcome', 'snapshotQuality', 'snapshotQualityNotes', 'targetActivation', diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts b/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts index a4ca574f3a..1e219b5116 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-open-execution-runtime.test.ts @@ -932,8 +932,11 @@ test('a fresh iOS open whose observation probe failed leaves the next capture to expect(pendingSurfaceStabilization(session)?.action).toBe('open'); }); -test('a fresh iOS open that observed its app leaves nothing pending', async () => { +test('a fresh iOS open that observed its app still leaves the next capture to wait for its first screen', async () => { const session = await openReportingObservation('observed-open', 'observable'); - expect(pendingSurfaceStabilization(session)).toBeUndefined(); + expect(pendingSurfaceStabilization(session)).toMatchObject({ + action: 'open', + readiness: 'first-screen', + }); }); diff --git a/packages/selectors/src/absence-observation.test.ts b/packages/selectors/src/absence-observation.test.ts index 9e239f5187..cb24e8997a 100644 --- a/packages/selectors/src/absence-observation.test.ts +++ b/packages/selectors/src/absence-observation.test.ts @@ -6,6 +6,7 @@ import { absenceCaptureOptionRefusal, classifyAbsenceObservation, } from './absence-observation.ts'; +import { absenceObservationError } from './absence-observation-errors.ts'; test('classifies a complete capture with no selector matches as absent', () => { const observation = classifyAbsenceObservation(makeSnapshotState([]), []); @@ -13,6 +14,24 @@ test('classifies a complete capture with no selector matches as absent', () => { assert.deepEqual(observation, { kind: 'absent', matches: 0 }); }); +test('classifies a miss on a still-moving or still-loading tree by its own cause, not as absent', () => { + const miss = (kind: 'unsettled' | 'loading' | 'no-effect') => + classifyAbsenceObservation( + { + ...makeSnapshotState([]), + postGestureOutcome: { kind, gesture: { action: 'open', positionals: [] } }, + }, + [], + ); + + assert.deepEqual(miss('unsettled'), { kind: 'unsettled', matches: 0 }); + assert.deepEqual(miss('loading'), { kind: 'loading', matches: 0 }); + assert.deepEqual(miss('no-effect'), { kind: 'absent', matches: 0 }); + const error = absenceObservationError('label=Done', miss('loading')); + assert.match(error.message, /the launched app was still loading its first screen/); + assert.equal(error.details?.observation, 'loading'); +}); + test('classifies matches as present and exposes only stable first-match fields', () => { const first = makeSnapshotState([ { diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index fbccf713bf..2dd0ef0d4e 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -5,7 +5,7 @@ import { truncateToUtf8Bytes, } from '@agent-device/ad-script'; import type { SnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { PostGestureOutcome, SnapshotState } from '@agent-device/contracts/snapshot-model'; export type AbsenceFirstMatch = { id?: string; @@ -30,17 +30,28 @@ export type AbsenceObservation = | { kind: 'present'; matches: number; firstMatch: AbsenceFirstMatch } | { kind: 'sparse'; matches: number; firstMatch?: AbsenceFirstMatch; quality: SparseQuality } | { kind: 'truncated'; matches: number; firstMatch?: AbsenceFirstMatch } - | { kind: 'unsettled'; matches: 0 }; + | { kind: 'unsettled'; matches: 0 } + | { kind: 'loading'; matches: 0 }; /** Why a capture with no match still cannot prove absence. */ export const UNPROVABLE_ABSENCE_CAUSES = { sparse: 'capture was sparse', truncated: 'capture was truncated', unsettled: 'the surface was still changing after a gesture', + loading: 'the launched app was still loading its first screen', } as const; export type UnprovableAbsenceKind = keyof typeof UNPROVABLE_ABSENCE_CAUSES; +/** Which post-gesture outcomes leave a zero-match capture unable to prove absence; every kind must be listed. */ +const ABSENCE_UNPROVEN_BY_OUTCOME: Readonly< + Record +> = { + unsettled: 'unsettled', + loading: 'loading', + 'no-effect': undefined, +}; + export function isUnprovableAbsence(kind: unknown): kind is UnprovableAbsenceKind { return typeof kind === 'string' && Object.hasOwn(UNPROVABLE_ABSENCE_CAUSES, kind); } @@ -96,15 +107,16 @@ export function classifyAbsenceObservation( }, }; } - if (matchCount === 0) { - return { - kind: snapshot.postGestureOutcome?.kind === 'unsettled' ? 'unsettled' : 'absent', - matches: 0, - }; - } + if (matchCount === 0) return zeroMatchObservation(snapshot.postGestureOutcome); return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; } +/** A zero-match capture proves absence unless its post-gesture outcome left the screen unproven. */ +function zeroMatchObservation(outcome: PostGestureOutcome | undefined): AbsenceObservation { + const unproven = outcome ? ABSENCE_UNPROVEN_BY_OUTCOME[outcome.kind] : undefined; + return unproven ? { kind: unproven, matches: 0 } : { kind: 'absent', matches: 0 }; +} + /** * The one definition of "this capture is too sparse to trust": the backend's own verdict, then the * legacy iOS shape that predates verdicts. Shared with `scroll --until`, which must not stop on a diff --git a/test/integration/interaction-contract/daemon-harness.ts b/test/integration/interaction-contract/daemon-harness.ts index 240b5a6dcc..826ccc9bee 100644 --- a/test/integration/interaction-contract/daemon-harness.ts +++ b/test/integration/interaction-contract/daemon-harness.ts @@ -1,5 +1,8 @@ import { assertRpcOk } from '../provider-scenarios/assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from '../provider-scenarios/fixtures.ts'; +import { + FIRST_SCREEN_NODES, + PROVIDER_SCENARIO_IOS_SIMULATOR, +} from '../provider-scenarios/fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource, @@ -20,14 +23,6 @@ export const CONTRACT_APP = 'com.example.app'; const CONTRACT_DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; const SETTLING_CAPTURE_READS = 2; -const SETTLING_CAPTURE_NODES = [ - { - index: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, -]; /** * Provider-transcript harness for contract scenarios whose path involves the @@ -47,7 +42,7 @@ export async function withIosContractDaemon( // scenario's own conversation with the runner exactly as scripted. const transcript = createProviderTranscript([ ...Array.from({ length: SETTLING_CAPTURE_READS }, () => - runnerSnapshotEntry(SETTLING_CAPTURE_NODES), + runnerSnapshotEntry(FIRST_SCREEN_NODES), ), ...entries, ]); diff --git a/test/integration/provider-scenarios/device-pin.test.ts b/test/integration/provider-scenarios/device-pin.test.ts index ca5fb3dce4..cc64f284a0 100644 --- a/test/integration/provider-scenarios/device-pin.test.ts +++ b/test/integration/provider-scenarios/device-pin.test.ts @@ -4,6 +4,7 @@ import { createAgentDeviceClient } from '@agent-device/client'; import { runCommand } from '@agent-device/command-surface'; import type { DevicePin } from '@agent-device/contracts/device'; import { + FIRST_SCREEN_NODES, PROVIDER_SCENARIO_ANDROID, PROVIDER_SCENARIO_IOS_SIMULATOR, PROVIDER_SCENARIO_MACOS, @@ -27,13 +28,10 @@ test('Provider-backed integration admits an existing session device through its deviceId: device.id, repeat: true, result: { + // A populated first screen, so the first capture after open settles on one quiet pair. nodes: [ - { - index: 0, - type: 'Application', - label: 'Existing Session App', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, + { ...FIRST_SCREEN_NODES[0], label: 'Existing Session App' }, + ...FIRST_SCREEN_NODES.slice(1), ], truncated: false, }, diff --git a/test/integration/provider-scenarios/fixtures.ts b/test/integration/provider-scenarios/fixtures.ts index f65b058644..68e0d15264 100644 --- a/test/integration/provider-scenarios/fixtures.ts +++ b/test/integration/provider-scenarios/fixtures.ts @@ -97,3 +97,24 @@ export function createDemoIosApp(prefix: string): { tempRoot: string; appPath: s ); return { tempRoot, appPath }; } + +/** + * A populated first screen for the reads that settle the first capture after `open`. The post-open + * wait refuses a tree that holds only the app root and holds a tree of four or fewer nodes for a + * quiet window, so these reads carry five rows and settle on one quiet pair. + */ +export const FIRST_SCREEN_NODES: readonly Record[] = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + ...Array.from({ length: 5 }, (_, row) => ({ + index: row + 1, + parentIndex: 0, + type: 'StaticText', + label: `Row ${row}`, + rect: { x: 20, y: 100 + row * 50, width: 300, height: 44 }, + })), +]; diff --git a/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts index 1a27db946e..54f5802848 100644 --- a/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts +++ b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { assertRpcError, assertRpcOk } from './assertions.ts'; import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from './test-timeouts.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource, @@ -49,8 +49,8 @@ function snapshotEntry(nodes: unknown[]): ProviderScenarioProviderEntry { // The scripted open is not read back, so the first capture after it settles over a quiet pair of // reads and the direct tap path stays paused until that capture has run. const SETTLING_CAPTURE_ENTRIES = [ - snapshotEntry([APPLICATION_NODE]), - snapshotEntry([APPLICATION_NODE]), + snapshotEntry([...FIRST_SCREEN_NODES]), + snapshotEntry([...FIRST_SCREEN_NODES]), ]; async function settleAfterOpen(daemon: ProviderScenarioHarness): Promise { @@ -99,9 +99,10 @@ test('Provider-backed iOS selector wait captures directly and strips selectorCha rect: { x: 100, y: 300, width: 200, height: 44 }, }, ]); - const transcript = createProviderTranscript([continueSnapshot, continueSnapshot]); + const transcript = createProviderTranscript([...SETTLING_CAPTURE_ENTRIES, continueSnapshot]); await withDirectSelectorScenario(transcript, async (daemon) => { + await settleAfterOpen(daemon); const wait = await daemon.callCommand('wait', ['label="Continue"']); const data = assertRpcOk(wait); assert.equal(data.selector, 'label="Continue"'); @@ -122,7 +123,7 @@ test('Provider-backed iOS selector wait polls a fresh capture after a selector m }, ]); const transcript = createProviderTranscript([ - loadingSnapshot, + ...SETTLING_CAPTURE_ENTRIES, loadingSnapshot, snapshotEntry([ APPLICATION_NODE, @@ -138,6 +139,7 @@ test('Provider-backed iOS selector wait polls a fresh capture after a selector m ]); await withDirectSelectorScenario(transcript, async (daemon) => { + await settleAfterOpen(daemon); const wait = await daemon.callCommand('wait', ['label="Continue"']); const data = assertRpcOk(wait); assert.equal(data.selector, 'label="Continue"'); diff --git a/test/integration/provider-scenarios/interaction-verify.test.ts b/test/integration/provider-scenarios/interaction-verify.test.ts index 733fdd74be..090a1efccf 100644 --- a/test/integration/provider-scenarios/interaction-verify.test.ts +++ b/test/integration/provider-scenarios/interaction-verify.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -57,8 +57,10 @@ function snapshotEntry(nodes: unknown[]): ProviderScenarioProviderEntry { test('Provider-backed integration press --verify returns post-action evidence digest', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i to obtain refs; the first capture after open settles: one extra read - snapshotEntry(BEFORE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry([...FIRST_SCREEN_NODES]), + snapshotEntry([...FIRST_SCREEN_NODES]), + // snapshot -i to obtain refs snapshotEntry(BEFORE_NODES), { command: 'ios.runner.tap', @@ -92,6 +94,7 @@ test('Provider-backed integration press --verify returns post-action evidence di udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/test/integration/provider-scenarios/ios-wait-selector.test.ts b/test/integration/provider-scenarios/ios-wait-selector.test.ts index bec41aad4a..d313d119da 100644 --- a/test/integration/provider-scenarios/ios-wait-selector.test.ts +++ b/test/integration/provider-scenarios/ios-wait-selector.test.ts @@ -5,11 +5,18 @@ import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { WAIT_REASONS } from '@agent-device/contracts/wait'; import { mkdtempForTestSync } from '@agent-device/test-kit'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { assertRpcOk } from './assertions.ts'; import { createProviderScenarioHarness } from './harness.ts'; import { createRecordingAppleToolProvider } from './providers.ts'; +// Rows that make each tree a populated first screen, so the first capture after open settles on one +// quiet pair. +const FIRST_SCREEN_ROWS = FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ + ...node, + index: row + 2, +})); + test.each([ ['wait', 'selector'], ['replay', 'selector'], @@ -65,6 +72,7 @@ test.each([ label: 'Ready', rect: { x: 10, y: 10, width: 100, height: 100 }, }, + ...FIRST_SCREEN_ROWS, ], }; }, @@ -153,6 +161,7 @@ test('Apple wait text ignores a text field’s typed value but matches its label value: 'benchad', rect: { x: 16, y: 200, width: 361, height: 44 }, }, + ...FIRST_SCREEN_ROWS, ], }; }, diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index c1cee904e6..fe0a765ea9 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -423,6 +423,23 @@ export async function createIosPhysicalReinstallWorld(): Promise ({ + index: firstIndex + row, + type: 'XCUIElementTypeCell', + label, + identifier: label, + rect: { x: 16, y: 520 + row * 50, width: 360, height: 44 }, + enabled: true, + hittable: true, + })); +} + function runnerSnapshot() { return { command: 'ios.runner.snapshot', @@ -448,6 +465,7 @@ function runnerSnapshot() { enabled: true, hittable: true, }, + ...settingsRows(2), ], truncated: false, }, @@ -479,6 +497,7 @@ function generalCellSnapshot(options: { y: number }) { enabled: true, hittable: true, }, + ...settingsRows(2), ], truncated: false, }, diff --git a/test/integration/provider-scenarios/press-selector-resolution.test.ts b/test/integration/provider-scenarios/press-selector-resolution.test.ts index 0dad270e79..1bb74a3c8b 100644 --- a/test/integration/provider-scenarios/press-selector-resolution.test.ts +++ b/test/integration/provider-scenarios/press-selector-resolution.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -67,8 +67,8 @@ async function withPressSelectorDaemon( // The scripted open is not read back, so the first capture after it settles over a quiet pair of // reads. A snapshot consumes that pair here, keeping the scenario's own captures one-for-one. const runnerTranscript = createProviderTranscript([ - snapshotEntry(APPLICATION_ONLY_NODES), - snapshotEntry(APPLICATION_ONLY_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), ...entries, ]); const appleRunnerProvider = createAppleRunnerProviderFromTranscript( diff --git a/test/integration/provider-scenarios/remote-proxy-parity.test.ts b/test/integration/provider-scenarios/remote-proxy-parity.test.ts index c75187839d..d55dab3563 100644 --- a/test/integration/provider-scenarios/remote-proxy-parity.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-parity.test.ts @@ -8,7 +8,7 @@ import { test, vi } from 'vitest'; import { sendToDaemon } from '@agent-device/daemon-client'; import { AppError, type DaemonError } from '@agent-device/kernel/errors'; import { skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, type ProviderScenarioHarness } from './harness.ts'; import { createProviderScenarioProxy, @@ -89,6 +89,13 @@ function scriptedTree() { enabled: true, hittable: true, }, + // More rows make this a populated first screen, so the first capture after open settles on one + // quiet pair. + ...FIRST_SCREEN_NODES.slice(1).map((node, row) => ({ + ...node, + index: row + 2, + parentIndex: 1, + })), ], truncated: false, }; diff --git a/test/integration/provider-scenarios/scroll-movement-observation.test.ts b/test/integration/provider-scenarios/scroll-movement-observation.test.ts index 411ad06fb6..7f1edb4429 100644 --- a/test/integration/provider-scenarios/scroll-movement-observation.test.ts +++ b/test/integration/provider-scenarios/scroll-movement-observation.test.ts @@ -4,7 +4,7 @@ import type { AppleToolProvider } from '@agent-device/platform-apple/tool-provid import type { SimulatorSnapshotSource } from '@agent-device/platform-apple/snapshot-source'; import type { ExecOptions, ExecResult } from '@agent-device/host-kit/command'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -148,8 +148,10 @@ function iosSimulatorTool(): { provider: AppleToolProvider } { test('Provider-backed integration scroll down answers moved on the first post-gesture capture', async () => { const runnerTranscript = createProviderTranscript([ + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), snapshotEntry(screen(0, true)), // pre-scroll baseline - snapshotEntry(screen(0, true)), // the first capture after open settles: one extra read scrollEntry(), snapshotEntry(screen(-100, true)), // post-gesture: rows shifted into view on the first read ]); @@ -169,6 +171,7 @@ test('Provider-backed integration scroll down answers moved on the first post-ge async (daemon) => { assertRpcOk(await daemon.callCommand('open', [APP], { platform: 'ios', udid: DEVICE_ID })); assertRpcOk(await daemon.callCommand('snapshot')); + assertRpcOk(await daemon.callCommand('snapshot')); const callsBeforeScroll = runnerTranscript.calls.length; const scroll = await daemon.callCommand('scroll', ['down']); @@ -194,8 +197,10 @@ test('Provider-backed integration scroll down answers moved on the first post-ge // unscripted runner call, so either regression fails this scenario. test('Provider-backed integration scroll with postGestureStabilization false skips the movement check and the next read waits for nothing', async () => { const runnerTranscript = createProviderTranscript([ + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), snapshotEntry(screen(0, true)), // pre-scroll baseline - snapshotEntry(screen(0, true)), // the first capture after open settles: one extra read scrollEntry(), snapshotEntry(screen(-100, true)), // the only read after the gesture: the caller's own ]); @@ -216,6 +221,7 @@ test('Provider-backed integration scroll with postGestureStabilization false ski const client = daemon.client(); await client.apps.open({ app: APP, platform: 'ios', udid: DEVICE_ID }); await client.capture.snapshot(); + await client.capture.snapshot(); const callsBeforeScroll = runnerTranscript.calls.length; const scroll = await client.interactions.scroll({ diff --git a/test/integration/provider-scenarios/settle-observation.test.ts b/test/integration/provider-scenarios/settle-observation.test.ts index e37a59c67a..de76eb22f1 100644 --- a/test/integration/provider-scenarios/settle-observation.test.ts +++ b/test/integration/provider-scenarios/settle-observation.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -178,8 +178,10 @@ test( { timeout: PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS }, async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues refs; the first capture after open settles: one extra read - snapshotEntry(BEFORE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues refs snapshotEntry(BEFORE_NODES), // press label=Continue --settle: resolution capture, tap, settle captures snapshotEntry(BEFORE_NODES), @@ -211,6 +213,7 @@ test( udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, @@ -310,10 +313,12 @@ test('Provider-backed integration never-settled press --settle does not issue di }, ]; const runnerTranscript = createProviderTranscript([ - // press label=Continue --settle: resolution capture (the first capture after open settles: one extra read), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // press label=Continue --settle: resolution capture, // tap, then a settle stream that never repeats itself, so the loop can only ever time out. snapshotEntry(BEFORE_NODES), - snapshotEntry(BEFORE_NODES), tapEntry(200, 322), changingSnapshotEntry(loadingNodes), ]); @@ -340,6 +345,7 @@ test('Provider-backed integration never-settled press --settle does not issue di udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const press = await daemon.callCommand('press', ['label=Continue'], { settle: true, @@ -371,8 +377,10 @@ test('Provider-backed integration never-settled press --settle does not issue di test('Provider-backed integration modal-dismiss press --settle attaches the unchanged interactive tail', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues refs; the first capture after open settles: one extra read - snapshotEntry(MODAL_BEFORE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues refs snapshotEntry(MODAL_BEFORE_NODES), // press label=Cancel --settle: resolution capture, tap, settle captures. // The dialog closes leaving Continue in place — a removals-only diff with @@ -406,6 +414,7 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/test/integration/provider-scenarios/stale-ref-warning.test.ts b/test/integration/provider-scenarios/stale-ref-warning.test.ts index 327b0df46e..115989e3d4 100644 --- a/test/integration/provider-scenarios/stale-ref-warning.test.ts +++ b/test/integration/provider-scenarios/stale-ref-warning.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -62,7 +62,7 @@ const MAGIC_CODE_NODES = [ }, ]; -function snapshotEntry(nodes = NODES): ProviderScenarioProviderEntry { +function snapshotEntry(nodes: readonly unknown[] = NODES): ProviderScenarioProviderEntry { return { command: 'ios.runner.snapshot', deviceId: DEVICE_ID, @@ -82,8 +82,10 @@ function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { test('Provider-backed integration iOS @refs reject after a selector press replaces the tree', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues refs to the client; the first capture after open settles: one extra read - snapshotEntry(), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues refs to the client snapshotEntry(), // press label=Continue: selector resolution capture replaces the stored tree snapshotEntry(), @@ -117,6 +119,7 @@ test('Provider-backed integration iOS @refs reject after a selector press replac udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, @@ -150,9 +153,10 @@ test('Provider-backed integration iOS @refs reject after a selector press replac test('Provider-backed iOS press rejects a stale ref after navigation', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: @e3 is Verify on the magic-code screen; the first capture after open settles: - // one extra read. - snapshotEntry(MAGIC_CODE_NODES), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: @e3 is Verify on the magic-code screen. snapshotEntry(MAGIC_CODE_NODES), // press label=Back: the pre-action capture still sees that screen. snapshotEntry(MAGIC_CODE_NODES), @@ -183,6 +187,7 @@ test('Provider-backed iOS press rejects a stale ref after navigation', async () udid: DEVICE_ID, }), ); + assertRpcOk(await daemon.callCommand('snapshot')); assertRpcOk( await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/test/integration/provider-scenarios/versioned-refs.test.ts b/test/integration/provider-scenarios/versioned-refs.test.ts index 171ae7d074..cf17021ff1 100644 --- a/test/integration/provider-scenarios/versioned-refs.test.ts +++ b/test/integration/provider-scenarios/versioned-refs.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { FIRST_SCREEN_NODES, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { createAppleRunnerProviderFromTranscript, @@ -46,12 +46,12 @@ const NODES = [ }, ]; -function snapshotEntry(): ProviderScenarioProviderEntry { +function snapshotEntry(nodes: readonly unknown[] = NODES): ProviderScenarioProviderEntry { return { command: 'ios.runner.snapshot', deviceId: DEVICE_ID, platform: 'apple', - result: { nodes: NODES, truncated: false }, + result: { nodes, truncated: false }, }; } @@ -66,9 +66,10 @@ function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { test('Provider-backed integration rejects stale pinned @refs and accepts current pins', async () => { const runnerTranscript = createProviderTranscript([ - // snapshot -i: issues the complete frame at the seeded generation g1; the first capture after - // open settles: one extra read - snapshotEntry(), + // The first capture after open settles over a quiet pair of first-screen reads. + snapshotEntry(FIRST_SCREEN_NODES), + snapshotEntry(FIRST_SCREEN_NODES), + // snapshot -i: issues the complete frame at generation g1 snapshotEntry(), // press @e2~s{g1}: pinned to the CURRENT frame — admitted, taps Cancel, // and crosses the seam so the frame expires @@ -103,6 +104,7 @@ test('Provider-backed integration rejects stale pinned @refs and accepts current udid: DEVICE_ID, }); assertRpcOk(open); + assertRpcOk(await daemon.callCommand('snapshot')); const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 927777d796..b6c7a94d8a 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -99,6 +99,7 @@ Open apps and links: - `open [app|url] [url]` boots the selected device when needed and opens the target in a session. `open ` opens deep links on Android and iOS, and `open ` opens a deep link in that app on iOS. See [Desktop targets](#desktop-targets) for macOS activation behavior. - Remote daemons refuse `file:` URLs in `open` targets and launch URLs with reason `REMOTE_HOST_INPUT_DENIED`. Serve local web content over HTTP(S) instead. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom-scheme URLs need an active app in the session. +- Local iOS simulator: after `open` launches an app and the backend reports that launch (local iOS simulators do; other devices and backends skip the wait, so their first capture can still show a launch screen), the first capture in the session polls for the app's first screen for up to 10 seconds. A capture that starts before the 10 seconds end still completes and is judged. This includes the initial snapshot of `open --foreground`. A tree counts as the first screen when an element other than the app root has a label or value, and two reads in a row agree. A splash image, an empty view, or only a busy indicator does not count, however long it stays unchanged. An app whose first screen has no labeled element, such as a game canvas, waits the full 10 seconds. If the wait runs out, the snapshot returns what it read with a warning: `postGestureOutcome.kind: "loading"` when it still showed no first screen, or `"unsettled"` when a first screen was there but kept changing. Run `snapshot -i` again, or `wait text `, before you act on it. - `open --wait ` waits up to that budget for a device another session holds, then fails with `DEVICE_IN_USE`. The wait extends the command's timeout, so you don't need a longer `--timeout`. A claim held by another workspace's daemon is not waited for; `open` returns its recovery command at once. Device-mutating commands like `install` and `reinstall` never wait either: a claim held by another session fails them at once, and when the caller sends a workspace, another workspace's claim under the same session name fails too. See [Sessions](/docs/sessions#wait-for-a-slow-or-busy-device). - `open --launch-console ` captures launch-time stdout/stderr for iOS simulator app launches, including Limrun. It is not valid for URL opens or other targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface; `app` is the default when you name an app. See [Desktop targets](#desktop-targets). `--save-script` records the surface so `replay` reopens it; with no surface given, replay opens an app session. @@ -111,7 +112,7 @@ Open apps and links: iOS Simulator launch behavior: -- An `open` that launches an app reports `timing.postOpenObservation`. When it is present and not `"observable"`, the next command that reads the screen waits briefly until two reads agree, so a `click` lands where the element comes to rest. If the screen is still moving after that wait, the command proceeds and warns that the surface was still changing. +- An `open` that launches an app reports `timing.postOpenObservation`. Whenever it is present, the next command that reads the screen waits for the app's first screen, up to the 10-second polling budget described under `open`, so a `click` lands on the real first screen where its element comes to rest. If the wait runs out, the command proceeds and warns with `postGestureOutcome.kind` `"loading"` (no first screen yet) or `"unsettled"` (a first screen that kept changing). - If CoreSimulator never answers a URL hand-off, `open` fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout`. The Simulator may still have received the URL; retry the open or answer the prompt with `alert accept`. - When `open --launch-url ` raises an `Open in ""?` prompt and the URL's scheme belongs to the session app, `open` taps **Open** and returns `launchConfirmation: "accepted"`. If the app still never starts, `open` hands over the URL once more. - On a local iOS Simulator, `open` recognizes the prompt in the Simulator's preferred languages, through the OS's own wording of its title and buttons, so a French Simulator's `Ouvrir dans « » ?` is answered like the English one. `alert accept` and `alert dismiss` answer it by the same wording. If the runner cannot read that wording, the prompt is not recognized. @@ -629,12 +630,13 @@ agent-device is text 'id="greeting"' "Welcome back" - Supported predicates are `visible`, `hidden`, `exists`, `absent`, `editable`, `selected`, `focused`, and `text`. - `is visible` passes when the element is in the visible viewport, or has no rect of its own but sits inside a visible ancestor. `is exists` only checks that the selector matches. - `is text ` compares the element's text with the value. `wait text` checks that text is present, not that it can be tapped. -- `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth capture. It does not mean hidden: `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, or unsettled captures fail. +- `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth capture. It does not mean hidden: `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, unsettled, or loading captures fail. - Predicates that name one element (all except `exists` and `absent`) and `get attrs` fail with `AMBIGUOUS_MATCH` when the selector matches more than one node; that is not a missing element. `error.details.matches` has the count, and up to five candidate `@ref`s are listed, pinned to `error.details.refsGeneration`, so you can act on one directly. Without a stored tree (a sparse capture), only the count is reported. See [Selectors](/docs/selectors#find-an-element). - On iOS, React Native text appears as two nodes with the same label. Selectors that must resolve one element (`is`, `get attrs`, `screenshot --crop-on`, replay) treat the pair as one; `find ... list` and `snapshot --json` show both. - On the web, on-screen text the browser cannot hand an action to (static text, a `status` announcement) is in the tree and readable by `wait text`, `find text`, `get text`, and `text=`, but carries no `@ref` and has no rect: `find ... list` prints it as `(no ref)`, and it is never a tap target. Such text is published on `value`, so `value="X"` can match page text as well as a field holding that text, and a predicate that needs one element then reports `AMBIGUOUS_MATCH`; use `text=` for page text, or add `editable=true` to name the field. - The first read after `scroll`, `swipe`, or `gesture swipe` waits for two captures to agree. When that proves something about the gesture, `is`, `get`, `find`, `wait`, interactions that captured it, and `snapshot` add a warning, and all but `snapshot` report `postGestureOutcome` (`{ "kind", "gesture": { "action", "positionals" } }`) in `data` or `error.details`: - - `kind: "unsettled"`: the screen was still changing. A miss is not proof of absence; read again. `is absent` refuses with `observation: "unsettled"`, and `wait absent` keeps polling. + - `kind: "unsettled"`: the screen was still changing (after a gesture, or a first screen after `open` that never quieted). A miss is not proof of absence; read again. `is absent` refuses with `observation: "unsettled"`, and `wait absent` keeps polling. + - `kind: "loading"`: the app that `open` launched still showed no first screen when the wait ran out. A miss is not proof of absence. `is absent` refuses with `observation: "loading"`, and `wait absent` keeps polling. - `kind: "no-effect"`: nothing changed. The list may be at its edge or ignore synthesized scrolls; a raw `swipe x1 y1 x2 y2` inside it moves such lists. The soft keyboard appearing or going away counts as a change only when the pre-gesture baseline and a later quiet capture both read the keyboard from their own tree and disagree: the iOS XCTest runner and the iOS Simulator AX bridge (the default Simulator route) do, a WebDriver (Appium) visibility probe does not. The AX bridge reads the keyboard as gone only from a complete tree that holds no keyboard window. Otherwise only the content is compared. - A caller that verifies the screen by polling it can skip that wait, and with it these outcomes, using `--no-post-gesture-stabilization` (see [Scroll](#scroll)).