Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6f4b568
fix(open): wait for the launched app's first screen before the first …
thymikee Nov 13, 2026
0d6b36d
refactor(capture-kit): keep the first-screen readiness rule module-pr…
thymikee Nov 13, 2026
61c230a
fix(open): keep the first-screen wait on iOS and settle fixtures on a…
thymikee Nov 13, 2026
efe4405
fix(open): qualify the first-screen wait by the launch observation; p…
thymikee Nov 13, 2026
1f137b4
style: format the parity fixture rows
thymikee Nov 13, 2026
a4a8036
fix(open): never accept a root-only tree as the first screen; an open…
thymikee Nov 13, 2026
093d6ee
docs(open): name both timeout outcomes of the first-screen wait
thymikee Nov 13, 2026
eea1539
fix(open): report loading only for a launch tree with no content
thymikee Nov 13, 2026
8638f3e
fix(open): judge a first screen at the deadline by the same readiness…
thymikee Nov 13, 2026
d6a0872
docs(open): limit the first-screen wait promise to local iOS simulators
thymikee Nov 13, 2026
61f5873
refactor(selectors): split the zero-match absence branch out of class…
thymikee Nov 13, 2026
7872ffd
docs(open): describe the post-open wait as the first-screen wait on e…
thymikee Nov 13, 2026
882598e
Merge origin/main into fix/open-waits-for-first-screen
thymikee Nov 14, 2026
896ab43
fix(capture): tell a first screen from a splash by its readable content
thymikee Nov 16, 2026
d67b371
test(daemon): name the splash wait test for its bounded behavior
thymikee Nov 16, 2026
885078f
Merge remote-tracking branch 'origin/main' into HEAD
thymikee Nov 16, 2026
72afa34
fix(mcp): keep the first-screen wait out of the always-loaded open sc…
thymikee Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 99 additions & 1 deletion packages/capture-kit/src/post-gesture-stability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { beforeEach, test, vi } from 'vitest';
import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
import type { ObservationClock } from './observe-until.ts';
import {
formatPostGestureOutcomeWarning,
runPostGestureStabilityLoop,
type PostGestureBaseline,
type PostGesturePreActionBaseline,
Expand All @@ -20,7 +21,13 @@ beforeEach(() => {
vi.mocked(emitDiagnostic).mockClear();
});

type Surface = Readonly<{ signature: readonly string[]; costMs: number; backend?: string }>;
type Surface = Readonly<{
signature: readonly string[];
costMs: number;
backend?: string;
/** Whether this read shows a launched app's first screen; absent reads count as one. */
firstScreen?: boolean;
}>;

// The negative half of these lines is type-level: each @ts-expect-error fails `pnpm typecheck`
// the moment the baseline union would accept a post-action reading as pre-action evidence again
Expand Down Expand Up @@ -93,6 +100,7 @@ function hooksFor(
same(baseline.signature, quiet.signature) ? 'unchanged' : 'changed',
surfacesIdentical: same,
summarizeDivergence: () => ({}),
showsFirstScreen: (value) => value.firstScreen ?? true,
};
}

Expand Down Expand Up @@ -218,3 +226,93 @@ test('an effect that lands after a rebase keeps the loop polling until the moved
assert.equal(diagnostics('post_gesture_snapshot_baseline_rebased').length, 1);
assert.equal(diagnostics('post_gesture_snapshot_stale_accept').length, 0);
});

// --- first-screen readiness after an app launch ---

const LAUNCH = { action: 'open', positionals: [], readiness: 'first-screen' as const };

function launchSurface(name: string, firstScreen: boolean, costMs = 50): Surface {
return { signature: [name], costMs, firstScreen };
}

test('a launch keeps polling through a splash that holds unchanged until the first screen appears', async () => {
const clock = fakeClock();
const splash = launchSurface('splash', false);
const home = launchSurface('home', true);

const outcome = await runPostGestureStabilityLoop({
pending: LAUNCH,
// Ten reads a 200 ms poll apart: the splash holds unchanged for over 2 s.
hooks: hooksFor(clock, [...Array.from({ length: 10 }, () => splash), home, home]),
clock,
});

assert.equal(outcome.value, home);
assert.equal(outcome.postGestureOutcome, undefined);
assert.ok(clock.now() > 2_000, `a quiet splash settled after ${clock.now()}ms`);
});

test('a launch returns on the first quiet pair that shows the first screen', async () => {
const clock = fakeClock();
const home = launchSurface('home', true);

const outcome = await runPostGestureStabilityLoop({
pending: LAUNCH,
hooks: hooksFor(clock, [home, home]),
clock,
});

assert.equal(outcome.value, home);
assert.equal(outcome.postGestureOutcome, undefined);
assert.ok(clock.now() < 500, `settled after ${clock.now()}ms`);
});

test('a launch whose budget expires on a loading tree reports the app as loading', async () => {
const clock = fakeClock();
const loading = launchSurface('loading', false);

const outcome = await runPostGestureStabilityLoop({
pending: LAUNCH,
hooks: hooksFor(
clock,
Array.from({ length: 60 }, () => loading),
),
clock,
});

assert.equal(outcome.value, loading);
assert.deepEqual(outcome.postGestureOutcome, {
kind: 'loading',
gesture: { action: 'open', positionals: [] },
});
assert.ok(clock.now() >= 10_000 && clock.now() < 10_500, `gave up after ${clock.now()}ms`);
assert.match(formatPostGestureOutcomeWarning(outcome.postGestureOutcome!), /still loading/);
});

test('a launch whose first screen never quiets reports it unsettled, not loading', async () => {
const clock = fakeClock();
const frames = Array.from({ length: 60 }, (_, frame) => launchSurface(`frame ${frame}`, true));

const outcome = await runPostGestureStabilityLoop({
pending: LAUNCH,
hooks: hooksFor(clock, frames),
clock,
});

assert.equal(outcome.postGestureOutcome?.kind, 'unsettled');
assert.ok(clock.now() >= 10_000, `gave up after ${clock.now()}ms`);
});

test('a gesture record never applies the first-screen rule to an empty tree', async () => {
const clock = fakeClock();
const empty = launchSurface('empty', false);

const outcome = await runPostGestureStabilityLoop({
pending: PENDING,
hooks: hooksFor(clock, [empty, empty]),
clock,
});

assert.equal(outcome.value, empty);
assert.equal(outcome.postGestureOutcome, undefined);
});
34 changes: 33 additions & 1 deletion packages/capture-kit/src/post-gesture-stability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,18 @@ const POST_GESTURE_STABILITY_SCHEDULE: ObservationSchedule = {
*/
const STABILIZATION_DISTRUST_DEADLINE_MS = POST_GESTURE_STABILITY_SCHEDULE.budgetMs + 2_000;

/**
* Cadence and budget for a launched app's first screen. A launch shows a splash or a loading
* surface before its first screen, and a JavaScript bundle can take seconds to arrive, so the
* budget is wider than a gesture's. It bounds when a poll may start; a slow remote capture that
* starts in time is still judged.
*/
const FIRST_SCREEN_SCHEDULE: ObservationSchedule = {
intervalMs: POST_GESTURE_STABILITY_SCHEDULE.intervalMs,
budgetMs: 10_000,
minPolls: 2,
};

export type BaselineSurfaceEvidence = 'changed' | 'unchanged' | 'ambiguous';

/**
Expand Down Expand Up @@ -86,6 +98,11 @@ export type PostGestureStabilityPending<S> = {
positionals: string[];
/** Absent when the writer stored no baseline (non-Apple, or nothing usable pre-gesture). */
baseline?: PostGesturePreActionBaseline<S>;
/**
* `first-screen`: an app launch marked this record. A quiet pair settles the loop only when
* `showsFirstScreen` accepts it, under {@link FIRST_SCREEN_SCHEDULE}.
*/
readiness?: 'first-screen';
};

export type PostGestureStabilityHooks<T, S extends readonly unknown[]> = {
Expand All @@ -106,6 +123,12 @@ export type PostGestureStabilityHooks<T, S extends readonly unknown[]> = {
surfacesIdentical: (baseline: S, current: S) => boolean;
/** Why `surfacesIdentical` said no, for the veto diagnostic. */
summarizeDivergence: (baseline: S, current: S) => Record<string, number>;
/**
* Whether one attempt shows a launched app's first screen rather than its launch or loading
* surface. The producer owns the rule because only it can read the tree. Read only for a launch
* record.
*/
showsFirstScreen: (value: T) => boolean;
};

export type PostGestureStabilityOutcome<T> = {
Expand All @@ -129,6 +152,9 @@ export type PostGestureStabilityOutcome<T> = {
*/
export function formatPostGestureOutcomeWarning({ kind, gesture }: PostGestureOutcome): string {
const named = [gesture.action, ...gesture.positionals].join(' ').trim();
if (kind === 'loading') {
return `The app was still loading after ${named} when this tree was read: it may show a launch or loading screen. Run snapshot -i again, or wait for expected content (wait text ...) before acting.`;
}
return kind === 'unsettled'
? `The surface was still changing after ${named} when this tree was read, so it may not match where the surface comes to rest: an element missing from it is not proof of absence.`
: `${named} produced no visible change: the tree still matches its pre-gesture state. ` +
Expand Down Expand Up @@ -193,6 +219,7 @@ export async function runPostGestureStabilityLoop<T, S extends readonly unknown[
// A rebase or a distrust verdict keeps polling on a pair that DID agree, so
// the budget can expire on a surface that is already at rest.
let lastPairAgreed = false;
const firstScreen = pending.readiness === 'first-screen';
let surfaceCache: CapturedSurface<T, S> | undefined;

const surfaceOf = (value: T): CapturedSurface<T, S> => {
Expand All @@ -204,7 +231,7 @@ export async function runPostGestureStabilityLoop<T, S extends readonly unknown[
const observed = await observeUntil<T, PostGestureStabilityOutcome<T>>({
...(params.initial !== undefined ? { initial: params.initial } : {}),
capture: () => hooks.capture(),
schedule: POST_GESTURE_STABILITY_SCHEDULE,
schedule: firstScreen ? FIRST_SCREEN_SCHEDULE : POST_GESTURE_STABILITY_SCHEDULE,
...(clock ? { clock } : {}),
verdict: (latest, previousValue) => {
attempts += 1;
Expand All @@ -216,6 +243,7 @@ export async function runPostGestureStabilityLoop<T, S extends readonly unknown[
if (!lastPairAgreed) return { kind: 'continue' };

const elapsedMs = now() - startedAt;
if (firstScreen && !hooks.showsFirstScreen(latest)) return { kind: 'continue' };
// A capture plan may fall back or be pre-empted by the XCTest-channel penalty at any time,
// so the serving surface can change mid-poll. This quiet pair says nothing about the
// gesture: adopt it as a post-action baseline and keep going (#1569). The pair that rebases
Expand Down Expand Up @@ -273,6 +301,10 @@ export async function runPostGestureStabilityLoop<T, S extends readonly unknown[
});
// minPolls guarantees at least one judged value before an `expired` end can fire.
const value = observed.last as T;
// Loading means the last read still showed no first screen.
if (firstScreen && !hooks.showsFirstScreen(value)) {
return { value, postGestureOutcome: postGestureOutcome('loading', pending) };
}
if (lastPairAgreed) return { value };
return { value, postGestureOutcome: postGestureOutcome('unsettled', pending) };
Comment thread
thymikee marked this conversation as resolved.
Resolved
Comment thread
thymikee marked this conversation as resolved.
Resolved
}
Expand Down
4 changes: 4 additions & 0 deletions packages/client/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { CommandFlags, InternalRequestOptions } from '@agent-device/contracts/command';
import { readPostGestureOutcome } from '@agent-device/contracts/snapshot';
import {
readSerializedSnapshotCaptureAnnotations,
readSnapshotDiagnosticsSummary,
Expand Down Expand Up @@ -589,6 +590,7 @@ function optionalSnapshotResponseFields(
| 'keyboard'
| 'viewport'
| 'presentedSurface'
| 'postGestureOutcome'
| 'warnings'
| 'snapshotQuality'
| 'snapshotDiagnostics'
Expand All @@ -602,10 +604,12 @@ function optionalSnapshotResponseFields(
const viewport = readSnapshotViewportSize(data.viewport);
const snapshotDiagnostics = readSnapshotDiagnosticsSummary(data.snapshotDiagnostics);
const presentedSurface = readPresentedSurfaceFact(data);
const postGestureOutcome = readPostGestureOutcome(data.postGestureOutcome);
return {
...(keyboard ? { keyboard } : {}),
...(viewport ? { viewport } : {}),
...(presentedSurface ? { presentedSurface } : {}),
...(postGestureOutcome ? { postGestureOutcome } : {}),
...(visibility ? { visibility: visibility as CaptureSnapshotResult['visibility'] } : {}),
...readSerializedSnapshotCaptureAnnotations(data),
...(unchanged ? { unchanged: unchanged as CaptureSnapshotResult['unchanged'] } : {}),
Expand Down
1 change: 1 addition & 0 deletions packages/command-output/src/output/result-serialization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ export function serializeSnapshotResult(result: CaptureSnapshotResult): Record<s
...(result.keyboard ? { keyboard: result.keyboard } : {}),
...(result.viewport ? { viewport: result.viewport } : {}),
...(result.presentedSurface ? { presentedSurface: result.presentedSurface } : {}),
...(result.postGestureOutcome ? { postGestureOutcome: result.postGestureOutcome } : {}),
Comment thread
thymikee marked this conversation as resolved.
Resolved
...publicSnapshotCaptureAnnotations({
...result,
...(result.snapshotQuality ? { quality: result.snapshotQuality } : {}),
Expand Down
26 changes: 26 additions & 0 deletions packages/command-output/src/output/snapshot-serialization.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { test } from 'vitest';
import assert from 'node:assert/strict';
import type { PostGestureOutcome } from '@agent-device/contracts/snapshot-model';
import { serializeSnapshotResult } from './result-serialization.ts';
import { snapshotViewportSizeFrom } from '@agent-device/kernel/rect';

Expand Down Expand Up @@ -144,6 +145,31 @@ test('serializeSnapshotResult omits a presented surface the producer did not rea
assert.equal('presentedSurface' in data, false);
});

test('serializeSnapshotResult publishes the post-gesture outcome beside the tree', () => {
const postGestureOutcome: PostGestureOutcome = {
kind: 'loading',
gesture: { action: 'open', positionals: [] },
};
const data = serializeSnapshotResult({
nodes: [],
truncated: false,
postGestureOutcome,
identifiers: { session: 'qa' },
});

assert.deepEqual(data.postGestureOutcome, postGestureOutcome);
});

test('serializeSnapshotResult omits the post-gesture outcome of a tree read at rest', () => {
const data = serializeSnapshotResult({
nodes: [],
truncated: false,
identifiers: { session: 'qa' },
});

assert.equal('postGestureOutcome' in data, false);
});

test('serializeSnapshotResult retains clipping notes with or without a capture verdict', () => {
const snapshotQualityNotes = [
{ code: 'regular-geometry-clipped' as const, clippedNodeCount: 1, droppedNodeCount: 1 },
Expand Down
2 changes: 1 addition & 1 deletion packages/command-registry/src/flag-definitions-action.ts
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
type: 'boolean',
usageLabel: '--foreground',
usageDescription:
'open: keep normal app/device selection and return an initial snapshot; without an app, resolve the sole running app on the sole booted iOS simulator',
'open: keep normal app/device selection and return an initial snapshot of the first screen (a local iOS simulator waits up to 10s for it); without an app, resolve the sole running app on the sole booted iOS simulator',
inputDescription:
'Include an initial interactive snapshot in a fresh open response. With no app argument, discover the sole running app on the sole booted iOS simulator; ambiguous environments fail closed.',
projectConfig: false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -851,6 +851,39 @@ test('runtime snapshot warns when its tree was read on a surface still moving af
const result = await device.capture.snapshot({ session: 'default' });

assert.deepEqual(result.warnings, [formatPostGestureOutcomeWarning(outcome)]);
assert.deepEqual(result.postGestureOutcome, outcome);
});

test('runtime snapshot publishes a launched app that was still loading as its outcome', async () => {
const outcome: PostGestureOutcome = {
kind: 'loading',
gesture: { action: 'open', positionals: [] },
};
const device = createSnapshotOnlyDevice({
snapshot: {
...makeSnapshotState([{ index: 0, depth: 0, type: 'Application', label: 'Demo' }], {
backend: 'xctest',
}),
postGestureOutcome: outcome,
},
});

const result = await device.capture.snapshot({ session: 'default' });

assert.deepEqual(result.postGestureOutcome, outcome);
assert.deepEqual(result.warnings, [formatPostGestureOutcomeWarning(outcome)]);
});

test('runtime snapshot publishes no outcome for a tree read at rest', async () => {
const device = createSnapshotOnlyDevice({
snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], {
backend: 'xctest',
}),
});

const result = await device.capture.snapshot({ session: 'default' });

assert.equal('postGestureOutcome' in result, false);
});

test('runtime snapshot warns when a gesture produced no visible change', async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import type {
SnapshotViewportSize,
SnapshotVisibility,
} from '@agent-device/kernel/snapshot';
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import type { PostGestureOutcome, SnapshotState } from '@agent-device/contracts/snapshot-model';
import type { BackendSnapshotResult } from '@agent-device/contracts/backend';
import type {
AgentDeviceRuntime,
Expand Down Expand Up @@ -76,6 +76,11 @@ export type SnapshotCommandResult = {
* non-iOS backend — publishes nothing.
*/
presentedSurface?: PresentedSurface;
/**
* What the post-gesture or post-open wait concluded about this tree, when it found the surface
* not at rest: still changing, unchanged by the gesture, or, after an app launch, still loading.
*/
postGestureOutcome?: PostGestureOutcome;
} & PublicSnapshotCaptureAnnotations;

type SnapshotCapture = {
Expand Down Expand Up @@ -123,6 +128,9 @@ export const snapshotCommand: RuntimeCommand<
...(capture.snapshot.presentedSurface
? { presentedSurface: capture.snapshot.presentedSurface }
: {}),
...(capture.snapshot.postGestureOutcome
? { postGestureOutcome: capture.snapshot.postGestureOutcome }
Comment thread
thymikee marked this conversation as resolved.
Resolved
: {}),
...snapshotAppFields(capture),
});
};
Expand Down
7 changes: 7 additions & 0 deletions packages/contracts/src/client-capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type {
SnapshotVisibility,
} from '@agent-device/kernel/snapshot';
import type { ScreenshotResultData } from './snapshot-types.ts';
import type { PostGestureOutcome } from './snapshot-model.ts';
import type {
AgentDeviceIdentifiers,
AgentDeviceRequestOverrides,
Expand Down Expand Up @@ -83,6 +84,12 @@ export type CaptureSnapshotResult = {
* of the field means a non-iOS producer or an older runner published nothing.
*/
presentedSurface?: PresentedSurface;
/**
* Present when the tree may not show the surface at rest: `unsettled` (still changing after a
* gesture), `no-effect` (a gesture changed nothing), or `loading` (the app launched by `open`
* still showed no first screen when its wait ran out).
*/
postGestureOutcome?: PostGestureOutcome;
/**
* Screenshot captured automatically when the semantic snapshot was sparse.
* Remote clients receive a materialized local path through the daemon artifact channel.
Expand Down
Loading
Loading