diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 87ca6d6a58..6c3b307aa3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -465,6 +465,9 @@ extension RunnerTests { guard let selectorKey = command.selectorKey, let selectorValue = command.selectorValue else { return Response(ok: false, error: ErrorPayload(message: "querySelector requires selectorKey and selectorValue")) } + if command.hittabilityProbe == true { + return try probeElementHittability(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) + } return try queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) case .readText: guard let x = command.x, let y = command.y else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index ab8add06bb..767a6cec36 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -334,17 +334,7 @@ extension RunnerTests { guard !value.isEmpty else { return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } - let predicate: NSPredicate - switch selectorKey { - case "id": - predicate = NSPredicate(format: "identifier ==[c] %@", value) - case "label": - predicate = NSPredicate(format: "label ==[c] %@", value) - case "value": - predicate = NSPredicate(format: "value ==[c] %@", value) - case "text": - predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value) - default: + guard let predicate = Self.selectorPredicate(key: selectorKey, value: value) else { return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } @@ -493,6 +483,47 @@ extension RunnerTests { try resolveRunnerWindow(app: app).frame } + static func selectorPredicate(key: String, value: String) -> NSPredicate? { + switch key { + case "id": + return NSPredicate(format: "identifier ==[c] %@", value) + case "label": + return NSPredicate(format: "label ==[c] %@", value) + case "value": + return NSPredicate(format: "value ==[c] %@", value) + case "text": + return NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value) + default: + return nil + } + } + + /// The pre-dispatch hittability read: every raw match is counted (hittable or not, unlike + /// `queryElement`'s hittable preference), and only a single match is read, for `isHittable` and + /// its frame, with none of `queryElement`'s other properties. The daemon treats any other count + /// as unknown, so a duplicate label can never describe a different element. + func probeElementHittability(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response { + let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !value.isEmpty, let predicate = Self.selectorPredicate(key: selectorKey, value: value) else { + return Response(ok: true, data: DataPayload(found: false, matchCount: 0)) + } + let matches = try elementQueries(app: app, matching: .any) + .flatMap { $0.matching(predicate).allElementsBoundByIndex } + .filter(\.exists) + guard matches.count == 1, let element = matches.first else { + return Response(ok: true, data: DataPayload(found: !matches.isEmpty, matchCount: matches.count)) + } + return Response( + ok: true, + data: DataPayload( + found: true, + matchCount: 1, + hittable: element.isHittable, + rect: SnapshotRect(element.frame) + ) + ) + } + func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response { // querySelector is a read — it backs get/is/wait and the offscreen-refusal // double-check, none of which mutate. The fail-closed raw-match rule exists diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index f4bb0a2786..a5b2ebc5cf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -246,6 +246,8 @@ struct Command: Codable { let text: String? let selectorKey: String? let selectorValue: String? + /// `querySelector` only: answer the lean per-element hittability read instead of a node. + let hittabilityProbe: Bool? let allowNonHittableCoordinateFallback: Bool? let delayMs: Int? let textEntryMode: String? @@ -572,6 +574,11 @@ struct DataPayload: Codable { /// Present on a screenshot the runner captured from a display it resolved, alongside the /// `message` path or `imageBase64` payload that carries the image itself (#2728). var screenshotMetadata: ScreenshotMetadataPayload? + /// The `querySelector` hittability probe: how many elements the selector matched, and, when + /// exactly one did, its `isHittable` and frame. + var matchCount: Int? + var hittable: Bool? + var rect: SnapshotRect? } /// Mirrors the TS `FillVerificationTarget`; its nullable identity fields encode as explicit nulls. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift index 243c01ef1b..988b50d4cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift @@ -84,9 +84,9 @@ extension RunnerTests { } #if os(iOS) + /// Launches the duplicate-selector fixture, runs one runner command against it, and closes it. @MainActor - func testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate() throws { - let duplicateIdentifier = "agent-device-selector-read-duplicate" + private func runAgainstDuplicateSelectorFixture(_ commandJson: String) throws -> Response? { app.launchArguments = ["--agent-device-selector-read-regression"] app.launch() defer { @@ -96,29 +96,48 @@ extension RunnerTests { XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let matches = app.descendants(matching: .any) - .matching(identifier: duplicateIdentifier) + .matching(identifier: "agent-device-selector-read-duplicate") .allElementsBoundByIndex .filter(\.exists) XCTAssertEqual(matches.count, 2, "fixture must expose two raw identifier matches") XCTAssertEqual(matches.filter(\.isHittable).count, 1, "fixture must expose exactly one hittable match") - let command = try JSONDecoder().decode( - Command.self, - from: Data( - #"{"command":"querySelector","commandId":"query-selector-duplicate","selectorKey":"id","selectorValue":"agent-device-selector-read-duplicate"}"#.utf8 - ) - ) + let command = try JSONDecoder().decode(Command.self, from: Data(commandJson.utf8)) let response = try executeOnMainPrepared(command: command, activeApp: app) - guard response.ok else { XCTFail(String(describing: response.error)) - return + return nil } + return response + } + + @MainActor + func testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate() throws { + let duplicateIdentifier = "agent-device-selector-read-duplicate" + guard let response = try runAgainstDuplicateSelectorFixture( + #"{"command":"querySelector","commandId":"query-selector-duplicate","selectorKey":"id","selectorValue":"agent-device-selector-read-duplicate"}"# + ) else { return } + XCTAssertEqual(response.data?.found, true) XCTAssertEqual(response.data?.nodes?.count, 1) XCTAssertEqual(response.data?.nodes?.first?.identifier, duplicateIdentifier) XCTAssertEqual(response.data?.nodes?.first?.hittable, true) } + + /// The hittability probe counts every raw match and reads none of them for a duplicate selector: + /// the daemon must never let a duplicate label describe a different element. + @MainActor + func testHittabilityProbeCountsEveryMatchAndReadsNoneForADuplicate() throws { + guard let response = try runAgainstDuplicateSelectorFixture( + #"{"command":"querySelector","commandId":"hittability-probe-duplicate","selectorKey":"id","selectorValue":"agent-device-selector-read-duplicate","hittabilityProbe":true}"# + ) else { return } + + XCTAssertEqual(response.data?.found, true) + XCTAssertEqual(response.data?.matchCount, 2) + XCTAssertNil(response.data?.hittable) + XCTAssertNil(response.data?.rect) + XCTAssertNil(response.data?.nodes) + } #endif #endif } diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index c2f037b60d..f7a46dcfcc 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -64,6 +64,7 @@ {"name": "ios-simulator.runner-command-recovery.status", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "status", "statusCommandId": ""}}, {"name": "ios-simulator.runner-disposal.shutdown", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "shutdown", "commandId": ""}}, {"name": "ios-simulator.runner-lifecycle-prepare.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, + {"name": "ios-simulator.runner-selector-query.hittability-probe", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "hittabilityProbe": true, "commandId": ""}}, {"name": "ios-simulator.runner-selector-query.label", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": ""}}, {"name": "ios-simulator.runner-session-readiness.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index b48b1bb765..790f8d286e 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -127,6 +127,8 @@ export type RunnerCommand = { text?: string; selectorKey?: ElementSelectorKey; selectorValue?: string; + /** `querySelector` only: answer the lean per-element hittability read instead of a node. */ + hittabilityProbe?: boolean; allowNonHittableCoordinateFallback?: boolean; delayMs?: number; textEntryMode?: 'append' | 'replace'; diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index dccc021be8..1ba85f14d8 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -54,6 +54,8 @@ export type PressCommandOptions = CommandContext & expectedResolvedTarget?: ExpectedResolvedTarget; /** #1654: a mutating `find`'s already-resolved node; see resolution.ts. */ preresolvedTarget?: PreresolvedInteractionTarget; + /** Internal Maestro replay policy: replay taps resolved bounds, so admission skips the hittability read. */ + allowNonHittableCoordinateFallback?: boolean; } & PostActionObservationOptions; export type ClickCommandOptions = PressCommandOptions; diff --git a/packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission-hittability.test.ts b/packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission-hittability.test.ts new file mode 100644 index 0000000000..2348674d51 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission-hittability.test.ts @@ -0,0 +1,128 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend, TargetHittability } from '@agent-device/contracts/backend'; +import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; +import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { createAgentDevice } from '../../../runtime.ts'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { ref } from './selector-read-utils.ts'; + +// The pre-dispatch hittability check on the press route: which targets pay for the native read, and +// what each verdict does. The pure gate and the verdict mapping have their own tests beside them. + +function node( + index: number, + type: string, + label: string, + rect: SnapshotNode['rect'], +): SnapshotNode { + return { index, depth: 1, parentIndex: 0, type, label, rect, hittable: true }; +} + +const TREE = makeSnapshotState([ + node(0, 'Application', 'App', { x: 0, y: 0, width: 402, height: 874 }), + node(1, 'Button', 'Delete Avocado', { x: 294, y: 425, width: 88, height: 82 }), + node(2, 'Cell', 'Avocado, 3 item, $4.50', { x: 20, y: 425, width: 362, height: 82 }), + node(3, 'Button', 'Add item', { x: 20, y: 610, width: 362, height: 52 }), +]); + +function createPressDevice(verdict: TargetHittability | (() => Promise)) { + const probes: string[] = []; + const taps: Point[] = []; + const backend: AgentDeviceBackend = { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: TREE }), + probeTargetHittability: async (_context, probed) => { + probes.push(probed.label ?? ''); + return typeof verdict === 'function' ? await verdict() : verdict; + }, + tap: async (_context, point) => { + taps.push(point); + return { ok: true }; + }, + }; + const device = createAgentDevice({ + backend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: TREE }]), + policy: localCommandPolicy(), + }); + return { device, probes, taps }; +} + +function refOf(label: string) { + const found = TREE.nodes.find((candidate) => candidate.label === label)!; + return ref(`@${found.ref}`); +} + +test('a press on a control the platform reports not hittable is refused before any tap', async () => { + const { device, probes, taps } = createPressDevice('notHittable'); + + await assert.rejects(device.interactions.press(refOf('Delete Avocado')), (error: unknown) => { + const { message, details } = error as { message: string; details?: Record }; + assert.equal(message, '"Delete Avocado" is not hittable: a tap would land on another element.'); + assert.equal(details?.reason, INTERACTION_ERROR_REASONS.targetNotHittable); + assert.equal(details?.dispatched, 'no'); + assert.equal(details?.coveredBy, undefined, 'no occlusion evidence is claimed'); + assert.equal( + details?.hint, + 'Reveal the control (for example swipe its row), then take a fresh snapshot -i.', + ); + return true; + }); + assert.deepEqual(probes, ['Delete Avocado']); + assert.deepEqual(taps, []); +}); + +for (const verdict of ['hittable', 'unknown'] as const) { + test(`a gated press the probe answers ${verdict} still taps`, async () => { + const { device, probes, taps } = createPressDevice(verdict); + + await device.interactions.press(refOf('Delete Avocado')); + + assert.deepEqual(probes, ['Delete Avocado']); + assert.equal(taps.length, 1); + }); +} + +test('a top-level press pays for no probe even when the backend would refuse', async () => { + const { device, probes, taps } = createPressDevice('notHittable'); + + await device.interactions.press(refOf('Add item')); + + assert.deepEqual(probes, []); + assert.equal(taps.length, 1); +}); + +test('a probe that rejects the request cancels before any tap', async () => { + const controller = new AbortController(); + const { device, taps } = createPressDevice(async () => { + controller.abort(); + return 'hittable'; + }); + + await assert.rejects( + device.interactions.press(refOf('Delete Avocado'), { signal: controller.signal }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'request_canceled'); + assert.equal(details?.dispatched, 'no'); + return true; + }, + ); + assert.deepEqual(taps, []); +}); + +test('the Maestro replay fallback keeps the hittability read out of a gated press', async () => { + const { device, probes, taps } = createPressDevice('notHittable'); + + await device.interactions.press(refOf('Delete Avocado'), { + allowNonHittableCoordinateFallback: true, + }); + + assert.deepEqual(probes, []); + assert.equal(taps.length, 1); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission.ts b/packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission.ts index 8e5e9ac816..497bbdccc3 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission.ts @@ -7,6 +7,10 @@ * A raw point has no resolved element to re-observe. A replay step's identity guard (ADR 0012 * step 4) and a mutating `find`'s carried node (#1654) own what they resolved, and a second look * would replace their evidence with another observation, so those targets bypass admission. + * + * A target nested in a larger touch control gets one native hittability read first + * (`refuseUnhittableTarget` below): a control the platform reports as not hittable refuses here, since + * a tap at its centre would land on another element. */ import type { InteractionTarget, @@ -20,7 +24,9 @@ import { discloseDispatch, } from '@agent-device/kernel/errors'; import { observeUntil, type ObservationClock } from '@agent-device/capture-kit/observe-until'; -import { now, sleep } from '../../runtime-common.ts'; +import { now, sleep, toBackendContext } from '../../runtime-common.ts'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; +import { isNestedTouchControl } from '@agent-device/selectors/interaction-targeting'; import { judgeMotionLook, motionRefusal, @@ -32,9 +38,18 @@ import { } from './moving-target-gate.ts'; import type { ResolveInteractionTargetParams } from './interaction-resolution-request.ts'; +/** + * `allowNonHittableCoordinateFallback` is the Maestro replay policy: replay taps resolved bounds even + * when the platform says the element is not hittable, so it keeps the hittability read out. + */ +type AdmissionOptions = CommandContext & { + target: InteractionTarget; + allowNonHittableCoordinateFallback?: boolean; +}; + export async function admitResolvedTarget( runtime: AgentDeviceRuntime, - options: CommandContext & { target: InteractionTarget }, + options: AdmissionOptions, params: ResolveInteractionTargetParams, resolved: ResolvedInteractionTarget, ): Promise { @@ -43,6 +58,7 @@ export async function admitResolvedTarget( if (!node?.rect) return resolved; if (params.expectedResolvedTarget || params.preresolvedTarget) return resolved; const budget = requestBudget(runtime, options); + await refuseUnhittableTarget(runtime, options, resolved, budget.read); const screening = await budget.read(() => screenMotion(runtime, options, node, resolved.preAction), ); @@ -202,3 +218,64 @@ async function awaitRest( throw motionRefusal(resolved, undefined, waitedMs); } } + +type ResolvedElementTarget = Exclude; + +/** + * Whether the resolved target is worth a native hittability read: a touch control nested inside a + * larger touch control of the tree that measured it, the shape of a swipe action beside its row. + * Every other press pays nothing. + */ +function needsHittabilityRead(resolved: ResolvedElementTarget): boolean { + const nodes = resolved.preAction?.nodes; + return Boolean(resolved.node && nodes && isNestedTouchControl(nodes, resolved.node)); +} + +/** + * Asks the backend whether the gated target is hittable, and refuses before any dispatch when the + * platform says it is not. Every other verdict, and a backend without the probe, admits the target + * unchanged. `read` carries the admission budget, so cancellation and the request deadline end the + * look the way they end every other admission wait. + */ +async function refuseUnhittableTarget( + runtime: AgentDeviceRuntime, + options: AdmissionOptions, + resolved: ResolvedElementTarget, + read: ( + read: (signal: AbortSignal | undefined) => Promise, + until?: 'deadline', + ) => Promise, +): Promise { + const node = resolved.node; + const probe = runtime.backend.probeTargetHittability; + if (options.allowNonHittableCoordinateFallback === true) return; + if (!node || !probe || !needsHittabilityRead(resolved)) return; + const verdict = await read( + (signal) => + probe.call(runtime.backend, toBackendContext(runtime, { ...options, signal }), node), + 'deadline', + ); + if (verdict === 'notHittable') throw notHittableRefusal(resolved, node); +} + +function notHittableRefusal( + resolved: ResolvedElementTarget, + node: NonNullable, +): AppError { + const selector = resolved.target.kind === 'selector' ? resolved.target.selector : undefined; + const ref = resolved.target.kind === 'ref' ? resolved.target.ref : undefined; + const name = node.label?.trim() || node.identifier?.trim() || selector || ref; + return discloseDispatch( + new AppError( + 'COMMAND_FAILED', + `"${name}" is not hittable: a tap would land on another element.`, + { + reason: INTERACTION_ERROR_REASONS.targetNotHittable, + ...(selector !== undefined ? { selector } : { ref }), + ...(node.rect ? { rect: node.rect } : {}), + hint: 'Reveal the control (for example swipe its row), then take a fresh snapshot -i.', + }, + ), + 'no', + ); +} diff --git a/packages/contracts/src/backend.ts b/packages/contracts/src/backend.ts index ac338961f4..d39bc8e394 100644 --- a/packages/contracts/src/backend.ts +++ b/packages/contracts/src/backend.ts @@ -27,6 +27,8 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; // The backend's public leaf platform (approach b): backends distinguish iOS from // macOS (e.g. snapshot backend routing, the macOS surface guard), so this carries the // leaf string, not the internal collapsed `apple`. +export type TargetHittability = 'hittable' | 'notHittable' | 'unknown'; + export type AgentDeviceBackendPlatform = PublicPlatform; export type BackendCommandContext = { @@ -438,6 +440,18 @@ export type AgentDeviceBackend = { node: Pick, rootViewport: Rect | null, ): Promise; + /** + * Pre-dispatch hittability probe of ONE resolved target. `notHittable` means the platform + * reports the element and says a tap would not reach it; `hittable` means it reports it + * reachable; `unknown` is every other outcome (no selector, no match, an ambiguous match, a + * frame that is not the target's own, or a transport failure) and the caller proceeds as if + * the probe did not exist. Never throws on a failed read. Backends without a live per-element + * read omit this method. + */ + probeTargetHittability?( + context: BackendCommandContext, + node: Pick, + ): Promise; tap?( context: BackendCommandContext, point: Point, diff --git a/packages/contracts/src/selector-observation-runtime.ts b/packages/contracts/src/selector-observation-runtime.ts index 8cce121889..f5e73424ce 100644 --- a/packages/contracts/src/selector-observation-runtime.ts +++ b/packages/contracts/src/selector-observation-runtime.ts @@ -13,7 +13,14 @@ type ObservationInput = Readonly<{ export type FindTextInput = ObservationInput & Readonly<{ text: string }>; export type QuerySelectorInput = ObservationInput & - Readonly<{ selector: Readonly<{ key: ElementSelectorKey; value: string }> }>; + Readonly<{ + selector: Readonly<{ key: ElementSelectorKey; value: string }>; + /** + * `hittability` swaps the read for the lean one-element probe: no property enumeration, no + * hittable preference among matches. The answer is `QuerySelectorResult.hittability`. + */ + probe?: 'hittability'; + }>; /** * Positive native observations are authoritative and preserve matches that an advertising owner's @@ -34,9 +41,20 @@ export type QuerySelectorNodeReading = Readonly<{ hittable: boolean; }>; +/** + * What the lean probe read: how many elements the selector matched (every match, hittable or not) + * and, when exactly one matched, its live frame and `isHittable`. + */ +export type HittabilityProbeReading = Readonly<{ + matchCount: number; + hittable?: boolean; + rect?: Rect; +}>; + export type QuerySelectorResult = SelectorObservationResult & Readonly<{ node?: QuerySelectorNodeReading; + hittability?: HittabilityProbeReading; }>; export type SelectorObservationRuntimeOperations = Readonly<{ diff --git a/packages/daemon/src/core/interaction/internal/types.ts b/packages/daemon/src/core/interaction/internal/types.ts index a3a015f13a..0976cbf806 100644 --- a/packages/daemon/src/core/interaction/internal/types.ts +++ b/packages/daemon/src/core/interaction/internal/types.ts @@ -124,6 +124,13 @@ export type InteractionRuntimeInput = { node: Pick, rootViewport: Rect | null, ) => Promise; + probeTargetHittability?: ( + node: Pick< + import('@agent-device/kernel/snapshot').SnapshotNode, + 'identifier' | 'label' | 'rect' + >, + signal: AbortSignal | undefined, + ) => Promise; pairedGestureViewport?: Rect; touchExecutor?: BoundTouchExecutor; gestures?: BoundGestureExecutor; diff --git a/packages/daemon/src/execution/__tests__/hittable-target-probe.test.ts b/packages/daemon/src/execution/__tests__/hittable-target-probe.test.ts new file mode 100644 index 0000000000..f22b61c336 --- /dev/null +++ b/packages/daemon/src/execution/__tests__/hittable-target-probe.test.ts @@ -0,0 +1,113 @@ +import { test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { IOS_SIMULATOR } from '@agent-device/test-kit'; +import type { + QuerySelectorInput, + QuerySelectorResult, +} from '@agent-device/contracts/selector-observation-runtime'; +import type { SessionState } from '../../core/session-state.ts'; +import { classifyHittabilityReading, probeIosTargetHittability } from '../hittable-target-probe.ts'; + +const TARGET = { x: 294, y: 425.66, width: 88, height: 82 }; + +function makeSession(): SessionState { + return { name: 'default', device: IOS_SIMULATOR, createdAt: Date.now(), actions: [] }; +} + +type ProbeQuery = (input: QuerySelectorInput) => Promise; + +function probeWith(querySelector: ProbeQuery, node: { identifier?: string; label?: string }) { + return probeIosTargetHittability({ + session: makeSession(), + node: { ...node, rect: TARGET }, + querySelector, + execution: {}, + }); +} + +test('classifyHittabilityReading: a single match on the target frame speaks for it', () => { + assert.equal( + classifyHittabilityReading({ matchCount: 1, hittable: false, rect: TARGET }, TARGET), + 'notHittable', + ); + assert.equal( + classifyHittabilityReading({ matchCount: 1, hittable: true, rect: TARGET }, TARGET), + 'hittable', + ); +}); + +test('classifyHittabilityReading: a frame that is not the target frame is unknown', () => { + const elsewhere = { ...TARGET, y: TARGET.y + 100 }; + + assert.equal( + classifyHittabilityReading({ matchCount: 1, hittable: false, rect: elsewhere }, TARGET), + 'unknown', + ); +}); + +test('classifyHittabilityReading: an ambiguous or missing match is unknown', () => { + assert.equal(classifyHittabilityReading({ matchCount: 2 }, TARGET), 'unknown'); + assert.equal(classifyHittabilityReading({ matchCount: 0 }, TARGET), 'unknown'); + assert.equal(classifyHittabilityReading(undefined, TARGET), 'unknown'); + assert.equal( + classifyHittabilityReading({ matchCount: 1, hittable: false }, TARGET), + 'unknown', + 'a single match with no frame cannot be tied to the target', + ); +}); + +test('probeIosTargetHittability: asks the lean probe with the derived selector and app scope', async () => { + const querySelector = vi.fn(async () => ({ + found: true, + hittability: { matchCount: 1, hittable: false, rect: TARGET }, + })); + + const verdict = await probeIosTargetHittability({ + session: { ...makeSession(), appBundleId: 'com.example.demo' }, + node: { label: 'Delete Avocado', rect: TARGET }, + querySelector, + execution: { requestId: 'req-1' }, + }); + + assert.equal(verdict, 'notHittable'); + assert.deepEqual(querySelector.mock.calls[0]?.[0], { + selector: { key: 'label', value: 'Delete Avocado' }, + probe: 'hittability', + options: { appBundleId: 'com.example.demo' }, + execution: { requestId: 'req-1' }, + }); +}); + +test('probeIosTargetHittability: hands the admission read signal to the native read', async () => { + const signal = new AbortController().signal; + const querySelector = vi.fn(async () => ({ found: false })); + + await probeIosTargetHittability({ + session: makeSession(), + node: { label: 'Delete Avocado', rect: TARGET }, + querySelector, + execution: {}, + signal, + }); + + assert.equal(querySelector.mock.calls[0]?.[0].signal, signal); +}); + +test('probeIosTargetHittability: a node with no label or identifier is unknown without a read', async () => { + const querySelector = vi.fn(); + + assert.equal(await probeWith(querySelector, {}), 'unknown'); + assert.equal(querySelector.mock.calls.length, 0); +}); + +test('probeIosTargetHittability: a failed read is unknown', async () => { + const verdict = await probeWith( + async () => { + throw new AppError('COMMAND_FAILED', 'runner unavailable'); + }, + { label: 'Delete Avocado' }, + ); + + assert.equal(verdict, 'unknown'); +}); diff --git a/packages/daemon/src/execution/hittable-target-probe.ts b/packages/daemon/src/execution/hittable-target-probe.ts new file mode 100644 index 0000000000..094220429e --- /dev/null +++ b/packages/daemon/src/execution/hittable-target-probe.ts @@ -0,0 +1,58 @@ +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { areRectsApproximatelyEqual } from '@agent-device/kernel/rect-center'; +import type { TargetHittability } from '@agent-device/contracts/backend'; +import type { + HittabilityProbeReading, + QuerySelectorRuntimeOperations, +} from '@agent-device/contracts/selector-observation-runtime'; +import type { SnapshotRuntimeExecution } from '@agent-device/contracts/snapshot-runtime'; +import { deriveDirectIosNodeSelector } from '../core/direct-ios-selector.ts'; +import type { SessionState } from '../core/session-state.ts'; + +/** + * Maps one lean probe reading to a verdict about the resolved target. Only a single match whose + * live frame is the target's own frame may speak for it: another count, a missing reading, or a + * frame elsewhere means the selector described a different element, and the verdict is `unknown`. + */ +export function classifyHittabilityReading( + reading: HittabilityProbeReading | undefined, + targetRect: SnapshotNode['rect'], +): TargetHittability { + if (reading?.matchCount !== 1) return 'unknown'; + if (reading.hittable === undefined || !areRectsApproximatelyEqual(reading.rect, targetRect)) { + return 'unknown'; + } + return reading.hittable ? 'hittable' : 'notHittable'; +} + +/** + * The pre-dispatch hittability read of one target: one lean `querySelector` probe on the caller's + * request-bound operation, scoped to the session's application like the off-screen rescue read. + * A node with no selector, and a failed read of any kind, answer `unknown`. + */ +export async function probeIosTargetHittability(params: { + session: SessionState; + node: Pick; + querySelector: QuerySelectorRuntimeOperations['querySelector']; + execution: SnapshotRuntimeExecution; + /** The admission read's own signal: cancellation and the request deadline end the native read. */ + signal?: AbortSignal; +}): Promise { + const { session, node, querySelector, execution, signal } = params; + const selector = deriveDirectIosNodeSelector(node); + if (!selector) return 'unknown'; + try { + const { hittability } = await querySelector({ + selector, + probe: 'hittability', + ...(signal ? { signal } : {}), + ...(session.appBundleId === undefined + ? {} + : { options: { appBundleId: session.appBundleId } }), + execution, + }); + return classifyHittabilityReading(hittability, node.rect); + } catch { + return 'unknown'; + } +} diff --git a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts index 701534bd09..db69ee4c16 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts @@ -14,6 +14,7 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { DaemonCommandContext } from '../../../core/context.ts'; import { createDaemonRuntimePolicy } from '../../runtime-policy.ts'; import { recordCaptureProof } from '../../../core/capture-disclosure.ts'; +import { probeIosTargetHittability } from '../../hittable-target-probe.ts'; import { confirmIosOffscreenTargetVisible } from '../../offscreen-target-probe.ts'; import { runtimeExecutionFromContext } from '../../../core/snapshot-runtime-capture-input.ts'; import { createDaemonRuntimeSessionStore } from '../../../core/runtime-session.ts'; @@ -102,6 +103,7 @@ export async function createInteractionRuntimeForRoute( // the verdict it may overturn come from one admission (ADR 0019 §2). A session whose owner did // not advertise the read keeps the bulk-tree verdict unchanged. confirmOffscreenTargetVisible: readOffscreenTargetRescue(params, session), + probeTargetHittability: readTargetHittabilityProbe(params, session), pairedGestureViewport: params.pairedGestureViewport, touchExecutor: params.touchExecutor, gestures: params.gestures, @@ -139,6 +141,32 @@ function readOffscreenTargetRescue( }); } +/** + * The pre-dispatch hittability probe over the same conditional live selector read as the rescue + * above, installed under the same admission. Only the local Apple owner on iOS-family leaves + * declares the read, so no other platform or provider-owned device holds the hook. + */ +function readTargetHittabilityProbe( + params: InteractionRouteInput & { + touchExecutor?: BoundTouchExecutor; + gestures?: BoundGestureExecutor; + }, + session: Parameters[0]['session'], +): InteractionRuntimeInput['probeTargetHittability'] { + const querySelector = params.touchExecutor?.querySelector ?? params.gestures?.querySelector; + if (!querySelector) return undefined; + return async (node, signal) => + await probeIosTargetHittability({ + session, + node, + signal, + querySelector, + execution: runtimeExecutionFromContext( + params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + ), + }); +} + type FinalizeTouchInteractionInput = Omit< Parameters[0], 'operations' @@ -213,6 +241,12 @@ function createInteractionBackend(params: InteractionRuntimeInput): AgentDeviceB await params.confirmOffscreenTargetVisible!(node, rootViewport), } : {}), + ...(params.probeTargetHittability + ? { + probeTargetHittability: async (context, node) => + await params.probeTargetHittability!(node, context.signal), + } + : {}), ...touchBackendMembers( params.touchExecutor, params.expireRefFrame, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index bd70e4641a..d124145eed 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -224,6 +224,9 @@ function pressRuntimeOptions( // longpress or hover, so declaring it on their options would be an // unconsumed claim (#1649 review). preresolvedTarget: params.preresolvedTarget, + ...(flags?.maestro?.allowNonHittableCoordinateFallback === true + ? { allowNonHittableCoordinateFallback: true } + : {}), }; } diff --git a/packages/platform-apple/src/__tests__/index-runner-requests.test.ts b/packages/platform-apple/src/__tests__/index-runner-requests.test.ts index 19bed7af82..3f86e304cd 100644 --- a/packages/platform-apple/src/__tests__/index-runner-requests.test.ts +++ b/packages/platform-apple/src/__tests__/index-runner-requests.test.ts @@ -223,6 +223,17 @@ const SCOPED_SITES: Record = { IOS_SIMULATOR, () => queryAppleRunnerSelector(IOS_SIMULATOR, { key: 'label', value: 'Go' }, APP, {}), ], + 'ios-simulator.runner-selector-query.hittability-probe': [ + IOS_SIMULATOR, + () => + queryAppleRunnerSelector( + IOS_SIMULATOR, + { key: 'label', value: 'Go' }, + APP, + {}, + 'hittability', + ), + ], 'macos.screenshot-runner.fullscreen': [ MACOS_DEVICE, (outPath) => diff --git a/packages/platform-apple/src/__tests__/runtime-selector-query.test.ts b/packages/platform-apple/src/__tests__/runtime-selector-query.test.ts index d9c85203ba..d2d18ecc06 100644 --- a/packages/platform-apple/src/__tests__/runtime-selector-query.test.ts +++ b/packages/platform-apple/src/__tests__/runtime-selector-query.test.ts @@ -49,9 +49,51 @@ test('forwards the selector, app scope, and the full runner-execution projection verbose: true, iosXctestrunFile: '/tmp/run.xctestrun', }), + undefined, ); }); +test('the hittability probe forwards its mode and projects the runner reading', async () => { + queryAppleRunnerSelector.mockResolvedValue({ + found: true, + matchCount: 1, + hittable: false, + rect: { x: 294, y: 425, width: 88, height: 82 }, + }); + + const result = await bind().querySelector({ + selector: { key: 'label', value: 'Delete Avocado' }, + probe: 'hittability', + }); + + expect(queryAppleRunnerSelector).toHaveBeenCalledWith( + IOS_SIMULATOR, + { key: 'label', value: 'Delete Avocado' }, + undefined, + expect.anything(), + 'hittability', + ); + expect(result).toEqual({ + found: true, + hittability: { + matchCount: 1, + hittable: false, + rect: { x: 294, y: 425, width: 88, height: 82 }, + }, + }); +}); + +test('a probe reading with several matches carries a count and no frame', async () => { + queryAppleRunnerSelector.mockResolvedValue({ found: true, matchCount: 2 }); + + const result = await bind().querySelector({ + selector: { key: 'label', value: 'Delete' }, + probe: 'hittability', + }); + + expect(result).toEqual({ found: true, hittability: { matchCount: 2 } }); +}); + test('refuses an already-aborted binding signal without querying the runner', async () => { const controller = new AbortController(); controller.abort(); diff --git a/packages/platform-apple/src/core/runner-selector-query.ts b/packages/platform-apple/src/core/runner-selector-query.ts index 1ff7eebdba..a62e4e1aca 100644 --- a/packages/platform-apple/src/core/runner-selector-query.ts +++ b/packages/platform-apple/src/core/runner-selector-query.ts @@ -11,6 +11,7 @@ export async function queryAppleRunnerSelector( selector: Readonly<{ key: ElementSelectorKey; value: string }>, appBundleId: string | undefined, options: AppleRunnerCommandOptions, + probe?: 'hittability', ): Promise> { const result = await runAppleRunnerCommand( device, @@ -19,6 +20,7 @@ export async function queryAppleRunnerSelector( selectorKey: selector.key, selectorValue: selector.value, appBundleId, + ...(probe === 'hittability' ? { hittabilityProbe: true } : {}), }, options, ); diff --git a/packages/platform-apple/src/runtime-selector-query.ts b/packages/platform-apple/src/runtime-selector-query.ts index 5ff076e49c..d158ac8a86 100644 --- a/packages/platform-apple/src/runtime-selector-query.ts +++ b/packages/platform-apple/src/runtime-selector-query.ts @@ -1,4 +1,5 @@ import type { + HittabilityProbeReading, QuerySelectorInput, QuerySelectorNodeReading, QuerySelectorResult, @@ -43,7 +44,15 @@ export function bindAppleQuerySelectorRuntime( input.selector, input.options?.appBundleId, options, + input.probe, ); + if (input.probe === 'hittability') { + const hittability = readHittabilityProbe(data); + return Object.freeze({ + found: data.found === true, + ...(hittability ? { hittability } : {}), + }); + } const node = readQuerySelectorNode(data); return Object.freeze({ found: data.found === true, @@ -53,6 +62,17 @@ export function bindAppleQuerySelectorRuntime( }); } +function readHittabilityProbe(data: Record): HittabilityProbeReading | undefined { + const { matchCount, hittable, rect } = data; + if (typeof matchCount !== 'number') return undefined; + const frame = parseRect(rect); + return Object.freeze({ + matchCount, + ...(typeof hittable === 'boolean' ? { hittable } : {}), + ...(frame ? { rect: frame } : {}), + }); +} + function readQuerySelectorNode( data: Record, ): QuerySelectorNodeReading | undefined { diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index e50ad1e79d..0e8f3950ab 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -27,6 +27,15 @@ export const INTERACTION_ERROR_REASONS = { targetBoundsInvalid: 'target_bounds_invalid', /** The target resolved, but another visible element covers it. */ targetCovered: 'target_covered', + /** + * A press or click refused before dispatch because the platform reports the resolved control as + * not hittable: a tap at its centre would land on another element. Distinct from + * `target_covered`, which needs occlusion evidence; this reason carries none, so it names no + * `coveredBy`. `details.selector` or `details.ref` names the target and `details.rect` is where + * the command resolved it. The owner is + * `packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission.ts`. + */ + targetNotHittable: 'target_not_hittable', /** `find focus` or `find type` resolved to a disabled text field, which `details.ref` names. */ targetDisabled: 'target_disabled', /** diff --git a/packages/selectors/src/interaction-targeting-nested-control.test.ts b/packages/selectors/src/interaction-targeting-nested-control.test.ts new file mode 100644 index 0000000000..a11745f9b4 --- /dev/null +++ b/packages/selectors/src/interaction-targeting-nested-control.test.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from './snapshot-geometry.fixtures.ts'; +import { isNestedTouchControl } from './interaction-targeting.ts'; + +function node(index: number, type: string, label: string, rect: Rect): SnapshotNode { + return { index, depth: 1, parentIndex: 0, type, label, rect, hittable: true }; +} + +const APP = node(0, 'Application', 'App', { x: 0, y: 0, width: 402, height: 874 }); +const ROW = node(1, 'Cell', 'Avocado, 3 item', { x: 20, y: 425, width: 362, height: 82 }); + +function gateOf(nodes: SnapshotNode[], label: string): boolean { + const { nodes: tree } = makeSnapshotState(nodes); + const target = tree.find((candidate) => candidate.label === label); + assert.ok(target, `fixture has no node labelled ${label}`); + return isNestedTouchControl(tree, target); +} + +test('a swipe action button inside its actionable row is gated', () => { + const action = node(2, 'Button', 'Delete Avocado', { x: 294, y: 425, width: 88, height: 82 }); + + assert.equal(gateOf([APP, action, ROW], 'Delete Avocado'), true); +}); + +test('the row itself, which holds the action, is not gated', () => { + const action = node(2, 'Button', 'Delete Avocado', { x: 294, y: 425, width: 88, height: 82 }); + + assert.equal(gateOf([APP, action, ROW], 'Avocado, 3 item'), false); +}); + +test('a top-level button beside the row is not gated', () => { + const add = node(2, 'Button', 'add, Add item', { x: 20, y: 610, width: 362, height: 52 }); + + assert.equal(gateOf([APP, ROW, add], 'add, Add item'), false); +}); + +test('a button inside a container that is not itself a touch target is not gated', () => { + const scroll = node(1, 'ScrollView', 'List', { x: 0, y: 116, width: 402, height: 675 }); + const action = node(2, 'Button', 'Delete Avocado', { x: 294, y: 425, width: 88, height: 82 }); + + assert.equal(gateOf([APP, scroll, action], 'Delete Avocado'), false); +}); + +test('a tab bar item inside its tab bar is not gated', () => { + const tabBar = node(1, 'TabBar', 'Tab Bar', { x: 0, y: 791, width: 402, height: 83 }); + const tab = node(2, 'Button', 'Profile', { x: 243, y: 795, width: 100, height: 54 }); + + assert.equal(gateOf([APP, tabBar, tab], 'Profile'), false); +}); + +test('a wrapper that reports the same rect as its control does not gate the control', () => { + const wrapper = node(1, 'Button', 'Save', { x: 348, y: 429, width: 22, height: 22 }); + const control = node(2, 'Button', 'Save control', { x: 348.5, y: 429.4, width: 22, height: 22 }); + + assert.equal(gateOf([APP, wrapper, control], 'Save control'), false); +}); + +test('a button that only overlaps a larger touch control is not gated', () => { + const card = node(1, 'Button', 'Lemon Herb Rice', { x: 32, y: 427, width: 308, height: 90 }); + const save = node(2, 'Button', 'Save Lemon Herb Rice', { x: 338, y: 429, width: 22, height: 22 }); + + assert.equal(gateOf([APP, card, save], 'Save Lemon Herb Rice'), false); +}); diff --git a/packages/selectors/src/interaction-targeting.ts b/packages/selectors/src/interaction-targeting.ts index d99e7e2dbe..66f79b872c 100644 --- a/packages/selectors/src/interaction-targeting.ts +++ b/packages/selectors/src/interaction-targeting.ts @@ -418,3 +418,35 @@ function isRectViewportSized(rect: Rect, viewportRect: Rect): boolean { const rectCoverage = overlapArea / rectArea; return viewportCoverage >= 0.9 && rectCoverage >= 0.8; } + +/** Rect edges the platform reports a nested control and its row with can differ by this much. */ +const NESTED_CONTROL_SLACK = 1; + +/** + * Whether `node` is a touch control that lies inside the area of a larger touch control of the same + * snapshot: the shape of a swipe action beside its row. Only such a control can be reported + * geometrically hittable while a tap at its centre reaches the row instead, so only such a control + * is worth a native hittability read before a press. A top-level control, and a control inside a + * container that is not itself a touch target (a tab bar, a scroll area), never match. + * + * Document order is not paint order, so this decides nothing about cover. It only selects which + * presses pay for the native read. + */ +export function isNestedTouchControl(nodes: readonly SnapshotNode[], node: SnapshotNode): boolean { + const rect = normalizeRect(node.rect); + if (!rect || !isSemanticTouchTarget(node)) return false; + return nodes.some((other) => { + if (other.index === node.index || !isSemanticTouchTarget(other)) return false; + const outer = normalizeRect(other.rect); + if (!outer) return false; + const encloses = + outer.x <= rect.x + NESTED_CONTROL_SLACK && + outer.y <= rect.y + NESTED_CONTROL_SLACK && + outer.x + outer.width >= rect.x + rect.width - NESTED_CONTROL_SLACK && + outer.y + outer.height >= rect.y + rect.height - NESTED_CONTROL_SLACK; + const larger = + outer.width > rect.width + NESTED_CONTROL_SLACK || + outer.height > rect.height + NESTED_CONTROL_SLACK; + return encloses && larger; + }); +} diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index b6c7a94d8a..6134d67b09 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -511,7 +511,7 @@ A successful `click` or `press` means the action was sent to where the command f Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. -A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. +A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. On iOS, a runtime-resolved `click` or `press` whose target is a control nested inside a larger touch control — a swipe action beside its row — also asks the runner whether the platform reports that control as hittable. When it does not, the command refuses with `target_not_hittable` and `dispatched: no`, because a tap at the control's centre would land on another element; `error.details.rect` is where the command resolved it. Maestro replay is the exception: a click that carries `allowNonHittableCoordinateFallback` skips this read, taps resolved bounds, and on the direct-iOS selector route sends the selector straight to the runner, which may use the non-hittable coordinate fallback. Reveal the control (for example swipe its row) and take a fresh `snapshot -i`. An answer the runner cannot tie to the target — no selector, an ambiguous match, another frame — lets the press proceed as before. Other presses pay no extra read. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. A comma-separated ref plan — `press @e13,@e14` or the same form on `click` — dispatches 2 to 16 snapshot refs as one command. The whole plan resolves and admits against the current ref frame before the first press (run `snapshot -i` first and copy the refs exactly); a repeated ref dispatches the same key twice. The first press expires the frame, and the later keys dispatch at the points the snapshot admitted — no member re-resolves. A refusal before the first press leaves the frame active with `dispatched: no`; a refusal after the first press reports `details.dispatchedSteps` (keys already dispatched), `details.refPlanKey` (the refused key), and `details.refPlanPending` (the keys that did not run) — replan from a fresh `snapshot -i`, because the frame stays expired; a failed plan returns those error details, not `sequence.points`. On a successful response, `ref` stays the first key and `sequence.points` holds every dispatched point. While script recording is armed, the plan refuses unless the step passes `--no-record`, because the recorder binds one target per step: record one press per key. This is ADR 0014's multi-ref logical command; see [ADR 0014](https://github.com/callstack/agent-device/blob/main/docs/adr/0014-session-ref-frame-lifetime.md).