Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,9 @@ extension RunnerTests {
guard let selectorKey = command.selectorKey, let selectorValue = command.selectorValue else {
return Response(ok: false, error: ErrorPayload(message: "querySelector requires selectorKey and selectorValue"))
}
if command.hittabilityProbe == true {
return try probeElementHittability(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue)
}
return try queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue)
case .readText:
guard let x = command.x, let y = command.y else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -334,17 +334,7 @@ extension RunnerTests {
guard !value.isEmpty else {
return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
}
let predicate: NSPredicate
switch selectorKey {
case "id":
predicate = NSPredicate(format: "identifier ==[c] %@", value)
case "label":
predicate = NSPredicate(format: "label ==[c] %@", value)
case "value":
predicate = NSPredicate(format: "value ==[c] %@", value)
case "text":
predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value)
default:
guard let predicate = Self.selectorPredicate(key: selectorKey, value: value) else {
return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
}

Expand Down Expand Up @@ -493,6 +483,47 @@ extension RunnerTests {
try resolveRunnerWindow(app: app).frame
}

static func selectorPredicate(key: String, value: String) -> NSPredicate? {
switch key {
case "id":
return NSPredicate(format: "identifier ==[c] %@", value)
case "label":
return NSPredicate(format: "label ==[c] %@", value)
case "value":
return NSPredicate(format: "value ==[c] %@", value)
case "text":
return NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value)
default:
return nil
}
}

/// The pre-dispatch hittability read: every raw match is counted (hittable or not, unlike
/// `queryElement`'s hittable preference), and only a single match is read, for `isHittable` and
/// its frame, with none of `queryElement`'s other properties. The daemon treats any other count
/// as unknown, so a duplicate label can never describe a different element.
func probeElementHittability(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response {
let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
guard !value.isEmpty, let predicate = Self.selectorPredicate(key: selectorKey, value: value) else {
return Response(ok: true, data: DataPayload(found: false, matchCount: 0))
}
let matches = try elementQueries(app: app, matching: .any)
.flatMap { $0.matching(predicate).allElementsBoundByIndex }
.filter(\.exists)
guard matches.count == 1, let element = matches.first else {
return Response(ok: true, data: DataPayload(found: !matches.isEmpty, matchCount: matches.count))
}
return Response(
ok: true,
data: DataPayload(
found: true,
matchCount: 1,
hittable: element.isHittable,
rect: SnapshotRect(element.frame)
)
)
}

func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response {
// querySelector is a read — it backs get/is/wait and the offscreen-refusal
// double-check, none of which mutate. The fail-closed raw-match rule exists
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,8 @@ struct Command: Codable {
let text: String?
let selectorKey: String?
let selectorValue: String?
/// `querySelector` only: answer the lean per-element hittability read instead of a node.
let hittabilityProbe: Bool?
let allowNonHittableCoordinateFallback: Bool?
let delayMs: Int?
let textEntryMode: String?
Expand Down Expand Up @@ -572,6 +574,11 @@ struct DataPayload: Codable {
/// Present on a screenshot the runner captured from a display it resolved, alongside the
/// `message` path or `imageBase64` payload that carries the image itself (#2728).
var screenshotMetadata: ScreenshotMetadataPayload?
/// The `querySelector` hittability probe: how many elements the selector matched, and, when
/// exactly one did, its `isHittable` and frame.
var matchCount: Int?
var hittable: Bool?
var rect: SnapshotRect?
}

/// Mirrors the TS `FillVerificationTarget`; its nullable identity fields encode as explicit nulls.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@ extension RunnerTests {
}

#if os(iOS)
/// Launches the duplicate-selector fixture, runs one runner command against it, and closes it.
@MainActor
func testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate() throws {
let duplicateIdentifier = "agent-device-selector-read-duplicate"
private func runAgainstDuplicateSelectorFixture(_ commandJson: String) throws -> Response? {
app.launchArguments = ["--agent-device-selector-read-regression"]
app.launch()
defer {
Expand All @@ -96,29 +96,48 @@ extension RunnerTests {
XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout))

let matches = app.descendants(matching: .any)
.matching(identifier: duplicateIdentifier)
.matching(identifier: "agent-device-selector-read-duplicate")
.allElementsBoundByIndex
.filter(\.exists)
XCTAssertEqual(matches.count, 2, "fixture must expose two raw identifier matches")
XCTAssertEqual(matches.filter(\.isHittable).count, 1, "fixture must expose exactly one hittable match")

let command = try JSONDecoder().decode(
Command.self,
from: Data(
#"{"command":"querySelector","commandId":"query-selector-duplicate","selectorKey":"id","selectorValue":"agent-device-selector-read-duplicate"}"#.utf8
)
)
let command = try JSONDecoder().decode(Command.self, from: Data(commandJson.utf8))
let response = try executeOnMainPrepared(command: command, activeApp: app)

guard response.ok else {
XCTFail(String(describing: response.error))
return
return nil
}
return response
}

@MainActor
func testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate() throws {
let duplicateIdentifier = "agent-device-selector-read-duplicate"
guard let response = try runAgainstDuplicateSelectorFixture(
#"{"command":"querySelector","commandId":"query-selector-duplicate","selectorKey":"id","selectorValue":"agent-device-selector-read-duplicate"}"#
) else { return }

XCTAssertEqual(response.data?.found, true)
XCTAssertEqual(response.data?.nodes?.count, 1)
XCTAssertEqual(response.data?.nodes?.first?.identifier, duplicateIdentifier)
XCTAssertEqual(response.data?.nodes?.first?.hittable, true)
}

/// The hittability probe counts every raw match and reads none of them for a duplicate selector:
/// the daemon must never let a duplicate label describe a different element.
@MainActor
func testHittabilityProbeCountsEveryMatchAndReadsNoneForADuplicate() throws {
guard let response = try runAgainstDuplicateSelectorFixture(
#"{"command":"querySelector","commandId":"hittability-probe-duplicate","selectorKey":"id","selectorValue":"agent-device-selector-read-duplicate","hittabilityProbe":true}"#
) else { return }

XCTAssertEqual(response.data?.found, true)
XCTAssertEqual(response.data?.matchCount, 2)
XCTAssertNil(response.data?.hittable)
XCTAssertNil(response.data?.rect)
XCTAssertNil(response.data?.nodes)
}
#endif
#endif
}
1 change: 1 addition & 0 deletions contracts/fixtures/runner-requests.json
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
{"name": "ios-simulator.runner-command-recovery.status", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "status", "statusCommandId": "<commandId>"}},
{"name": "ios-simulator.runner-disposal.shutdown", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "shutdown", "commandId": "<commandId>"}},
{"name": "ios-simulator.runner-lifecycle-prepare.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": "<commandId>"}},
{"name": "ios-simulator.runner-selector-query.hittability-probe", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "hittabilityProbe": true, "commandId": "<commandId>"}},
{"name": "ios-simulator.runner-selector-query.label", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "commandId": "<commandId>"}},
{"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": "<commandId>"}},
{"name": "ios-simulator.runner-session-readiness.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": "<commandId>"}},
Expand Down
2 changes: 2 additions & 0 deletions packages/apple-runner/src/runner-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,8 @@ export type RunnerCommand = {
text?: string;
selectorKey?: ElementSelectorKey;
selectorValue?: string;
/** `querySelector` only: answer the lean per-element hittability read instead of a node. */
hittabilityProbe?: boolean;
allowNonHittableCoordinateFallback?: boolean;
delayMs?: number;
textEntryMode?: 'append' | 'replace';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ export type PressCommandOptions = CommandContext &
expectedResolvedTarget?: ExpectedResolvedTarget;
/** #1654: a mutating `find`'s already-resolved node; see resolution.ts. */
preresolvedTarget?: PreresolvedInteractionTarget;
/** Internal Maestro replay policy: replay taps resolved bounds, so admission skips the hittability read. */
allowNonHittableCoordinateFallback?: boolean;
} & PostActionObservationOptions;

export type ClickCommandOptions = PressCommandOptions;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import type { AgentDeviceBackend, TargetHittability } from '@agent-device/contracts/backend';
import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot';
import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error';
import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter';
import { localCommandPolicy } from '@agent-device/contracts/command-policy';
import { createAgentDevice } from '../../../runtime.ts';
import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts';
import { ref } from './selector-read-utils.ts';

// The pre-dispatch hittability check on the press route: which targets pay for the native read, and
// what each verdict does. The pure gate and the verdict mapping have their own tests beside them.

function node(
index: number,
type: string,
label: string,
rect: SnapshotNode['rect'],
): SnapshotNode {
return { index, depth: 1, parentIndex: 0, type, label, rect, hittable: true };
}

const TREE = makeSnapshotState([
node(0, 'Application', 'App', { x: 0, y: 0, width: 402, height: 874 }),
node(1, 'Button', 'Delete Avocado', { x: 294, y: 425, width: 88, height: 82 }),
node(2, 'Cell', 'Avocado, 3 item, $4.50', { x: 20, y: 425, width: 362, height: 82 }),
node(3, 'Button', 'Add item', { x: 20, y: 610, width: 362, height: 52 }),
]);

function createPressDevice(verdict: TargetHittability | (() => Promise<TargetHittability>)) {
const probes: string[] = [];
const taps: Point[] = [];
const backend: AgentDeviceBackend = {
platform: 'ios',
captureSnapshot: async () => ({ snapshot: TREE }),
probeTargetHittability: async (_context, probed) => {
probes.push(probed.label ?? '');
return typeof verdict === 'function' ? await verdict() : verdict;
},
tap: async (_context, point) => {
taps.push(point);
return { ok: true };
},
};
const device = createAgentDevice({
backend,
artifacts: createLocalArtifactAdapter(),
sessions: createMemorySessionStore([{ name: 'default', snapshot: TREE }]),
policy: localCommandPolicy(),
});
return { device, probes, taps };
}

function refOf(label: string) {
const found = TREE.nodes.find((candidate) => candidate.label === label)!;
return ref(`@${found.ref}`);
}

test('a press on a control the platform reports not hittable is refused before any tap', async () => {
const { device, probes, taps } = createPressDevice('notHittable');

await assert.rejects(device.interactions.press(refOf('Delete Avocado')), (error: unknown) => {
const { message, details } = error as { message: string; details?: Record<string, unknown> };
assert.equal(message, '"Delete Avocado" is not hittable: a tap would land on another element.');
assert.equal(details?.reason, INTERACTION_ERROR_REASONS.targetNotHittable);
assert.equal(details?.dispatched, 'no');
assert.equal(details?.coveredBy, undefined, 'no occlusion evidence is claimed');
assert.equal(
details?.hint,
'Reveal the control (for example swipe its row), then take a fresh snapshot -i.',
);
return true;
});
assert.deepEqual(probes, ['Delete Avocado']);
assert.deepEqual(taps, []);
});

for (const verdict of ['hittable', 'unknown'] as const) {
test(`a gated press the probe answers ${verdict} still taps`, async () => {
const { device, probes, taps } = createPressDevice(verdict);

await device.interactions.press(refOf('Delete Avocado'));

assert.deepEqual(probes, ['Delete Avocado']);
assert.equal(taps.length, 1);
});
}

test('a top-level press pays for no probe even when the backend would refuse', async () => {
const { device, probes, taps } = createPressDevice('notHittable');

await device.interactions.press(refOf('Add item'));

assert.deepEqual(probes, []);
assert.equal(taps.length, 1);
});

test('a probe that rejects the request cancels before any tap', async () => {
const controller = new AbortController();
const { device, taps } = createPressDevice(async () => {
controller.abort();
return 'hittable';
});

await assert.rejects(
device.interactions.press(refOf('Delete Avocado'), { signal: controller.signal }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'request_canceled');
assert.equal(details?.dispatched, 'no');
return true;
},
);
Comment thread
thymikee marked this conversation as resolved.
Resolved
assert.deepEqual(taps, []);
});

test('the Maestro replay fallback keeps the hittability read out of a gated press', async () => {
const { device, probes, taps } = createPressDevice('notHittable');

await device.interactions.press(refOf('Delete Avocado'), {
allowNonHittableCoordinateFallback: true,
});

assert.deepEqual(probes, []);
assert.equal(taps.length, 1);
});
Loading
Loading