diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 2080a023e9..0cc89a7b01 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -190,8 +190,36 @@ extension RunnerTests { #endif } + /// The `wait text` rule for native `findText`, pinned to `contracts/fixtures/wait-text-policy.json`. + /// An enabled text-entry element's value is typed content unless it is the placeholder, so such + /// an element matches only by identifier or by a label that is not its value. The types are the + /// ones the snapshot marks editable. + static func findTextPredicate(_ text: String) -> NSPredicate { + let editableTypes = [ + XCUIElement.ElementType.textField, + .secureTextField, + .searchField, + .textView, + ].map(\.rawValue) + let typed = NSPredicate( + format: "elementType IN %@ AND enabled == YES AND value != placeholderValue", + editableTypes + ) + return NSCompoundPredicate(orPredicateWithSubpredicates: [ + NSPredicate(format: "identifier CONTAINS[c] %@", text), + NSCompoundPredicate(andPredicateWithSubpredicates: [ + NSCompoundPredicate(notPredicateWithSubpredicate: typed), + NSPredicate(format: "label CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text), + ]), + NSCompoundPredicate(andPredicateWithSubpredicates: [ + typed, + NSPredicate(format: "label CONTAINS[c] %@ AND label != value", text), + ]), + ]) + } + func findElement(app: XCUIApplication, text: String) throws -> XCUIElement? { - let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text) + let predicate = Self.findTextPredicate(text) return try elementQueries(app: app, matching: .any) .lazy .map { $0.matching(predicate).firstMatch } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index f610e44596..05bb33f0ec 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -23,6 +23,54 @@ extension RunnerTests { XCTAssertEqual(events.count, 1) XCTAssertEqual(events.first?.vertical, -200) } + + func testFindTextPredicateMatchesWaitTextGoldenTable() throws { + struct Case: Decodable { + struct Node: Decodable { + let editable: Bool? + let enabled: Bool? + let label: String? + let value: String? + let identifier: String? + let placeholder: String? + } + let name: String + let platform: String? + let text: String + let node: Node + let expected: Bool + } + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts/fixtures/wait-text-policy.json") + let cases = try JSONDecoder().decode([Case].self, from: Data(contentsOf: fixtureURL)) + XCTAssertFalse(cases.isEmpty, "wait text policy table must not be empty") + + let editableTypes: [XCUIElement.ElementType] = [.textField, .secureTextField, .searchField, .textView] + // Android and web rows pin the TS twin; the runner sees Apple element types only. + for fixture in cases where fixture.platform == nil { + for type in fixture.node.editable == true ? editableTypes : [.staticText, .other] { + var attributes: [String: Any] = [ + "elementType": type.rawValue, + "enabled": fixture.node.enabled ?? true, + ] + attributes["label"] = fixture.node.label + attributes["value"] = fixture.node.value + attributes["identifier"] = fixture.node.identifier + attributes["placeholderValue"] = fixture.node.placeholder + XCTAssertEqual( + Self.findTextPredicate(fixture.text).evaluate(with: attributes), + fixture.expected, + "\(fixture.name) (elementType \(type.rawValue))" + ) + } + } + } } #endif diff --git a/contracts/fixtures/wait-text-policy.json b/contracts/fixtures/wait-text-policy.json new file mode 100644 index 0000000000..17a3adc41d --- /dev/null +++ b/contracts/fixtures/wait-text-policy.json @@ -0,0 +1,162 @@ +[ + { + "name": "static text matches by label, case-insensitively", + "text": "saved", + "node": { + "label": "Name Saved" + }, + "expected": true + }, + { + "name": "a non-editable value still matches", + "text": "bench-ad", + "node": { + "label": "Device name", + "value": "bench-ad" + }, + "expected": true + }, + { + "name": "an editable field's typed value does not match", + "text": "benchad", + "node": { + "editable": true, + "label": "First name", + "value": "benchad" + }, + "expected": false + }, + { + "name": "part of an editable field's typed value does not match", + "text": "bench", + "node": { + "editable": true, + "label": "First name", + "value": "benchad" + }, + "expected": false + }, + { + "name": "an editable field still matches by its label", + "text": "first name", + "node": { + "editable": true, + "label": "First name", + "value": "benchad" + }, + "expected": true + }, + { + "name": "an editable field whose label repeats its typed value does not match", + "text": "bench-ad", + "node": { + "editable": true, + "label": "bench-ad", + "value": "bench-ad", + "identifier": "android:id/edit" + }, + "expected": false + }, + { + "name": "an editable field still matches by its identifier", + "text": "id/edit", + "node": { + "editable": true, + "label": "bench-ad", + "value": "bench-ad", + "identifier": "android:id/edit" + }, + "expected": true + }, + { + "name": "an empty editable field matches by its hint label", + "text": "device name", + "node": { + "editable": true, + "label": "Device name", + "value": "" + }, + "expected": true + }, + { + "name": "a hint reported as both label and value still matches as the placeholder", + "text": "device name", + "node": { + "editable": true, + "label": "Device name", + "value": "Device name", + "placeholder": "Device name" + }, + "expected": true + }, + { + "name": "an empty field whose value is its placeholder matches by that placeholder", + "text": "last name", + "node": { + "editable": true, + "value": "Last name", + "placeholder": "Last name" + }, + "expected": true + }, + { + "name": "typed text equal to the placeholder does not match when the field reports its hint is hidden", + "platform": "android", + "text": "device name", + "node": { + "type": "android.widget.EditText", + "editable": true, + "hintShowing": false, + "label": "Device name", + "value": "Device name", + "placeholder": "Device name" + }, + "expected": false + }, + { + "name": "a field that reports its hint showing matches by that hint even without a placeholder", + "platform": "android", + "text": "device name", + "node": { + "type": "android.widget.EditText", + "editable": true, + "hintShowing": true, + "label": "Device name", + "value": "Device name" + }, + "expected": true + }, + { + "name": "a disabled field's value is app text and matches", + "text": "bench-ad", + "node": { + "editable": true, + "enabled": false, + "label": "Name", + "value": "bench-ad" + }, + "expected": true + }, + { + "name": "an Android EditText without an editable fact is editable by its class", + "platform": "android", + "text": "bench-ad", + "node": { + "type": "android.widget.EditText", + "label": "bench-ad", + "value": "bench-ad" + }, + "expected": false + }, + { + "name": "a web textbox without an editable fact is editable by its role", + "platform": "web", + "text": "bench-ad", + "node": { + "type": "textbox", + "label": "Device name", + "value": "bench-ad" + }, + "expected": false + } +] diff --git a/packages/command-runtime/src/commands/interaction/runtime/wait-text.ts b/packages/command-runtime/src/commands/interaction/runtime/wait-text.ts index 70732cff07..d69f9ea0f4 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/wait-text.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/wait-text.ts @@ -5,7 +5,7 @@ import type { WaitCommandOptions, WaitCommandResult, } from './selector-wait.ts'; -import { findNodeByLabel } from './selector-read-utils.ts'; +import { matchesWaitText } from '@agent-device/selectors'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import { createWaitPolling, type WaitPollDeadline, waitTimeoutError } from './wait-polling.ts'; @@ -57,7 +57,9 @@ async function snapshotContainsText( updateSession: true, includeHiddenContentHints: false, }); - return Boolean(findNodeByLabel(capture.snapshot.nodes, text)); + return capture.snapshot.nodes.some((node) => + matchesWaitText(node, text, runtime.backend.platform), + ); } function backendContext( diff --git a/packages/command-surface/src/capture/wait.ts b/packages/command-surface/src/capture/wait.ts index 2840e68202..f2a1f777e6 100644 --- a/packages/command-surface/src/capture/wait.ts +++ b/packages/command-surface/src/capture/wait.ts @@ -59,7 +59,7 @@ export const WAIT_COMMAND_OUTPUT_SCHEMAS = { } satisfies Pick, 'wait'>; const waitCommandDescription = - 'Wait for a duration, text, snapshot ref, selector to appear, selector to be strictly absent, or stable UI. Use the strict absence target when zero selector matches are required; stable waits until the UI stays quiet for the requested window.'; + 'Wait for a duration, text, snapshot ref, selector to appear, selector to be strictly absent, or stable UI. Text matches a label, identifier, or value; an editable field never matches by its current value or by a label that only repeats it, so text you typed does not satisfy it. Use the strict absence target when zero selector matches are required; stable waits until the UI stays quiet for the requested window.'; const waitCommandMetadata = defineFieldCommandMetadata(WAIT_COMMAND_NAME, waitCommandDescription, { kind: enumField(WAIT_KIND_VALUES), diff --git a/packages/contracts/src/snapshot-text.test.ts b/packages/contracts/src/snapshot-text.test.ts index 332c5d6a8e..1370f5e92b 100644 --- a/packages/contracts/src/snapshot-text.test.ts +++ b/packages/contracts/src/snapshot-text.test.ts @@ -19,3 +19,9 @@ test('unknown fillable type does not establish Android field content', () => { expect(isFillableType('', 'android')).toBe(true); expect(androidFieldValue({ type: '', text: 'Static label' })).toBeUndefined(); }); + +test('a web text input is fillable by its ARIA role, a button is not', () => { + expect(isFillableType('textbox', 'web')).toBe(true); + expect(isFillableType('searchbox', 'web')).toBe(true); + expect(isFillableType('button', 'web')).toBe(false); +}); diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index c14af4f99c..446575ce0d 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -20,6 +20,8 @@ export function isFillableType(type: string, platform: Platform | PublicPlatform if (platform === 'android') { return normalized.includes('edittext') || normalized.includes('autocompletetextview'); } + // Web nodes carry ARIA roles: a text input is a `textbox` or `searchbox`. + if (platform === 'web' && (normalized === 'textbox' || normalized === 'searchbox')) return true; return ( normalized.includes('textfield') || normalized.includes('securetextfield') || diff --git a/packages/selectors/src/index.ts b/packages/selectors/src/index.ts index c49b28bce8..0a27c616d3 100644 --- a/packages/selectors/src/index.ts +++ b/packages/selectors/src/index.ts @@ -24,6 +24,7 @@ import { normalizeIsPositionals, } from './internal/predicates.ts'; import { selectorFailureHint, STALE_REF_HINT } from './internal/resolve.ts'; +import { matchesWaitText } from './internal/node.ts'; import { findBestMatchesByLocator, checkFindArgs, @@ -70,6 +71,7 @@ export { isRoleHintWord, isSelectorToken, isValidSelectorExpression, + matchesWaitText, normalizeIsPositionals, normalizeSelectorText, parseFindArgs, diff --git a/packages/selectors/src/internal/node-wait-text.test.ts b/packages/selectors/src/internal/node-wait-text.test.ts new file mode 100644 index 0000000000..a12b8362c1 --- /dev/null +++ b/packages/selectors/src/internal/node-wait-text.test.ts @@ -0,0 +1,40 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { matchesWaitText } from './node.ts'; + +// Golden `wait text` table: the Swift runner's native findText predicate is asserted against the +// same JSON (UnitTests/RunnerTests+InteractionTests.swift). +type WaitTextCase = { + name: string; + platform?: 'android' | 'web'; + text: string; + node: Omit; + expected: boolean; +}; + +const TABLE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'wait-text-policy.json', +); + +test('wait text matches visible text but not an editable field’s typed content', () => { + const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as WaitTextCase[]; + assert.ok(cases.length > 0, 'wait text policy table must not be empty'); + for (const fixture of cases) { + const node = { index: 0, ref: 'e1', type: 'StaticText', ...fixture.node }; + assert.equal( + matchesWaitText(node, fixture.text, fixture.platform ?? 'ios'), + fixture.expected, + fixture.name, + ); + } +}); diff --git a/packages/selectors/src/internal/node.ts b/packages/selectors/src/internal/node.ts index c06520fc6a..6ef0aa8689 100644 --- a/packages/selectors/src/internal/node.ts +++ b/packages/selectors/src/internal/node.ts @@ -1,6 +1,10 @@ import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; -import { isFillableType, normalizeType } from '@agent-device/contracts/snapshot'; +import { + isFillableType, + matchesSnapshotScope, + normalizeType, +} from '@agent-device/contracts/snapshot'; export function isNodeVisible(node: SnapshotNode): boolean { if (node.hittable === true) return true; @@ -17,6 +21,27 @@ export function isNodeEditable(node: SnapshotNode, platform: Platform | PublicPl return node.editable ?? isFillableType(node.type ?? '', platform); } +/** + * The `wait text` match rule: the snapshot scope rule minus typed content. An editable node's + * value is what the user typed, and Android also reports that text as the node's label, so an + * editable node matches only by its identifier or by a label that is not its value. A field showing + * its hint (the `hintShowing` fact, else a value equal to the placeholder) holds no typed text. + * The Swift runner's native `findText` is pinned to the same table, + * `contracts/fixtures/wait-text-policy.json`. + */ +export function matchesWaitText( + node: SnapshotNode, + text: string, + platform: Platform | PublicPlatform, +): boolean { + const showsHint = node.hintShowing ?? node.value === node.placeholder; + if (!isNodeEditable(node, platform) || showsHint) { + return matchesSnapshotScope(node, text); + } + const label = node.label === node.value ? undefined : node.label; + return matchesSnapshotScope({ identifier: node.identifier, label }, text); +} + /** * Every `role=` spelling this node accepts, in ONE place (#3021): the * canonical `kind` vocabulary first, then — during the deprecation window — diff --git a/test/integration/provider-scenarios/android-find.test.ts b/test/integration/provider-scenarios/android-find.test.ts index 333ec1a6f1..59bbb1491e 100644 --- a/test/integration/provider-scenarios/android-find.test.ts +++ b/test/integration/provider-scenarios/android-find.test.ts @@ -3,7 +3,8 @@ import { test } from 'vitest'; import { ANDROID_IME_HELPER_FIXTURE_ARTIFACT } from '../../../src/__tests__/test-utils/android-ime-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from '../../../src/__tests__/test-utils/android-snapshot-helper.ts'; import type { AndroidAdbProvider } from '@agent-device/platform-android/mechanics'; -import { arrayEqual, assertCommandCall } from './assertions.ts'; +import { WAIT_REASONS } from '@agent-device/contracts/wait'; +import { arrayEqual, assertCommandCall, assertRpcOk } from './assertions.ts'; import { androidSettingsXml, androidSnapshotHelperOutput } from './android-world.ts'; import { androidImeClearTextBroadcast, @@ -114,6 +115,15 @@ test('Provider-backed integration Android find flow covers refs, wait, ambiguity }); assert.equal(filled.text, 'Network'); + // The field's label repeats its typed value (no editable fact: the EditText class decides), + // so only its identifier satisfies `wait text`. + const typedWait = await daemon.callCommand('wait', ['text', 'Network', '100'], selection); + assert.equal(typedWait.json?.error?.data?.details?.reason, WAIT_REASONS.targetAbsent); + assert.equal( + assertRpcOk(await daemon.callCommand('wait', ['text', 'id/search', '100'], selection)).text, + 'id/search', + ); + const waitForApps = await client.interactions.find({ locator: 'label', query: 'Apps', diff --git a/test/integration/provider-scenarios/ios-wait-selector.test.ts b/test/integration/provider-scenarios/ios-wait-selector.test.ts index 92cfe8a903..bec41aad4a 100644 --- a/test/integration/provider-scenarios/ios-wait-selector.test.ts +++ b/test/integration/provider-scenarios/ios-wait-selector.test.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; +import { WAIT_REASONS } from '@agent-device/contracts/wait'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { assertRpcOk } from './assertions.ts'; @@ -109,3 +110,68 @@ test.each([ await daemon.close(); } }); + +test('Apple wait text ignores a text field’s typed value but matches its label', async () => { + const appleTool = createRecordingAppleToolProvider({ + simctl: async (args) => { + if (args[0] === 'list') { + return { + stdout: JSON.stringify({ + devices: { + 'iOS 18.5': [{ udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, state: 'Booted' }], + }, + }), + stderr: '', + exitCode: 0, + }; + } + if (args[0] === 'launch') return { stdout: '', stderr: '', exitCode: 0 }; + throw new Error(`Unexpected Apple simulator call: ${args.join(' ')}`); + }, + }); + const daemon = await createProviderScenarioHarness({ + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + appleToolProvider: () => appleTool.provider, + appleRunnerProvider: () => ({ + hasLiveSession: () => true, + runCommand: async (_device, command) => { + if (command.command === 'findText') return { found: false }; + assert.equal(command.command, 'snapshot'); + return { + nodes: [ + { + index: 0, + type: 'Application', + label: 'Contacts', + rect: { x: 0, y: 0, width: 393, height: 852 }, + }, + { + index: 1, + parentIndex: 0, + type: 'TextField', + label: 'First name', + value: 'benchad', + rect: { x: 16, y: 200, width: 361, height: 44 }, + }, + ], + }; + }, + }), + }); + try { + await daemon.client().apps.open({ + app: 'com.apple.MobileAddressBook', + platform: 'ios', + udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + }); + const flags = { platform: 'ios' as const }; + const typed = await daemon.callCommand('wait', ['text', 'benchad', '1000'], flags); + assert.equal(typed.json?.error?.data?.details?.reason, WAIT_REASONS.targetAbsent); + assert.equal( + assertRpcOk(await daemon.callCommand('wait', ['text', 'First name', '100'], flags)).text, + 'First name', + ); + } finally { + await daemon.close(); + } +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 03ccafee12..a82e3681fa 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -430,7 +430,8 @@ agent-device alert dismiss - `wait` accepts a millisecond duration, `text `, a snapshot ref (`@eN`), a selector, or strict `absent `. Every form except a plain duration takes an optional `[timeoutMs]`. Selector waits, including those inside `replay`, poll snapshot captures with the same backend and recovery as `is`. `wait` resolves selectors the same way as `click`, `fill`, `get`, and `is`. - `wait absent ` is strict absence, not a visibility check: it passes only when a complete, readable capture has zero matches, so hidden or off-screen matches keep it pending. It rejects `--scope` and `--depth`. Sparse, truncated, incomplete, and Android unreadable-content captures never satisfy it. -- `wait @ref` needs a snapshot from a prior `snapshot` command. It resolves the ref to its label or text, then waits for that text, so a duplicate label can match a different element than the original ref. +- `wait text ` matches a node whose label, identifier, or value contains the text, ignoring case. An enabled editable field never matches by its current value, or by a label that only repeats that value. Its identifier, a label distinct from the value, or the placeholder an empty field shows still match. So `wait text` after `fill` waits until the app shows that text elsewhere, for example after Save. To check the field itself, use `is text ` or `get text`. +- `wait @ref` needs a snapshot from a prior `snapshot` command. It resolves the ref to its label or text, then waits for that text, so a duplicate label can match a different element than the original ref. It uses the same text rule, so for a filled field whose text is its typed value it waits for that text to appear elsewhere; to wait for the field itself, use a selector such as `wait 'id="email"'`. - Wait failures carry `error.details.reason` in `--json` output: - `wait_target_absent`: a positive wait never found a match. - `wait_target_present`: strict `wait absent` reached its deadline with valid captures that still contained matches.