Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,36 @@ extension RunnerTests {
#endif
}

/// The `wait text` rule for native `findText`, pinned to `contracts/fixtures/wait-text-policy.json`.
/// An enabled text-entry element's value is typed content unless it is the placeholder, so such
/// an element matches only by identifier or by a label that is not its value. The types are the
/// ones the snapshot marks editable.
static func findTextPredicate(_ text: String) -> NSPredicate {
let editableTypes = [
XCUIElement.ElementType.textField,
.secureTextField,
.searchField,
.textView,
].map(\.rawValue)
let typed = NSPredicate(
format: "elementType IN %@ AND enabled == YES AND value != placeholderValue",
editableTypes
)
return NSCompoundPredicate(orPredicateWithSubpredicates: [
NSPredicate(format: "identifier CONTAINS[c] %@", text),
NSCompoundPredicate(andPredicateWithSubpredicates: [
NSCompoundPredicate(notPredicateWithSubpredicate: typed),
NSPredicate(format: "label CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text),
]),
NSCompoundPredicate(andPredicateWithSubpredicates: [
typed,
NSPredicate(format: "label CONTAINS[c] %@ AND label != value", text),
]),
])
}

func findElement(app: XCUIApplication, text: String) throws -> XCUIElement? {
let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
let predicate = Self.findTextPredicate(text)
return try elementQueries(app: app, matching: .any)
.lazy
.map { $0.matching(predicate).firstMatch }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,54 @@ extension RunnerTests {
XCTAssertEqual(events.count, 1)
XCTAssertEqual(events.first?.vertical, -200)
}

func testFindTextPredicateMatchesWaitTextGoldenTable() throws {
struct Case: Decodable {
struct Node: Decodable {
let editable: Bool?
let enabled: Bool?
let label: String?
let value: String?
let identifier: String?
let placeholder: String?
}
let name: String
let platform: String?
let text: String
let node: Node
let expected: Bool
}
let fixtureURL = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent() // UnitTests
.deletingLastPathComponent() // AgentDeviceRunnerUITests
.deletingLastPathComponent() // AgentDeviceRunner
.deletingLastPathComponent() // runner
.deletingLastPathComponent() // apple
.deletingLastPathComponent() // repo root
.appendingPathComponent("contracts/fixtures/wait-text-policy.json")
let cases = try JSONDecoder().decode([Case].self, from: Data(contentsOf: fixtureURL))
XCTAssertFalse(cases.isEmpty, "wait text policy table must not be empty")

let editableTypes: [XCUIElement.ElementType] = [.textField, .secureTextField, .searchField, .textView]
// Android and web rows pin the TS twin; the runner sees Apple element types only.
for fixture in cases where fixture.platform == nil {
for type in fixture.node.editable == true ? editableTypes : [.staticText, .other] {
var attributes: [String: Any] = [
"elementType": type.rawValue,
"enabled": fixture.node.enabled ?? true,
]
attributes["label"] = fixture.node.label
attributes["value"] = fixture.node.value
attributes["identifier"] = fixture.node.identifier
attributes["placeholderValue"] = fixture.node.placeholder
XCTAssertEqual(
Self.findTextPredicate(fixture.text).evaluate(with: attributes),
fixture.expected,
"\(fixture.name) (elementType \(type.rawValue))"
)
}
}
}
}
#endif

Expand Down
162 changes: 162 additions & 0 deletions contracts/fixtures/wait-text-policy.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
[
{
"name": "static text matches by label, case-insensitively",
"text": "saved",
"node": {
"label": "Name Saved"
},
"expected": true
},
{
"name": "a non-editable value still matches",
"text": "bench-ad",
"node": {
"label": "Device name",
"value": "bench-ad"
},
"expected": true
},
{
"name": "an editable field's typed value does not match",
"text": "benchad",
"node": {
"editable": true,
"label": "First name",
"value": "benchad"
},
"expected": false
},
{
"name": "part of an editable field's typed value does not match",
"text": "bench",
"node": {
"editable": true,
"label": "First name",
"value": "benchad"
},
"expected": false
},
{
"name": "an editable field still matches by its label",
"text": "first name",
"node": {
"editable": true,
"label": "First name",
"value": "benchad"
},
"expected": true
},
{
"name": "an editable field whose label repeats its typed value does not match",
"text": "bench-ad",
"node": {
"editable": true,
"label": "bench-ad",
"value": "bench-ad",
"identifier": "android:id/edit"
},
"expected": false
},
{
"name": "an editable field still matches by its identifier",
"text": "id/edit",
"node": {
"editable": true,
"label": "bench-ad",
"value": "bench-ad",
"identifier": "android:id/edit"
},
"expected": true
},
{
"name": "an empty editable field matches by its hint label",
"text": "device name",
"node": {
"editable": true,
"label": "Device name",
"value": ""
},
"expected": true
},
{
"name": "a hint reported as both label and value still matches as the placeholder",
"text": "device name",
"node": {
"editable": true,
"label": "Device name",
"value": "Device name",
"placeholder": "Device name"
},
"expected": true
},
{
"name": "an empty field whose value is its placeholder matches by that placeholder",
"text": "last name",
"node": {
"editable": true,
"value": "Last name",
"placeholder": "Last name"
},
"expected": true
},
{
"name": "typed text equal to the placeholder does not match when the field reports its hint is hidden",
"platform": "android",
"text": "device name",
"node": {
"type": "android.widget.EditText",
"editable": true,
"hintShowing": false,
Comment thread
thymikee marked this conversation as resolved.
Resolved
"label": "Device name",
"value": "Device name",
"placeholder": "Device name"
},
"expected": false
},
{
"name": "a field that reports its hint showing matches by that hint even without a placeholder",
"platform": "android",
"text": "device name",
"node": {
"type": "android.widget.EditText",
"editable": true,
"hintShowing": true,
"label": "Device name",
"value": "Device name"
},
"expected": true
},
{
"name": "a disabled field's value is app text and matches",
"text": "bench-ad",
"node": {
"editable": true,
"enabled": false,
"label": "Name",
"value": "bench-ad"
},
"expected": true
},
{
"name": "an Android EditText without an editable fact is editable by its class",
"platform": "android",
"text": "bench-ad",
"node": {
"type": "android.widget.EditText",
"label": "bench-ad",
"value": "bench-ad"
},
"expected": false
},
{
"name": "a web textbox without an editable fact is editable by its role",
"platform": "web",
"text": "bench-ad",
"node": {
"type": "textbox",
"label": "Device name",
"value": "bench-ad"
},
"expected": false
}
]
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type {
WaitCommandOptions,
WaitCommandResult,
} from './selector-wait.ts';
import { findNodeByLabel } from './selector-read-utils.ts';
import { matchesWaitText } from '@agent-device/selectors';
import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy';
import { createWaitPolling, type WaitPollDeadline, waitTimeoutError } from './wait-polling.ts';

Expand Down Expand Up @@ -57,7 +57,9 @@ async function snapshotContainsText<Runtime extends SelectorWaitRuntime>(
updateSession: true,
includeHiddenContentHints: false,
});
return Boolean(findNodeByLabel(capture.snapshot.nodes, text));
return capture.snapshot.nodes.some((node) =>
matchesWaitText(node, text, runtime.backend.platform),
);
}

function backendContext(
Expand Down
2 changes: 1 addition & 1 deletion packages/command-surface/src/capture/wait.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export const WAIT_COMMAND_OUTPUT_SCHEMAS = {
} satisfies Pick<Record<keyof CommandResultMap, JsonSchema>, 'wait'>;

const waitCommandDescription =
'Wait for a duration, text, snapshot ref, selector to appear, selector to be strictly absent, or stable UI. Use the strict absence target when zero selector matches are required; stable waits until the UI stays quiet for the requested window.';
'Wait for a duration, text, snapshot ref, selector to appear, selector to be strictly absent, or stable UI. Text matches a label, identifier, or value; an editable field never matches by its current value or by a label that only repeats it, so text you typed does not satisfy it. Use the strict absence target when zero selector matches are required; stable waits until the UI stays quiet for the requested window.';

const waitCommandMetadata = defineFieldCommandMetadata(WAIT_COMMAND_NAME, waitCommandDescription, {
kind: enumField(WAIT_KIND_VALUES),
Expand Down
6 changes: 6 additions & 0 deletions packages/contracts/src/snapshot-text.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,9 @@ test('unknown fillable type does not establish Android field content', () => {
expect(isFillableType('', 'android')).toBe(true);
expect(androidFieldValue({ type: '', text: 'Static label' })).toBeUndefined();
});

test('a web text input is fillable by its ARIA role, a button is not', () => {
expect(isFillableType('textbox', 'web')).toBe(true);
expect(isFillableType('searchbox', 'web')).toBe(true);
expect(isFillableType('button', 'web')).toBe(false);
});
2 changes: 2 additions & 0 deletions packages/contracts/src/snapshot-text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ export function isFillableType(type: string, platform: Platform | PublicPlatform
if (platform === 'android') {
return normalized.includes('edittext') || normalized.includes('autocompletetextview');
}
// Web nodes carry ARIA roles: a text input is a `textbox` or `searchbox`.
if (platform === 'web' && (normalized === 'textbox' || normalized === 'searchbox')) return true;
return (
normalized.includes('textfield') ||
normalized.includes('securetextfield') ||
Expand Down
2 changes: 2 additions & 0 deletions packages/selectors/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
normalizeIsPositionals,
} from './internal/predicates.ts';
import { selectorFailureHint, STALE_REF_HINT } from './internal/resolve.ts';
import { matchesWaitText } from './internal/node.ts';
import {
findBestMatchesByLocator,
checkFindArgs,
Expand Down Expand Up @@ -70,6 +71,7 @@ export {
isRoleHintWord,
isSelectorToken,
isValidSelectorExpression,
matchesWaitText,
normalizeIsPositionals,
normalizeSelectorText,
parseFindArgs,
Expand Down
40 changes: 40 additions & 0 deletions packages/selectors/src/internal/node-wait-text.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { test } from 'vitest';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import type { SnapshotNode } from '@agent-device/kernel/snapshot';
import { matchesWaitText } from './node.ts';

// Golden `wait text` table: the Swift runner's native findText predicate is asserted against the
// same JSON (UnitTests/RunnerTests+InteractionTests.swift).
type WaitTextCase = {
name: string;
platform?: 'android' | 'web';
text: string;
node: Omit<SnapshotNode, 'index' | 'ref'>;
expected: boolean;
};

const TABLE_PATH = path.resolve(
import.meta.dirname,
'..',
'..',
'..',
'..',
'contracts',
'fixtures',
'wait-text-policy.json',
);

test('wait text matches visible text but not an editable field’s typed content', () => {
const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as WaitTextCase[];
assert.ok(cases.length > 0, 'wait text policy table must not be empty');
for (const fixture of cases) {
const node = { index: 0, ref: 'e1', type: 'StaticText', ...fixture.node };
assert.equal(
matchesWaitText(node, fixture.text, fixture.platform ?? 'ios'),
fixture.expected,
fixture.name,
);
}
});
Loading
Loading