From bfd8ce71b11ffca5c4cb55f770aa87dfe9dd46ff Mon Sep 17 00:00:00 2001 From: Vahagn Zakaryan Date: Sat, 14 Nov 2026 02:22:04 +0400 Subject: [PATCH] fix(android): carry an empty text on a node labelled by its content description --- .../runtime/snapshot-unchanged.test.ts | 1 + .../capture/runtime/snapshot-unchanged.ts | 1 + .../server/__tests__/response-views.test.ts | 1 + packages/daemon/src/server/response-views.ts | 1 + packages/kernel/src/snapshot.ts | 6 ++++++ .../platform-android/src/ui-hierarchy-node.ts | 1 + packages/platform-android/src/ui-hierarchy.ts | 1 + .../src/webdriver-source.test.ts | 8 +++++--- .../src/webdriver-source.ts | 11 ++++++---- src/__tests__/android-ui-hierarchy.test.ts | 20 +++++++++++++++++++ 10 files changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts index 2cce5c8a84..5d1b1d304b 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -75,6 +75,7 @@ test.each>([ { value: 'Draft' }, { identifier: 'create' }, { contentDescription: 'Create a draft' }, + { text: '' }, { enabled: false }, { selected: true }, { checked: true }, diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts index 90957fd910..7d293b61d8 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts @@ -119,6 +119,7 @@ const PRESENTATION_SCALAR_FIELDS = { label: true, value: true, contentDescription: true, + text: true, identifier: true, enabled: true, editable: true, diff --git a/packages/daemon/src/server/__tests__/response-views.test.ts b/packages/daemon/src/server/__tests__/response-views.test.ts index 61dfd7e185..90eec16bb9 100644 --- a/packages/daemon/src/server/__tests__/response-views.test.ts +++ b/packages/daemon/src/server/__tests__/response-views.test.ts @@ -336,6 +336,7 @@ test('get attrs digest compacts the node under a ref target', () => { test('attrs digest keeps explicit false/zero/empty field facts; unavailable ones stay absent (#2288)', () => { const fieldFacts = { value: '', + text: '', editable: false, password: false, hintShowing: false, diff --git a/packages/daemon/src/server/response-views.ts b/packages/daemon/src/server/response-views.ts index 117b818b19..54edab34c7 100644 --- a/packages/daemon/src/server/response-views.ts +++ b/packages/daemon/src/server/response-views.ts @@ -116,6 +116,7 @@ const SELECTOR_DIGEST_NODE_FIELDS = [ 'label', 'value', 'contentDescription', + 'text', 'identifier', 'enabled', 'selected', diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index dab8618c9d..e8dc06b0c0 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -300,6 +300,12 @@ export type RawSnapshotNode = { * filled or hinted field) is carried here for consumers that want the accessible name. */ contentDescription?: string; + /** + * Android view text when it is not already the `label`: the empty string on a view that has + * a content description and no text, where the description is the label. Absent otherwise, + * so `text ?? label` is the text the view shows. + */ + text?: string; identifier?: string; rect?: Rect; enabled?: boolean; diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index 1499f45034..34197445dc 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -4,6 +4,7 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; type AndroidPassiveNodeFacts = Pick< RawSnapshotNode, | 'contentDescription' + | 'text' | 'enabled' | 'selected' | 'heading' diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index 2ca401b34d..7ef9b77d94 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -331,6 +331,7 @@ function normalizeAndroidUiHierarchyNode( value: androidFieldValue(attrs), passiveFacts: { contentDescription: attrs.desc && attrs.desc !== label ? attrs.desc : undefined, + text: label && !attrs.text ? '' : undefined, enabled: attrs.enabled, selected: attrs.selected, heading: attrs.heading, diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index 04d11df0bf..312f7267cd 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -35,10 +35,11 @@ test('Android WebDriver facts treat invalid state attributes as false', () => { // by, so a filled field used to read as a node labelled with its own value and no value at all: // `toHaveValue` saw "" on a field holding "Ada Lovelace". test("Android WebDriver facts name a node by its text, carry a content description beside it, and read a field's text as its value", () => { - const [field, label] = parseWebDriverSourceFacts( + const [field, label, described] = parseWebDriverSourceFacts( '' + '' + '' + + '' + '', 'android', ).nodes; @@ -55,9 +56,10 @@ test("Android WebDriver facts name a node by its text, carry a content descripti ['Ada Lovelace', 'Ada Lovelace', 'Name field', 'Your name', undefined, undefined], ); assert.deepEqual( - [label!.label, label!.value, 'editable' in label!], - ['Ada Lovelace', undefined, false], + [label!.label, label!.value, 'editable' in label!, 'text' in label!, 'text' in field!], + ['Ada Lovelace', undefined, false, false, false], ); + assert.deepEqual([described!.label, described!.text], ['Name field', '']); }); test('Android WebDriver facts read a hinted field as empty, a masked field as a password, and editability as unavailable when unreported', () => { diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 6da5680ebf..243cede760 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -194,22 +194,25 @@ function sourceNodeFromAttributes( * How a node is named. An Android node is labelled by its text and falls back to the content * description only when it has none, as `normalizeAndroidUiHierarchyNode` reads the same * attributes for the native helper; a content description beside visible text travels as - * `contentDescription`. A hinted empty field is labelled by its hint, which is its text. XCUITest - * names a node by `label`, else `name`. + * `contentDescription`, and a node labelled by its description carries an empty `text`. A hinted + * empty field is labelled by its hint, which is its text. XCUITest names a node by `label`, else + * `name`. */ function labelFacts( attrs: Record, platform: WebDriverSourcePlatform, -): Pick { +): Pick { if (platform === 'ios') return { label: firstWebDriverAttribute(attrs, ['label', 'text', 'name']) }; const description = nonEmptyWebDriverAttribute(attrs['content-desc']); - const label = nonEmptyWebDriverAttribute(attrs.text) ?? description; + const text = nonEmptyWebDriverAttribute(attrs.text); + const label = text ?? description; return { label, ...(description !== undefined && description !== label ? { contentDescription: description } : {}), + ...(label !== undefined && text === undefined ? { text: '' } : {}), }; } diff --git a/src/__tests__/android-ui-hierarchy.test.ts b/src/__tests__/android-ui-hierarchy.test.ts index b94b105a69..0f8dbcbea4 100644 --- a/src/__tests__/android-ui-hierarchy.test.ts +++ b/src/__tests__/android-ui-hierarchy.test.ts @@ -136,6 +136,26 @@ test('an Android content description beside text is carried, and one that is the assert.equal(result.nodes[3]?.contentDescription, undefined); }); +test('an Android view labelled by its content description carries an empty text, and one labelled by its text carries none', () => { + const xml = ` + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + + assert.equal(result.nodes[0]?.label, 'saved_task'); + assert.equal(result.nodes[0]?.text, ''); + assert.equal(result.nodes[1]?.label, 'saved_task'); + assert.equal(result.nodes[1]?.text, undefined); + assert.equal(result.nodes[2]?.label, 'Release checklist'); + assert.equal(result.nodes[2]?.text, undefined); + assert.equal(result.nodes[3]?.label, undefined); + assert.equal(result.nodes[3]?.text, undefined); +}); + test('interactive Android snapshots keep a fixed sibling outside filtered scroll content (#1377)', () => { const xml = `