diff --git a/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts b/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts index badab9cc8b..c2cf85c931 100644 --- a/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts +++ b/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts @@ -1,6 +1,10 @@ import { expect, test } from 'vitest'; import { shouldIncludeAndroidNode } from '../ui-hierarchy-inclusion.ts'; import type { AndroidNode } from '../ui-hierarchy-node.ts'; +import { + ANDROID_DEVICE_NAME_DIALOG_RAW_NODES, + walkInteractiveOnlyAndroidFixture, +} from '../ui-hierarchy.fixtures.ts'; test('interactive inclusion keeps positive agent targets and rejects degenerate geometry', () => { expect( @@ -10,6 +14,7 @@ test('interactive inclusion keeps positive agent targets and rejects degenerate false, false, false, + false, ), ).toBe(true); expect( @@ -19,6 +24,7 @@ test('interactive inclusion keeps positive agent targets and rejects degenerate false, false, false, + false, ), ).toBe(false); }); @@ -26,10 +32,33 @@ test('interactive inclusion keeps positive agent targets and rejects degenerate test('interactive inclusion keeps a semantic Compose proxy only in actionable context', () => { const semanticView = node({ type: 'android.view.View', label: 'Start a call' }); expect( - shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, true, false, false), + shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, true, false, false, false), ).toBe(true); expect( - shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, false, false, false), + shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, false, false, false, false), + ).toBe(false); +}); + +test('interactive inclusion keeps an AlertDialog title and drops screen text outside the dialog', () => { + const labels = walkInteractiveOnlyAndroidFixture(ANDROID_DEVICE_NAME_DIALOG_RAW_NODES).map( + (presented) => presented.label, + ); + expect(labels).toContain('Device name'); + expect(labels).toEqual(expect.arrayContaining(['sdk_gphone64_arm64', 'Cancel', 'OK'])); + expect(labels).not.toContain('4:37'); +}); + +test('interactive inclusion keeps passive text only inside an alert panel', () => { + const title = node({ type: 'android.widget.TextView', label: 'Device name' }); + expect( + shouldIncludeAndroidNode(title, { interactiveOnly: true }, false, false, false, true), + ).toBe(true); + expect( + shouldIncludeAndroidNode(title, { interactiveOnly: true }, false, false, false, false), + ).toBe(false); + const panelLayout = node({ type: 'android.widget.LinearLayout', label: 'Panel' }); + expect( + shouldIncludeAndroidNode(panelLayout, { interactiveOnly: true }, false, false, false, true), ).toBe(false); }); @@ -41,13 +70,14 @@ test('regular inclusion preserves meaningful structural nodes and actionable des false, false, false, + false, ), ).toBe(true); expect( - shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, true, false), + shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, true, false, false), ).toBe(true); expect( - shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, false, false), + shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, false, false, false), ).toBe(false); }); diff --git a/packages/platform-android/src/alert-detection.ts b/packages/platform-android/src/alert-detection.ts index faffea3ac9..60af1eebce 100644 --- a/packages/platform-android/src/alert-detection.ts +++ b/packages/platform-android/src/alert-detection.ts @@ -57,6 +57,11 @@ const ANDROID_PERMISSION_ID_PATTERN = /(?:^|:)id\/permission_/i; const ANDROID_BLOCKING_DIALOG_PATTERN = /\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i; +/** The AlertController panel that holds a dialog's title, message, and buttons, under the platform or an AppCompat package. */ +export function isAndroidAlertPanelIdentifier(identifier: string | null): boolean { + return identifier !== null && /(?:^|:)id\/parentPanel$/i.test(identifier); +} + export function classifyAndroidAlertIdentifier( identifier: string | null, ): 'button' | 'content' | undefined { diff --git a/packages/platform-android/src/fixtures/android-device-name-dialog.raw.json b/packages/platform-android/src/fixtures/android-device-name-dialog.raw.json new file mode 100644 index 0000000000..e8611741d7 --- /dev/null +++ b/packages/platform-android/src/fixtures/android-device-name-dialog.raw.json @@ -0,0 +1,778 @@ +[ + { + "index": 0, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 1, + "parentIndex": 0, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/taskbar_container", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 2, + "parentIndex": 1, + "type": "android.view.View", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/taskbar_scrim", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 3, + "parentIndex": 1, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/navbuttons_view", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 4, + "parentIndex": 3, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/start_contextual_buttons", + "rect": { + "x": 21, + "y": 2274, + "width": 103, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 5, + "parentIndex": 3, + "type": "android.widget.LinearLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/end_nav_buttons", + "rect": { + "x": 124, + "y": 2274, + "width": 830, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 6, + "parentIndex": 5, + "type": "android.widget.ImageView", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/back", + "label": "Back", + "rect": { + "x": 125, + "y": 2279, + "width": 207, + "height": 116 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 7, + "parentIndex": 5, + "type": "android.widget.ImageView", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/home", + "label": "Home", + "rect": { + "x": 434, + "y": 2279, + "width": 210, + "height": 116 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 8, + "parentIndex": 5, + "type": "android.widget.ImageView", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/recent_apps", + "label": "Recents", + "rect": { + "x": 746, + "y": 2279, + "width": 207, + "height": 116 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 9, + "parentIndex": 3, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/end_contextual_buttons", + "rect": { + "x": 956, + "y": 2274, + "width": 103, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 10, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 11, + "parentIndex": 10, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_launch_animation_container", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 12, + "parentIndex": 10, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_container", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 13, + "parentIndex": 12, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 14, + "parentIndex": 13, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_contents", + "rect": { + "x": 0, + "y": 0, + "width": 1027, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 15, + "parentIndex": 14, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_start_side_container", + "rect": { + "x": 42, + "y": 1, + "width": 426, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 16, + "parentIndex": 15, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_start_side_content", + "rect": { + "x": 42, + "y": 1, + "width": 220, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 17, + "parentIndex": 16, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_start_side_except_heads_up", + "rect": { + "x": 42, + "y": 1, + "width": 220, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 18, + "parentIndex": 17, + "type": "android.widget.TextView", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/clock", + "label": "4:37", + "rect": { + "x": 42, + "y": 45, + "width": 104, + "height": 43 + }, + "visibleToUser": true + }, + { + "index": 19, + "parentIndex": 17, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/notification_icon_area", + "rect": { + "x": 146, + "y": 1, + "width": 116, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 20, + "parentIndex": 19, + "type": "android.view.ViewGroup", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/notificationIcons", + "rect": { + "x": 146, + "y": 1, + "width": 116, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 21, + "parentIndex": 20, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "label": "Security & privacy notification: Set a screen lock", + "rect": { + "x": 146, + "y": 1, + "width": 58, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 22, + "parentIndex": 20, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "label": "Android System notification: Serial console enabled", + "rect": { + "x": 204, + "y": 1, + "width": 58, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 23, + "parentIndex": 14, + "type": "android.view.View", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/cutout_space_view", + "rect": { + "x": 468, + "y": 1, + "width": 122, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 24, + "parentIndex": 14, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_end_side_container", + "rect": { + "x": 590, + "y": 1, + "width": 426, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 25, + "parentIndex": 24, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_end_side_content", + "rect": { + "x": 857, + "y": 1, + "width": 159, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 26, + "parentIndex": 25, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/system_icons", + "rect": { + "x": 857, + "y": 37, + "width": 159, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 27, + "parentIndex": 26, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/statusIcons", + "rect": { + "x": 865, + "y": 37, + "width": 120, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 28, + "parentIndex": 27, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_combo", + "label": "Phone signal full.", + "rect": { + "x": 865, + "y": 37, + "width": 104, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 29, + "parentIndex": 28, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_group", + "rect": { + "x": 865, + "y": 37, + "width": 104, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 30, + "parentIndex": 29, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_type_container", + "rect": { + "x": 872, + "y": 42, + "width": 55, + "height": 47 + }, + "visibleToUser": true + }, + { + "index": 31, + "parentIndex": 30, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_type", + "label": "3G", + "rect": { + "x": 872, + "y": 44, + "width": 55, + "height": 42 + }, + "visibleToUser": true + }, + { + "index": 32, + "parentIndex": 29, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "rect": { + "x": 930, + "y": 46, + "width": 39, + "height": 39 + }, + "visibleToUser": true + }, + { + "index": 33, + "parentIndex": 32, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_signal", + "rect": { + "x": 930, + "y": 46, + "width": 39, + "height": 39 + }, + "visibleToUser": true + }, + { + "index": 34, + "parentIndex": 27, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "rect": { + "x": 865, + "y": 66, + "width": 0, + "height": 0 + }, + "visibleToUser": false + }, + { + "index": 35, + "parentIndex": 26, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/battery", + "label": "Battery charging, 100 percent.", + "rect": { + "x": 985, + "y": 49, + "width": 20, + "height": 34 + }, + "visibleToUser": true + }, + { + "index": 36, + "parentIndex": 35, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "rect": { + "x": 985, + "y": 49, + "width": 20, + "height": 34 + }, + "visibleToUser": true + }, + { + "index": 37, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 28, + "y": 957, + "width": 1024, + "height": 492 + }, + "visibleToUser": true + }, + { + "index": 38, + "parentIndex": 37, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 39, + "parentIndex": 38, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 40, + "parentIndex": 39, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/action_bar_root", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 41, + "parentIndex": 40, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "android:id/content", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 42, + "parentIndex": 41, + "type": "androidx.appcompat.widget.LinearLayoutCompat", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/parentPanel", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 43, + "parentIndex": 42, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/topPanel", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 118 + }, + "visibleToUser": true + }, + { + "index": 44, + "parentIndex": 43, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/title_template", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 118 + }, + "visibleToUser": true + }, + { + "index": 45, + "parentIndex": 44, + "type": "android.widget.TextView", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/alertTitle", + "label": "Device name", + "rect": { + "x": 133, + "y": 1046, + "width": 814, + "height": 71 + }, + "visibleToUser": true + }, + { + "index": 46, + "parentIndex": 42, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/customPanel", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 47, + "parentIndex": 46, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/custom", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 48, + "parentIndex": 47, + "type": "android.widget.ScrollView", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 49, + "parentIndex": 48, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 50, + "parentIndex": 49, + "type": "android.widget.EditText", + "bundleId": "com.android.settings", + "identifier": "android:id/edit", + "label": "sdk_gphone64_arm64", + "value": "sdk_gphone64_arm64", + "rect": { + "x": 123, + "y": 1117, + "width": 834, + "height": 126 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 51, + "parentIndex": 42, + "type": "android.widget.ScrollView", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/buttonPanel", + "rect": { + "x": 70, + "y": 1243, + "width": 940, + "height": 164 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 52, + "parentIndex": 51, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 1243, + "width": 940, + "height": 164 + }, + "visibleToUser": true + }, + { + "index": 53, + "parentIndex": 52, + "type": "android.widget.Button", + "bundleId": "com.android.settings", + "identifier": "android:id/button2", + "label": "Cancel", + "rect": { + "x": 636, + "y": 1254, + "width": 174, + "height": 142 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 54, + "parentIndex": 52, + "type": "android.widget.Button", + "bundleId": "com.android.settings", + "identifier": "android:id/button1", + "label": "OK", + "rect": { + "x": 810, + "y": 1254, + "width": 168, + "height": 142 + }, + "hittable": true, + "visibleToUser": true + } +] diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts index 609c60ed7b..2821885b39 100644 --- a/packages/platform-android/src/ui-hierarchy-builder.ts +++ b/packages/platform-android/src/ui-hierarchy-builder.ts @@ -28,6 +28,7 @@ import { import { collectAndroidHiddenNodes } from './ui-hierarchy-visibility.ts'; import { scopePresentedAndroidSnapshot } from './ui-hierarchy-scope.ts'; import { isCollectionContainerType, shouldIncludeAndroidNode } from './ui-hierarchy-inclusion.ts'; +import { isAndroidAlertPanelIdentifier } from './alert-detection.ts'; import { createAndroidSnapshotPresentationNode, isAndroidSnapshotPresentationFailure, @@ -242,6 +243,7 @@ function walkUiHierarchyNode( parentIndex?: number, ancestorHittable: boolean = false, ancestorCollection: boolean = false, + ancestorAlertPanel: boolean = false, ancestorSystemChrome: boolean = false, ancestorWindowRect?: Rect, ancestorClip?: Rect, @@ -264,6 +266,7 @@ function walkUiHierarchyNode( parentIndex, ancestorHittable, ancestorCollection, + ancestorAlertPanel, ancestorSystemChrome, effectiveRect, currentWindowRect, @@ -272,6 +275,8 @@ function walkUiHierarchyNode( ); const nextAncestorHittable = ancestorHittable || isAgentTarget(node); const nextAncestorCollection = ancestorCollection || isCollectionContainerType(node.type); + const nextAncestorAlertPanel = + ancestorAlertPanel || isAndroidAlertPanelIdentifier(node.identifier); const nextAncestorClip = resolveAndroidDescendantClip(state, node, effectiveRect, ancestorClip); walkAndroidChildren( state, @@ -280,6 +285,7 @@ function walkUiHierarchyNode( currentIndex, nextAncestorHittable, nextAncestorCollection, + nextAncestorAlertPanel, ancestorSystemChrome || isAndroidSystemChromeWindowResourceId(node.identifier), currentWindowRect, nextAncestorClip, @@ -336,6 +342,7 @@ function appendAndroidNodeIfPresented( parentIndex: number | undefined, ancestorHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ancestorSystemChrome: boolean, effectiveRect: Rect | undefined, windowRect: Rect | undefined, @@ -359,6 +366,7 @@ function appendAndroidNodeIfPresented( ancestorHittable, hasInteractiveDescendant(state, node), ancestorCollection, + ancestorAlertPanel, ) ) { return parentIndex; @@ -405,6 +413,7 @@ function walkAndroidChildren( parentIndex: number | undefined, ancestorHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ancestorSystemChrome: boolean, ancestorWindowRect: Rect | undefined, ancestorClip: Rect | undefined, @@ -418,6 +427,7 @@ function walkAndroidChildren( parentIndex, ancestorHittable, ancestorCollection, + ancestorAlertPanel, ancestorSystemChrome, ancestorWindowRect, ancestorClip, diff --git a/packages/platform-android/src/ui-hierarchy-inclusion.ts b/packages/platform-android/src/ui-hierarchy-inclusion.ts index bfe27c56f2..4436411047 100644 --- a/packages/platform-android/src/ui-hierarchy-inclusion.ts +++ b/packages/platform-android/src/ui-hierarchy-inclusion.ts @@ -16,6 +16,7 @@ export function shouldIncludeAndroidNode( ancestorHittable: boolean, descendantHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ): boolean { if (node.visibleToUser === false) return false; const info = getAndroidNodeInclusionInfo(node); @@ -26,6 +27,7 @@ export function shouldIncludeAndroidNode( ancestorHittable, descendantHittable, ancestorCollection, + ancestorAlertPanel, ); } if (info.isStructural || info.isVisual) { @@ -53,10 +55,12 @@ function shouldIncludeInteractiveAndroidNode( ancestorHittable: boolean, descendantHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ): boolean { if (hasNonPositiveRect(node)) return false; if (isAgentTarget(node)) return true; if (isScrollableType(info.type) && descendantHittable) return true; + if (ancestorAlertPanel && isTextAndroidNode(info)) return true; return shouldIncludeInteractiveProxyNode( info, ancestorHittable, @@ -80,6 +84,10 @@ function shouldIncludeInteractiveProxyNode( return ancestorHittable || descendantHittable || ancestorCollection; } +function isTextAndroidNode(info: AndroidNodeInclusionInfo): boolean { + return info.hasMeaningfulText && !info.isStructural && !info.isVisual; +} + function hasNonPositiveRect(node: AndroidNode): boolean { return Boolean(node.rect && (node.rect.width <= 0 || node.rect.height <= 0)); } diff --git a/packages/platform-android/src/ui-hierarchy.fixtures.ts b/packages/platform-android/src/ui-hierarchy.fixtures.ts index f483518d74..da02209b60 100644 --- a/packages/platform-android/src/ui-hierarchy.fixtures.ts +++ b/packages/platform-android/src/ui-hierarchy.fixtures.ts @@ -1,6 +1,7 @@ // Real archived device trees, regenerated from a device and never hand-edited (a // hand-written id is how a fixture stops proving anything). Static import, not an // `fs` read: it typechecks each tree against `RawSnapshotNode[]`. +import deviceNameDialogCapture from './fixtures/android-device-name-dialog.raw.json' with { type: 'json' }; import imeCapture from './fixtures/android-ime-capture.raw.json' with { type: 'json' }; import qsShadeCapture from './fixtures/android-qs-shade-capture.raw.json' with { type: 'json' }; import { @@ -141,3 +142,11 @@ export const ANDROID_IME_CAPTURE_RAW_NODES: RawSnapshotNode[] = imeCapture; * hand-written ids. `--raw` keeps the structural wrappers a default capture drops. */ export const ANDROID_QS_SHADE_CAPTURE_RAW_NODES: RawSnapshotNode[] = qsShadeCapture; + +/** + * Real device capture (ReactNative_API_35B, Android 15 emulator, Settings > About emulated device > + * Device name) taken with `snapshot --raw --json` and projected to the fields the walk reads. An + * AppCompat AlertDialog: the title `com.android.settings:id/alertTitle` is passive text inside + * `com.android.settings:id/parentPanel`, beside the edit field and the Cancel and OK buttons. + */ +export const ANDROID_DEVICE_NAME_DIALOG_RAW_NODES: RawSnapshotNode[] = deviceNameDialogCapture; diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index b2801462be..9534ad1710 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -79,6 +79,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue - iOS and Android produce the same kind of snapshot: visible elements first, refs you can act on now, and hints that point to hidden list content. - Use `snapshot -i` by default in agent loops. +- On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm. - On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway. - The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate. - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`.