From d0fa5d17e1ebec03c0fd246c4cf642fba323f2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 16:46:25 +0100 Subject: [PATCH] fix(android): keep alert dialog title and message in snapshot -i The interactive Android projection dropped every passive text node, so an AlertDialog showed only its buttons and nothing said what OK confirms. Keep meaningful text inside an AlertController parentPanel (platform or AppCompat); other passive text stays dropped. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/ui-hierarchy-inclusion.test.ts | 38 +- .../platform-android/src/alert-detection.ts | 5 + .../android-device-name-dialog.raw.json | 778 ++++++++++++++++++ .../src/ui-hierarchy-builder.ts | 10 + .../src/ui-hierarchy-inclusion.ts | 8 + .../src/ui-hierarchy.fixtures.ts | 9 + website/docs/docs/snapshots.md | 1 + 7 files changed, 845 insertions(+), 4 deletions(-) create mode 100644 packages/platform-android/src/fixtures/android-device-name-dialog.raw.json diff --git a/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts b/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts index badab9cc8b..c2cf85c931 100644 --- a/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts +++ b/packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts @@ -1,6 +1,10 @@ import { expect, test } from 'vitest'; import { shouldIncludeAndroidNode } from '../ui-hierarchy-inclusion.ts'; import type { AndroidNode } from '../ui-hierarchy-node.ts'; +import { + ANDROID_DEVICE_NAME_DIALOG_RAW_NODES, + walkInteractiveOnlyAndroidFixture, +} from '../ui-hierarchy.fixtures.ts'; test('interactive inclusion keeps positive agent targets and rejects degenerate geometry', () => { expect( @@ -10,6 +14,7 @@ test('interactive inclusion keeps positive agent targets and rejects degenerate false, false, false, + false, ), ).toBe(true); expect( @@ -19,6 +24,7 @@ test('interactive inclusion keeps positive agent targets and rejects degenerate false, false, false, + false, ), ).toBe(false); }); @@ -26,10 +32,33 @@ test('interactive inclusion keeps positive agent targets and rejects degenerate test('interactive inclusion keeps a semantic Compose proxy only in actionable context', () => { const semanticView = node({ type: 'android.view.View', label: 'Start a call' }); expect( - shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, true, false, false), + shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, true, false, false, false), ).toBe(true); expect( - shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, false, false, false), + shouldIncludeAndroidNode(semanticView, { interactiveOnly: true }, false, false, false, false), + ).toBe(false); +}); + +test('interactive inclusion keeps an AlertDialog title and drops screen text outside the dialog', () => { + const labels = walkInteractiveOnlyAndroidFixture(ANDROID_DEVICE_NAME_DIALOG_RAW_NODES).map( + (presented) => presented.label, + ); + expect(labels).toContain('Device name'); + expect(labels).toEqual(expect.arrayContaining(['sdk_gphone64_arm64', 'Cancel', 'OK'])); + expect(labels).not.toContain('4:37'); +}); + +test('interactive inclusion keeps passive text only inside an alert panel', () => { + const title = node({ type: 'android.widget.TextView', label: 'Device name' }); + expect( + shouldIncludeAndroidNode(title, { interactiveOnly: true }, false, false, false, true), + ).toBe(true); + expect( + shouldIncludeAndroidNode(title, { interactiveOnly: true }, false, false, false, false), + ).toBe(false); + const panelLayout = node({ type: 'android.widget.LinearLayout', label: 'Panel' }); + expect( + shouldIncludeAndroidNode(panelLayout, { interactiveOnly: true }, false, false, false, true), ).toBe(false); }); @@ -41,13 +70,14 @@ test('regular inclusion preserves meaningful structural nodes and actionable des false, false, false, + false, ), ).toBe(true); expect( - shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, true, false), + shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, true, false, false), ).toBe(true); expect( - shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, false, false), + shouldIncludeAndroidNode(node({ type: 'android.view.View' }), {}, false, false, false, false), ).toBe(false); }); diff --git a/packages/platform-android/src/alert-detection.ts b/packages/platform-android/src/alert-detection.ts index faffea3ac9..60af1eebce 100644 --- a/packages/platform-android/src/alert-detection.ts +++ b/packages/platform-android/src/alert-detection.ts @@ -57,6 +57,11 @@ const ANDROID_PERMISSION_ID_PATTERN = /(?:^|:)id\/permission_/i; const ANDROID_BLOCKING_DIALOG_PATTERN = /\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i; +/** The AlertController panel that holds a dialog's title, message, and buttons, under the platform or an AppCompat package. */ +export function isAndroidAlertPanelIdentifier(identifier: string | null): boolean { + return identifier !== null && /(?:^|:)id\/parentPanel$/i.test(identifier); +} + export function classifyAndroidAlertIdentifier( identifier: string | null, ): 'button' | 'content' | undefined { diff --git a/packages/platform-android/src/fixtures/android-device-name-dialog.raw.json b/packages/platform-android/src/fixtures/android-device-name-dialog.raw.json new file mode 100644 index 0000000000..e8611741d7 --- /dev/null +++ b/packages/platform-android/src/fixtures/android-device-name-dialog.raw.json @@ -0,0 +1,778 @@ +[ + { + "index": 0, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 1, + "parentIndex": 0, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/taskbar_container", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 2, + "parentIndex": 1, + "type": "android.view.View", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/taskbar_scrim", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 3, + "parentIndex": 1, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/navbuttons_view", + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 4, + "parentIndex": 3, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/start_contextual_buttons", + "rect": { + "x": 21, + "y": 2274, + "width": 103, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 5, + "parentIndex": 3, + "type": "android.widget.LinearLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/end_nav_buttons", + "rect": { + "x": 124, + "y": 2274, + "width": 830, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 6, + "parentIndex": 5, + "type": "android.widget.ImageView", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/back", + "label": "Back", + "rect": { + "x": 125, + "y": 2279, + "width": 207, + "height": 116 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 7, + "parentIndex": 5, + "type": "android.widget.ImageView", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/home", + "label": "Home", + "rect": { + "x": 434, + "y": 2279, + "width": 210, + "height": 116 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 8, + "parentIndex": 5, + "type": "android.widget.ImageView", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/recent_apps", + "label": "Recents", + "rect": { + "x": 746, + "y": 2279, + "width": 207, + "height": 116 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 9, + "parentIndex": 3, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.apps.nexuslauncher", + "identifier": "com.google.android.apps.nexuslauncher:id/end_contextual_buttons", + "rect": { + "x": 956, + "y": 2274, + "width": 103, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 10, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 11, + "parentIndex": 10, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_launch_animation_container", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 12, + "parentIndex": 10, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_container", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 13, + "parentIndex": 12, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 14, + "parentIndex": 13, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_contents", + "rect": { + "x": 0, + "y": 0, + "width": 1027, + "height": 132 + }, + "visibleToUser": true + }, + { + "index": 15, + "parentIndex": 14, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_start_side_container", + "rect": { + "x": 42, + "y": 1, + "width": 426, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 16, + "parentIndex": 15, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_start_side_content", + "rect": { + "x": 42, + "y": 1, + "width": 220, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 17, + "parentIndex": 16, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_start_side_except_heads_up", + "rect": { + "x": 42, + "y": 1, + "width": 220, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 18, + "parentIndex": 17, + "type": "android.widget.TextView", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/clock", + "label": "4:37", + "rect": { + "x": 42, + "y": 45, + "width": 104, + "height": 43 + }, + "visibleToUser": true + }, + { + "index": 19, + "parentIndex": 17, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/notification_icon_area", + "rect": { + "x": 146, + "y": 1, + "width": 116, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 20, + "parentIndex": 19, + "type": "android.view.ViewGroup", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/notificationIcons", + "rect": { + "x": 146, + "y": 1, + "width": 116, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 21, + "parentIndex": 20, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "label": "Security & privacy notification: Set a screen lock", + "rect": { + "x": 146, + "y": 1, + "width": 58, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 22, + "parentIndex": 20, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "label": "Android System notification: Serial console enabled", + "rect": { + "x": 204, + "y": 1, + "width": 58, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 23, + "parentIndex": 14, + "type": "android.view.View", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/cutout_space_view", + "rect": { + "x": 468, + "y": 1, + "width": 122, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 24, + "parentIndex": 14, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_end_side_container", + "rect": { + "x": 590, + "y": 1, + "width": 426, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 25, + "parentIndex": 24, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/status_bar_end_side_content", + "rect": { + "x": 857, + "y": 1, + "width": 159, + "height": 131 + }, + "visibleToUser": true + }, + { + "index": 26, + "parentIndex": 25, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/system_icons", + "rect": { + "x": 857, + "y": 37, + "width": 159, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 27, + "parentIndex": 26, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/statusIcons", + "rect": { + "x": 865, + "y": 37, + "width": 120, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 28, + "parentIndex": 27, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_combo", + "label": "Phone signal full.", + "rect": { + "x": 865, + "y": 37, + "width": 104, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 29, + "parentIndex": 28, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_group", + "rect": { + "x": 865, + "y": 37, + "width": 104, + "height": 58 + }, + "visibleToUser": true + }, + { + "index": 30, + "parentIndex": 29, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_type_container", + "rect": { + "x": 872, + "y": 42, + "width": 55, + "height": 47 + }, + "visibleToUser": true + }, + { + "index": 31, + "parentIndex": 30, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_type", + "label": "3G", + "rect": { + "x": 872, + "y": 44, + "width": 55, + "height": 42 + }, + "visibleToUser": true + }, + { + "index": 32, + "parentIndex": 29, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "rect": { + "x": 930, + "y": 46, + "width": 39, + "height": 39 + }, + "visibleToUser": true + }, + { + "index": 33, + "parentIndex": 32, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/mobile_signal", + "rect": { + "x": 930, + "y": 46, + "width": 39, + "height": 39 + }, + "visibleToUser": true + }, + { + "index": 34, + "parentIndex": 27, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.systemui", + "rect": { + "x": 865, + "y": 66, + "width": 0, + "height": 0 + }, + "visibleToUser": false + }, + { + "index": 35, + "parentIndex": 26, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.systemui", + "identifier": "com.android.systemui:id/battery", + "label": "Battery charging, 100 percent.", + "rect": { + "x": 985, + "y": 49, + "width": 20, + "height": 34 + }, + "visibleToUser": true + }, + { + "index": 36, + "parentIndex": 35, + "type": "android.widget.ImageView", + "bundleId": "com.android.systemui", + "rect": { + "x": 985, + "y": 49, + "width": 20, + "height": 34 + }, + "visibleToUser": true + }, + { + "index": 37, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 28, + "y": 957, + "width": 1024, + "height": 492 + }, + "visibleToUser": true + }, + { + "index": 38, + "parentIndex": 37, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 39, + "parentIndex": 38, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 40, + "parentIndex": 39, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/action_bar_root", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 41, + "parentIndex": 40, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "android:id/content", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 42, + "parentIndex": 41, + "type": "androidx.appcompat.widget.LinearLayoutCompat", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/parentPanel", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 408 + }, + "visibleToUser": true + }, + { + "index": 43, + "parentIndex": 42, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/topPanel", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 118 + }, + "visibleToUser": true + }, + { + "index": 44, + "parentIndex": 43, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/title_template", + "rect": { + "x": 70, + "y": 999, + "width": 940, + "height": 118 + }, + "visibleToUser": true + }, + { + "index": 45, + "parentIndex": 44, + "type": "android.widget.TextView", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/alertTitle", + "label": "Device name", + "rect": { + "x": 133, + "y": 1046, + "width": 814, + "height": 71 + }, + "visibleToUser": true + }, + { + "index": 46, + "parentIndex": 42, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/customPanel", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 47, + "parentIndex": 46, + "type": "android.widget.FrameLayout", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/custom", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 48, + "parentIndex": 47, + "type": "android.widget.ScrollView", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 49, + "parentIndex": 48, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 1117, + "width": 940, + "height": 126 + }, + "visibleToUser": true + }, + { + "index": 50, + "parentIndex": 49, + "type": "android.widget.EditText", + "bundleId": "com.android.settings", + "identifier": "android:id/edit", + "label": "sdk_gphone64_arm64", + "value": "sdk_gphone64_arm64", + "rect": { + "x": 123, + "y": 1117, + "width": 834, + "height": 126 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 51, + "parentIndex": 42, + "type": "android.widget.ScrollView", + "bundleId": "com.android.settings", + "identifier": "com.android.settings:id/buttonPanel", + "rect": { + "x": 70, + "y": 1243, + "width": 940, + "height": 164 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 52, + "parentIndex": 51, + "type": "android.widget.LinearLayout", + "bundleId": "com.android.settings", + "rect": { + "x": 70, + "y": 1243, + "width": 940, + "height": 164 + }, + "visibleToUser": true + }, + { + "index": 53, + "parentIndex": 52, + "type": "android.widget.Button", + "bundleId": "com.android.settings", + "identifier": "android:id/button2", + "label": "Cancel", + "rect": { + "x": 636, + "y": 1254, + "width": 174, + "height": 142 + }, + "hittable": true, + "visibleToUser": true + }, + { + "index": 54, + "parentIndex": 52, + "type": "android.widget.Button", + "bundleId": "com.android.settings", + "identifier": "android:id/button1", + "label": "OK", + "rect": { + "x": 810, + "y": 1254, + "width": 168, + "height": 142 + }, + "hittable": true, + "visibleToUser": true + } +] diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts index 609c60ed7b..2821885b39 100644 --- a/packages/platform-android/src/ui-hierarchy-builder.ts +++ b/packages/platform-android/src/ui-hierarchy-builder.ts @@ -28,6 +28,7 @@ import { import { collectAndroidHiddenNodes } from './ui-hierarchy-visibility.ts'; import { scopePresentedAndroidSnapshot } from './ui-hierarchy-scope.ts'; import { isCollectionContainerType, shouldIncludeAndroidNode } from './ui-hierarchy-inclusion.ts'; +import { isAndroidAlertPanelIdentifier } from './alert-detection.ts'; import { createAndroidSnapshotPresentationNode, isAndroidSnapshotPresentationFailure, @@ -242,6 +243,7 @@ function walkUiHierarchyNode( parentIndex?: number, ancestorHittable: boolean = false, ancestorCollection: boolean = false, + ancestorAlertPanel: boolean = false, ancestorSystemChrome: boolean = false, ancestorWindowRect?: Rect, ancestorClip?: Rect, @@ -264,6 +266,7 @@ function walkUiHierarchyNode( parentIndex, ancestorHittable, ancestorCollection, + ancestorAlertPanel, ancestorSystemChrome, effectiveRect, currentWindowRect, @@ -272,6 +275,8 @@ function walkUiHierarchyNode( ); const nextAncestorHittable = ancestorHittable || isAgentTarget(node); const nextAncestorCollection = ancestorCollection || isCollectionContainerType(node.type); + const nextAncestorAlertPanel = + ancestorAlertPanel || isAndroidAlertPanelIdentifier(node.identifier); const nextAncestorClip = resolveAndroidDescendantClip(state, node, effectiveRect, ancestorClip); walkAndroidChildren( state, @@ -280,6 +285,7 @@ function walkUiHierarchyNode( currentIndex, nextAncestorHittable, nextAncestorCollection, + nextAncestorAlertPanel, ancestorSystemChrome || isAndroidSystemChromeWindowResourceId(node.identifier), currentWindowRect, nextAncestorClip, @@ -336,6 +342,7 @@ function appendAndroidNodeIfPresented( parentIndex: number | undefined, ancestorHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ancestorSystemChrome: boolean, effectiveRect: Rect | undefined, windowRect: Rect | undefined, @@ -359,6 +366,7 @@ function appendAndroidNodeIfPresented( ancestorHittable, hasInteractiveDescendant(state, node), ancestorCollection, + ancestorAlertPanel, ) ) { return parentIndex; @@ -405,6 +413,7 @@ function walkAndroidChildren( parentIndex: number | undefined, ancestorHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ancestorSystemChrome: boolean, ancestorWindowRect: Rect | undefined, ancestorClip: Rect | undefined, @@ -418,6 +427,7 @@ function walkAndroidChildren( parentIndex, ancestorHittable, ancestorCollection, + ancestorAlertPanel, ancestorSystemChrome, ancestorWindowRect, ancestorClip, diff --git a/packages/platform-android/src/ui-hierarchy-inclusion.ts b/packages/platform-android/src/ui-hierarchy-inclusion.ts index bfe27c56f2..4436411047 100644 --- a/packages/platform-android/src/ui-hierarchy-inclusion.ts +++ b/packages/platform-android/src/ui-hierarchy-inclusion.ts @@ -16,6 +16,7 @@ export function shouldIncludeAndroidNode( ancestorHittable: boolean, descendantHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ): boolean { if (node.visibleToUser === false) return false; const info = getAndroidNodeInclusionInfo(node); @@ -26,6 +27,7 @@ export function shouldIncludeAndroidNode( ancestorHittable, descendantHittable, ancestorCollection, + ancestorAlertPanel, ); } if (info.isStructural || info.isVisual) { @@ -53,10 +55,12 @@ function shouldIncludeInteractiveAndroidNode( ancestorHittable: boolean, descendantHittable: boolean, ancestorCollection: boolean, + ancestorAlertPanel: boolean, ): boolean { if (hasNonPositiveRect(node)) return false; if (isAgentTarget(node)) return true; if (isScrollableType(info.type) && descendantHittable) return true; + if (ancestorAlertPanel && isTextAndroidNode(info)) return true; return shouldIncludeInteractiveProxyNode( info, ancestorHittable, @@ -80,6 +84,10 @@ function shouldIncludeInteractiveProxyNode( return ancestorHittable || descendantHittable || ancestorCollection; } +function isTextAndroidNode(info: AndroidNodeInclusionInfo): boolean { + return info.hasMeaningfulText && !info.isStructural && !info.isVisual; +} + function hasNonPositiveRect(node: AndroidNode): boolean { return Boolean(node.rect && (node.rect.width <= 0 || node.rect.height <= 0)); } diff --git a/packages/platform-android/src/ui-hierarchy.fixtures.ts b/packages/platform-android/src/ui-hierarchy.fixtures.ts index f483518d74..da02209b60 100644 --- a/packages/platform-android/src/ui-hierarchy.fixtures.ts +++ b/packages/platform-android/src/ui-hierarchy.fixtures.ts @@ -1,6 +1,7 @@ // Real archived device trees, regenerated from a device and never hand-edited (a // hand-written id is how a fixture stops proving anything). Static import, not an // `fs` read: it typechecks each tree against `RawSnapshotNode[]`. +import deviceNameDialogCapture from './fixtures/android-device-name-dialog.raw.json' with { type: 'json' }; import imeCapture from './fixtures/android-ime-capture.raw.json' with { type: 'json' }; import qsShadeCapture from './fixtures/android-qs-shade-capture.raw.json' with { type: 'json' }; import { @@ -141,3 +142,11 @@ export const ANDROID_IME_CAPTURE_RAW_NODES: RawSnapshotNode[] = imeCapture; * hand-written ids. `--raw` keeps the structural wrappers a default capture drops. */ export const ANDROID_QS_SHADE_CAPTURE_RAW_NODES: RawSnapshotNode[] = qsShadeCapture; + +/** + * Real device capture (ReactNative_API_35B, Android 15 emulator, Settings > About emulated device > + * Device name) taken with `snapshot --raw --json` and projected to the fields the walk reads. An + * AppCompat AlertDialog: the title `com.android.settings:id/alertTitle` is passive text inside + * `com.android.settings:id/parentPanel`, beside the edit field and the Cancel and OK buttons. + */ +export const ANDROID_DEVICE_NAME_DIALOG_RAW_NODES: RawSnapshotNode[] = deviceNameDialogCapture; diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index b2801462be..9534ad1710 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -79,6 +79,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue - iOS and Android produce the same kind of snapshot: visible elements first, refs you can act on now, and hints that point to hidden list content. - Use `snapshot -i` by default in agent loops. +- On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm. - On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway. - The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate. - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`.