From 9925d1443c4c2dbe2e937b9ab941f4f3c8088738 Mon Sep 17 00:00:00 2001 From: Mike Olson Date: Mon, 9 Nov 2026 22:54:10 -0500 Subject: [PATCH 1/2] fix(android): return framework share sheets as a system surface --- ...droid-framework-activity-system-surface.md | 5 +++ .../snapshot-content-recovery.test.ts | 43 +++++++++++++++++++ .../src/snapshot-content-recovery.ts | 15 ++++++- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .changeset/android-framework-activity-system-surface.md diff --git a/.changeset/android-framework-activity-system-surface.md b/.changeset/android-framework-activity-system-surface.md new file mode 100644 index 0000000000..efb6b42193 --- /dev/null +++ b/.changeset/android-framework-activity-system-surface.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +Fix Android snapshots failing with "insufficient foreground app content" while a framework activity such as the share sheet (`ChooserActivity` or `ResolverActivity`) covers the app. Those activities run in an application window owned by the `android` package on Android 13 and older, so the snapshot now returns them as a system surface, the same way it returns the notification shade, and the agent can act on or dismiss them. diff --git a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts index e87b5566ef..e9ebca83f1 100644 --- a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts @@ -310,6 +310,49 @@ test('returns a meaningful active system surface (notification shade) faithfully expect(result.outcome).toBe('system-surface-only'); }); +test('returns a framework share sheet over the foreground app as a system surface', () => { + // ChooserActivity and ResolverActivity run in an application window owned by `android`, while + // the foreground app below still owns the task, so no node matches the foreground package. + const result = classifyAndroidHelperContent( + helperXml([ + node({ + windowType: 1, + windowActive: true, + packageName: 'android', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'repro text', + resourceId: 'android:id/content_preview_text', + packageName: 'android', + className: 'android.widget.TextView', + }), + node({ + text: 'Copy', + resourceId: 'android:id/copy', + packageName: 'android', + className: 'android.widget.Button', + }), + node({ + text: 'Messages', + resourceId: 'android:id/text1', + packageName: 'android', + className: 'android.widget.TextView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'com.example.app' }, + ); + + expect(result.outcome).toBe('system-surface-only'); +}); + test('rejects an active navigation-bar window with three-button chrome', () => { // Back + Home + Recents are 3 meaningful nodes, but they are status/nav chrome: an active // nav-bar window is missing-app-content residue, never a usable shade/quick-settings surface. diff --git a/packages/platform-android/src/snapshot-content-recovery.ts b/packages/platform-android/src/snapshot-content-recovery.ts index 3c0c0b5e45..83269ce198 100644 --- a/packages/platform-android/src/snapshot-content-recovery.ts +++ b/packages/platform-android/src/snapshot-content-recovery.ts @@ -107,7 +107,7 @@ function unusable( function isScreenOwnedByMeaningfulSystemSurface(summary: AndroidHelperXmlSummary): boolean { return ( summary.windowRootCount > 0 && - summary.applicationWindowRootCount === 0 && + summary.applicationWindowRootCount === summary.systemApplicationWindowRootCount && summary.activeSystemSurfaceMeaningfulNodeCount >= MIN_SYSTEM_SURFACE_MEANINGFUL_NODES ); } @@ -189,6 +189,8 @@ type AndroidHelperXmlSummary = { systemUiNodeCount: number; windowRootCount: number; applicationWindowRootCount: number; + /** Application windows owned by the framework, such as ResolverActivity and ChooserActivity. */ + systemApplicationWindowRootCount: number; activeSystemSurfaceMeaningfulNodeCount: number; meaningfulNodeCount: number; applicationMeaningfulNodeCount: number; @@ -204,6 +206,7 @@ type AndroidHelperXmlSummary = { type AndroidHelperXmlSummaryState = Omit & { currentWindowType?: number; currentWindowActiveOrFocused?: boolean; + currentWindowSystemOwned?: boolean; windowTypes: Set; }; @@ -228,6 +231,7 @@ function createAndroidHelperXmlSummaryState( systemUiNodeCount: 0, windowRootCount: 0, applicationWindowRootCount: 0, + systemApplicationWindowRootCount: 0, activeSystemSurfaceMeaningfulNodeCount: 0, meaningfulNodeCount: 0, applicationMeaningfulNodeCount: 0, @@ -270,10 +274,12 @@ function recordAndroidHelperWindowNode( summary.currentWindowType = node.windowType; summary.currentWindowActiveOrFocused = node.windowActive === true || node.windowFocused === true; + summary.currentWindowSystemOwned = isExplicitAndroidSystemPackage(node.packageName); summary.windowRootCount += 1; summary.windowTypes.add(node.windowType); if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) { summary.applicationWindowRootCount += 1; + if (summary.currentWindowSystemOwned) summary.systemApplicationWindowRootCount += 1; } } @@ -306,10 +312,14 @@ function recordMeaningfulWindowOwnership( } } +// Framework activities such as the share sheet's ChooserActivity and ResolverActivity run in an +// application window owned by the `android` package (Android 13 and older, and some AOSP builds), +// so they count as a system surface rather than as missing app content. function isInActiveSystemSurfaceWindow(summary: AndroidHelperXmlSummaryState): boolean { return ( summary.currentWindowType !== undefined && - summary.currentWindowType !== ANDROID_WINDOW_TYPE_APPLICATION && + (summary.currentWindowType !== ANDROID_WINDOW_TYPE_APPLICATION || + summary.currentWindowSystemOwned === true) && summary.currentWindowActiveOrFocused === true ); } @@ -345,6 +355,7 @@ function finalizeAndroidHelperXmlSummary( systemUiNodeCount: summary.systemUiNodeCount, windowRootCount: summary.windowRootCount, applicationWindowRootCount: summary.applicationWindowRootCount, + systemApplicationWindowRootCount: summary.systemApplicationWindowRootCount, activeSystemSurfaceMeaningfulNodeCount: summary.activeSystemSurfaceMeaningfulNodeCount, meaningfulNodeCount: summary.meaningfulNodeCount, applicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount, From b1146eaca79365c268c9206e09584e23be0fdb0c Mon Sep 17 00:00:00 2001 From: Mike Olson Date: Mon, 9 Nov 2026 23:31:21 -0500 Subject: [PATCH 2/2] test(android): cover systemui-owned application windows as a system surface --- .../snapshot-content-recovery.test.ts | 85 ++++++++++--------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts index e9ebca83f1..c434c98e52 100644 --- a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts @@ -310,48 +310,51 @@ test('returns a meaningful active system surface (notification shade) faithfully expect(result.outcome).toBe('system-surface-only'); }); -test('returns a framework share sheet over the foreground app as a system surface', () => { - // ChooserActivity and ResolverActivity run in an application window owned by `android`, while - // the foreground app below still owns the task, so no node matches the foreground package. - const result = classifyAndroidHelperContent( - helperXml([ - node({ - windowType: 1, - windowActive: true, - packageName: 'android', - className: 'android.widget.FrameLayout', - }), - node({ - text: 'repro text', - resourceId: 'android:id/content_preview_text', - packageName: 'android', - className: 'android.widget.TextView', - }), - node({ - text: 'Copy', - resourceId: 'android:id/copy', - packageName: 'android', - className: 'android.widget.Button', - }), - node({ - text: 'Messages', - resourceId: 'android:id/text1', - packageName: 'android', - className: 'android.widget.TextView', - }), - ]), - { - backend: 'android-helper', - nodeCount: 4, - rootPresent: true, - windowCount: 1, - captureMode: 'interactive-windows', - }, - { foregroundAppPackage: 'com.example.app' }, - ); +test.each(['android', 'com.android.systemui'])( + 'returns a framework-owned application window over the foreground app as a system surface (%s)', + (packageName) => { + // ChooserActivity and ResolverActivity run in an application window owned by `android`, while + // the foreground app below still owns the task, so no node matches the foreground package. + const result = classifyAndroidHelperContent( + helperXml([ + node({ + windowType: 1, + windowActive: true, + packageName, + className: 'android.widget.FrameLayout', + }), + node({ + text: 'repro text', + resourceId: `${packageName}:id/content_preview_text`, + packageName, + className: 'android.widget.TextView', + }), + node({ + text: 'Copy', + resourceId: `${packageName}:id/copy`, + packageName, + className: 'android.widget.Button', + }), + node({ + text: 'Messages', + resourceId: `${packageName}:id/text1`, + packageName, + className: 'android.widget.TextView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'com.example.app' }, + ); - expect(result.outcome).toBe('system-surface-only'); -}); + expect(result.outcome).toBe('system-surface-only'); + }, +); test('rejects an active navigation-bar window with three-button chrome', () => { // Back + Home + Recents are 3 meaningful nodes, but they are status/nav chrome: an active