Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/android-framework-activity-system-surface.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'agent-device': patch
---

Fix Android snapshots failing with "insufficient foreground app content" while a framework activity such as the share sheet (`ChooserActivity` or `ResolverActivity`) covers the app. Those activities run in an application window owned by the `android` package on Android 13 and older, so the snapshot now returns them as a system surface, the same way it returns the notification shade, and the agent can act on or dismiss them.
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,52 @@ test('returns a meaningful active system surface (notification shade) faithfully
expect(result.outcome).toBe('system-surface-only');
});

test.each(['android', 'com.android.systemui'])(
'returns a framework-owned application window over the foreground app as a system surface (%s)',
(packageName) => {
// ChooserActivity and ResolverActivity run in an application window owned by `android`, while
// the foreground app below still owns the task, so no node matches the foreground package.
const result = classifyAndroidHelperContent(
helperXml([
node({
windowType: 1,
windowActive: true,
packageName,
className: 'android.widget.FrameLayout',
}),
node({
text: 'repro text',
resourceId: `${packageName}:id/content_preview_text`,
packageName,
className: 'android.widget.TextView',
}),
node({
text: 'Copy',
resourceId: `${packageName}:id/copy`,
packageName,
className: 'android.widget.Button',
}),
node({
text: 'Messages',
resourceId: `${packageName}:id/text1`,
packageName,
className: 'android.widget.TextView',
}),
]),
{
backend: 'android-helper',
nodeCount: 4,
rootPresent: true,
windowCount: 1,
captureMode: 'interactive-windows',
},
{ foregroundAppPackage: 'com.example.app' },
);

expect(result.outcome).toBe('system-surface-only');
},
);

test('rejects an active navigation-bar window with three-button chrome', () => {
// Back + Home + Recents are 3 meaningful nodes, but they are status/nav chrome: an active
// nav-bar window is missing-app-content residue, never a usable shade/quick-settings surface.
Expand Down
15 changes: 13 additions & 2 deletions packages/platform-android/src/snapshot-content-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ function unusable(
function isScreenOwnedByMeaningfulSystemSurface(summary: AndroidHelperXmlSummary): boolean {
return (
summary.windowRootCount > 0 &&
summary.applicationWindowRootCount === 0 &&
summary.applicationWindowRootCount === summary.systemApplicationWindowRootCount &&
summary.activeSystemSurfaceMeaningfulNodeCount >= MIN_SYSTEM_SURFACE_MEANINGFUL_NODES
);
}
Expand Down Expand Up @@ -189,6 +189,8 @@ type AndroidHelperXmlSummary = {
systemUiNodeCount: number;
windowRootCount: number;
applicationWindowRootCount: number;
/** Application windows owned by the framework, such as ResolverActivity and ChooserActivity. */
systemApplicationWindowRootCount: number;
activeSystemSurfaceMeaningfulNodeCount: number;
meaningfulNodeCount: number;
applicationMeaningfulNodeCount: number;
Expand All @@ -204,6 +206,7 @@ type AndroidHelperXmlSummary = {
type AndroidHelperXmlSummaryState = Omit<AndroidHelperXmlSummary, 'windowTypes'> & {
currentWindowType?: number;
currentWindowActiveOrFocused?: boolean;
currentWindowSystemOwned?: boolean;
windowTypes: Set<number>;
};

Expand All @@ -228,6 +231,7 @@ function createAndroidHelperXmlSummaryState(
systemUiNodeCount: 0,
windowRootCount: 0,
applicationWindowRootCount: 0,
systemApplicationWindowRootCount: 0,
activeSystemSurfaceMeaningfulNodeCount: 0,
meaningfulNodeCount: 0,
applicationMeaningfulNodeCount: 0,
Expand Down Expand Up @@ -270,10 +274,12 @@ function recordAndroidHelperWindowNode(

summary.currentWindowType = node.windowType;
summary.currentWindowActiveOrFocused = node.windowActive === true || node.windowFocused === true;
summary.currentWindowSystemOwned = isExplicitAndroidSystemPackage(node.packageName);
summary.windowRootCount += 1;
summary.windowTypes.add(node.windowType);
if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) {
summary.applicationWindowRootCount += 1;
if (summary.currentWindowSystemOwned) summary.systemApplicationWindowRootCount += 1;
}
}

Expand Down Expand Up @@ -306,10 +312,14 @@ function recordMeaningfulWindowOwnership(
}
}

// Framework activities such as the share sheet's ChooserActivity and ResolverActivity run in an
// application window owned by the `android` package (Android 13 and older, and some AOSP builds),
// so they count as a system surface rather than as missing app content.
function isInActiveSystemSurfaceWindow(summary: AndroidHelperXmlSummaryState): boolean {
return (
summary.currentWindowType !== undefined &&
summary.currentWindowType !== ANDROID_WINDOW_TYPE_APPLICATION &&
(summary.currentWindowType !== ANDROID_WINDOW_TYPE_APPLICATION ||
summary.currentWindowSystemOwned === true) &&
summary.currentWindowActiveOrFocused === true
);
}
Expand Down Expand Up @@ -345,6 +355,7 @@ function finalizeAndroidHelperXmlSummary(
systemUiNodeCount: summary.systemUiNodeCount,
windowRootCount: summary.windowRootCount,
applicationWindowRootCount: summary.applicationWindowRootCount,
systemApplicationWindowRootCount: summary.systemApplicationWindowRootCount,
activeSystemSurfaceMeaningfulNodeCount: summary.activeSystemSurfaceMeaningfulNodeCount,
meaningfulNodeCount: summary.meaningfulNodeCount,
applicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount,
Expand Down
Loading