diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts new file mode 100644 index 0000000000..7cba3cd10d --- /dev/null +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; +import { createAgentDevice } from '../../../runtime.ts'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; + +async function snapshotWithForeground( + foregroundWindowPackage: string | undefined, + sessionApp: { appName: string; appBundleId: string } | undefined = { + appName: 'Settings', + appBundleId: 'com.android.settings', + }, +) { + let stored: Parameters[0] | undefined; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async () => ({ + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'TextView', label: 'Gmail' }], { + backend: 'android', + }), + androidSnapshot: { + backend: 'android-helper', + ...(foregroundWindowPackage ? { foregroundWindowPackage } : {}), + }, + ...sessionApp, + }), + }, + artifacts: createLocalArtifactAdapter(), + sessions: { + get: () => undefined, + set: (record) => { + stored = record; + }, + }, + policy: localCommandPolicy(), + }); + const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); + return { result, stored }; +} + +test('an Android snapshot names the captured foreground app when it is not the session app', async () => { + const { result, stored } = await snapshotWithForeground('com.google.android.apps.nexuslauncher'); + + // A foreign app has no label of its own here, so the header shows its package. + assert.equal(result.appName, 'com.google.android.apps.nexuslauncher'); + assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); + assert.equal(stored?.appBundleId, 'com.android.settings'); +}); + +test('an Android snapshot keeps the session app label when the session app is in front', async () => { + const { result } = await snapshotWithForeground('com.android.settings'); + + assert.equal(result.appName, 'Settings'); + assert.equal(result.appBundleId, 'com.android.settings'); +}); + +test('a snapshot without a foreground window fact names the session app', async () => { + const { result } = await snapshotWithForeground(undefined); + + assert.equal(result.appName, 'Settings'); + assert.equal(result.appBundleId, 'com.android.settings'); +}); + +test('an Android snapshot in a session with no app names the captured foreground app', async () => { + const { result } = await snapshotWithForeground( + 'com.google.android.apps.nexuslauncher', + undefined, + ); + + assert.equal(result.appName, 'com.google.android.apps.nexuslauncher'); + assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); +}); + +test('a repeated unchanged snapshot of a foreign foreground app is acknowledged under its header', async () => { + const snapshot = makeSnapshotState([{ index: 0, depth: 0, type: 'TextView', label: 'Gmail' }], { + backend: 'android', + }); + let stored: Parameters[0] | undefined = { + name: 'default', + appBundleId: 'com.android.settings', + }; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async () => ({ + snapshot, + androidSnapshot: { + backend: 'android-helper', + foregroundWindowPackage: 'com.google.android.apps.nexuslauncher', + }, + appBundleId: 'com.android.settings', + }), + }, + artifacts: createLocalArtifactAdapter(), + sessions: { + get: () => stored, + set: (record) => { + stored = record; + }, + }, + policy: localCommandPolicy(), + }); + + await device.capture.snapshot({ session: 'default' }); + const result = await device.capture.snapshot({ session: 'default' }); + + // The acknowledgement compares against the last stored tree, the launcher's, which the header names. + assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); + assert.ok(result.unchanged); +}); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 12758af04b..d99479a72c 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -234,15 +234,22 @@ function nextSnapshotSession( }; } +/** + * The app the response header names: the foreground application window the Android snapshot + * helper captured, such as the launcher after a failed open, else the session app. The session + * keeps its app. + */ function snapshotAppFields(capture: SnapshotCapture): { appName?: string; appBundleId?: string; } { - const appName = capture.result.appName ?? capture.session?.appName; - const appBundleId = capture.result.appBundleId ?? capture.session?.appBundleId; + const sessionApp = capture.result.appBundleId ?? capture.session?.appBundleId; + const shownApp = capture.annotations.androidSnapshot?.foregroundWindowPackage ?? sessionApp; + const appName = + shownApp === sessionApp ? (capture.result.appName ?? capture.session?.appName) : shownApp; return { - ...(appName || appBundleId ? { appName: appName ?? appBundleId } : {}), - ...(appBundleId ? { appBundleId } : {}), + ...(appName || shownApp ? { appName: appName ?? shownApp } : {}), + ...(shownApp ? { appBundleId: shownApp } : {}), }; } diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index 6df1bb1549..dd843f06a3 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -79,6 +79,11 @@ export type AndroidSnapshotBackendMetadata = { cacheInvalidated?: boolean; captureMode?: string; systemSurfaceOnly?: boolean; + /** + * Package of the foreground application window the snapshot helper captured: active or focused, + * at the highest layer, and not owned by the system. Absent when the helper saw none. + */ + foregroundWindowPackage?: string; windowCount?: number; /** * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because diff --git a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts index c434c98e52..9a52b3d5f6 100644 --- a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts @@ -270,6 +270,88 @@ test('rejects system chrome inherited under an empty application window', () => expect(decision?.diagnostics.helperNonSystemMeaningfulNodeCount).toBe(0); }); +function foregroundWindowPackageOf(windows: string[]): string | undefined { + const result = classifyAndroidHelperContent(helperXml(windows), { + backend: 'android-helper', + nodeCount: windows.length * 2, + rootPresent: true, + windowCount: windows.length, + captureMode: 'interactive-windows', + }); + return result.outcome === 'unusable' ? undefined : result.foregroundWindowPackage; +} + +function window(params: { + windowType: number; + packageName: string; + text: string; + windowLayer: number; + windowActive?: boolean; +}): string { + const { text, ...windowAttributes } = params; + return node({ + ...windowAttributes, + className: 'android.widget.FrameLayout', + children: [ + node({ text, packageName: params.packageName, className: 'android.widget.TextView' }), + ], + }); +} + +test('names the active application window, not a stale one listed first or a system window above it', () => { + expect( + foregroundWindowPackageOf([ + window({ + windowType: 3, + packageName: 'com.android.systemui', + text: '12:00', + windowLayer: 9, + windowActive: true, + }), + window({ + windowType: 2, + packageName: 'com.google.android.inputmethod.latin', + text: 'Space', + windowLayer: 8, + }), + // A stale session-app window Android kept after the transition: inactive, lower layer. + window({ + windowType: 1, + packageName: 'com.android.settings', + text: 'Network and internet', + windowLayer: 1, + }), + window({ + windowType: 1, + packageName: 'com.google.android.apps.nexuslauncher', + text: 'Gmail', + windowLayer: 2, + windowActive: true, + }), + ]), + ).toBe('com.google.android.apps.nexuslauncher'); +}); + +test('names no foreground app when only a framework window is active', () => { + expect( + foregroundWindowPackageOf([ + window({ + windowType: 1, + packageName: 'android', + text: 'Allow?', + windowLayer: 3, + windowActive: true, + }), + window({ + windowType: 1, + packageName: 'com.android.settings', + text: 'Network and internet', + windowLayer: 1, + }), + ]), + ).toBeUndefined(); +}); + test('returns a meaningful active system surface (notification shade) faithfully', () => { const result = classifyAndroidHelperContent( helperXml([ @@ -526,12 +608,14 @@ function node(options: { packageName: string; className: string; windowType?: number; + windowLayer?: number; windowActive?: boolean; /** Nested children, so chrome leaves can sit inside their status/nav-bar container as on a device. */ children?: string[]; }): string { const windowAttrs = (options.windowType === undefined ? '' : ` window-type="${options.windowType}"`) + + (options.windowLayer === undefined ? '' : ` window-layer="${options.windowLayer}"`) + (options.windowActive === undefined ? '' : ` window-active="${options.windowActive}"`); const open = ` node.label === 'Fabric dashboard'), - true, - ); + const { backend, foregroundWindowPackage } = result.androidSnapshot; + assert.deepEqual([backend, foregroundWindowPackage], ['android-helper', 'io.example.fabric']); + assert.ok(result.nodes.some((node) => node.label === 'Fabric dashboard')); }); test('snapshotAndroid emits helper failure diagnostics', async () => { diff --git a/packages/platform-android/src/snapshot-content-recovery.ts b/packages/platform-android/src/snapshot-content-recovery.ts index 83269ce198..4e4e29b8e7 100644 --- a/packages/platform-android/src/snapshot-content-recovery.ts +++ b/packages/platform-android/src/snapshot-content-recovery.ts @@ -5,6 +5,7 @@ import { classifyAndroidAlertIdentifier } from './alert-detection.ts'; import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; import { ANDROID_WINDOW_TYPE_APPLICATION } from './ui-hierarchy-node.ts'; +import { isAndroidForegroundWindow } from './ui-hierarchy-visibility.ts'; const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; @@ -44,10 +45,19 @@ export function classifyAndroidHelperContent( xml: string, metadata: AndroidSnapshotBackendMetadata, options: { foregroundAppPackage?: string } = {}, -): AndroidHelperContentClassification { +): AndroidHelperContentClassification & { foregroundWindowPackage?: string } { if (metadata.backend !== 'android-helper') return { outcome: 'ok' }; - const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage); + const content = classifyAndroidHelperSummary(summary, metadata); + return summary.foregroundWindow + ? { ...content, foregroundWindowPackage: summary.foregroundWindow.packageName } + : content; +} + +function classifyAndroidHelperSummary( + summary: AndroidHelperXmlSummary, + metadata: AndroidSnapshotBackendMetadata, +): AndroidHelperContentClassification { if (isEmptyHelperOutput(summary, metadata)) { return unusable( summary, @@ -201,6 +211,11 @@ type AndroidHelperXmlSummary = { foregroundAppPackage?: string; foregroundAppMeaningfulNodeCount?: number; windowTypes: number[]; + /** + * The non-system application window the presentation keeps as foreground: active or focused, at + * the highest layer. A stale application window under it is not foreground. + */ + foregroundWindow?: { packageName: string; layer: number }; }; type AndroidHelperXmlSummaryState = Omit & { @@ -280,9 +295,20 @@ function recordAndroidHelperWindowNode( if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) { summary.applicationWindowRootCount += 1; if (summary.currentWindowSystemOwned) summary.systemApplicationWindowRootCount += 1; + else recordAndroidForegroundWindow(summary, node); } } +function recordAndroidForegroundWindow( + summary: AndroidHelperXmlSummaryState, + node: AndroidUiNodeMetadata, +): void { + if (node.packageName === null || !isAndroidForegroundWindow(node)) return; + const layer = node.windowLayer ?? -1; + if (summary.foregroundWindow && summary.foregroundWindow.layer >= layer) return; + summary.foregroundWindow = { packageName: node.packageName, layer }; +} + function recordAndroidHelperMeaningfulNode( summary: AndroidHelperXmlSummaryState, node: AndroidUiNodeMetadata, @@ -370,6 +396,7 @@ function finalizeAndroidHelperXmlSummary( } : {}), windowTypes: [...summary.windowTypes].sort((a, b) => a - b).slice(0, MAX_REPORTED_WINDOW_TYPES), + ...(summary.foregroundWindow ? { foregroundWindow: summary.foregroundWindow } : {}), }; } diff --git a/packages/platform-android/src/snapshot-types.ts b/packages/platform-android/src/snapshot-types.ts index 5ca8274e74..ac0163a6cb 100644 --- a/packages/platform-android/src/snapshot-types.ts +++ b/packages/platform-android/src/snapshot-types.ts @@ -41,6 +41,11 @@ export type AndroidSnapshotBackendMetadata = { cacheInvalidated?: boolean; captureMode?: AndroidSnapshotCaptureMode; systemSurfaceOnly?: boolean; + /** + * Package of the foreground application window the snapshot helper captured: active or focused, + * at the highest layer, and not owned by the system. Absent when the helper saw none. + */ + foregroundWindowPackage?: string; windowCount?: number; /** * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 8cfb8b56c4..c8adf975f3 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -639,9 +639,13 @@ async function captureAndroidHelperContentAttempt(params: { capture: { xml: helperCapture.xml, helperMetadata: helperCapture.helperMetadata, - metadata: systemSurfaceOnly - ? { ...helperCapture.metadata, systemSurfaceOnly: true } - : helperCapture.metadata, + metadata: { + ...helperCapture.metadata, + ...(systemSurfaceOnly ? { systemSurfaceOnly: true } : {}), + ...(content.foregroundWindowPackage + ? { foregroundWindowPackage: content.foregroundWindowPackage } + : {}), + }, }, }; } diff --git a/packages/platform-android/src/ui-hierarchy-visibility.ts b/packages/platform-android/src/ui-hierarchy-visibility.ts index a93d7152e4..ae6cb082c8 100644 --- a/packages/platform-android/src/ui-hierarchy-visibility.ts +++ b/packages/platform-android/src/ui-hierarchy-visibility.ts @@ -92,6 +92,9 @@ function isAndroidApplicationWindow(node: AndroidNode): boolean { return node.windowType === ANDROID_WINDOW_TYPE_APPLICATION; } -function isAndroidForegroundWindow(node: AndroidNode): boolean { +/** A window the user is in: active or focused. Shared with the snapshot header's foreground app. */ +export function isAndroidForegroundWindow( + node: Pick, +): boolean { return node.windowActive === true || node.windowFocused === true; } diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 9534ad1710..a2a19fda31 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -80,6 +80,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue now, and hints that point to hidden list content. - Use `snapshot -i` by default in agent loops. - On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm. +- When the Android snapshot helper captures the tree, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the foreground application window: active or focused, at the highest layer, and not a system UI or keyboard window. After a failed `open`, for example, they name the launcher. An app other than the session app shows its package name in both lines. `androidSnapshot.foregroundWindowPackage` in `--json` carries it, and the session keeps its app. The UIAutomator fallback names the session app. - On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway. - The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate. - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`.