From bd2ceee18e1de3a2d475ed7fe0b4172184c80fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 18:55:36 +0100 Subject: [PATCH 1/7] fix(snapshot): name the captured Android app in the snapshot header After a failed open the launcher was in front, but the Page/App header named the session app, so an agent read launcher nodes as Settings. Android nodes carry their window package; when no app node belongs to the session app the header (appName/appBundleId in the response) names the captured package. The session keeps its app. Fixes #4483 Co-Authored-By: Claude Opus 5.5 --- .../commands/capture/runtime/snapshot.test.ts | 56 +++++++++++++++++++ .../src/commands/capture/runtime/snapshot.ts | 41 +++++++++++--- website/docs/docs/snapshots.md | 1 + 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts index a2320dc896..d0d2e09b58 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts @@ -48,6 +48,62 @@ test('runtime snapshot captures nodes and updates the session baseline', async ( assert.equal(stored?.snapshot?.nodes[0]?.label, 'Home'); }); +test('an Android snapshot names the app it captured, not a session app that is not in front', async () => { + const captureWith = async (overlay: { type: string; label: string; bundleId: string }) => { + let stored: Parameters[0] | undefined; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async () => ({ + snapshot: makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'TextView', + label: '12:00', + bundleId: 'com.android.systemui', + systemBar: 'status', + }, + { index: 1, depth: 0, ...overlay }, + ], + { backend: 'android' }, + ), + appName: 'Settings', + appBundleId: 'com.android.settings', + }), + }, + artifacts: createLocalArtifactAdapter(), + sessions: { + get: () => undefined, + set: (record) => { + stored = record; + }, + }, + policy: localCommandPolicy(), + }); + const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); + return { result, stored }; + }; + + const launcher = await captureWith({ + type: 'ScrollView', + label: 'Workspace', + bundleId: 'com.google.android.apps.nexuslauncher', + }); + assert.equal(launcher.result.appName, 'com.google.android.apps.nexuslauncher'); + assert.equal(launcher.result.appBundleId, 'com.google.android.apps.nexuslauncher'); + assert.equal(launcher.stored?.appBundleId, 'com.android.settings'); + + const settings = await captureWith({ + type: 'TextView', + label: 'Network & internet', + bundleId: 'com.android.settings', + }); + assert.equal(settings.result.appName, 'Settings'); + assert.equal(settings.result.appBundleId, 'com.android.settings'); +}); + test('runtime snapshot preserves unknown hierarchy completeness for engine-owned iOS acquisition', async () => { const device = createSnapshotOnlyDevice({ snapshot: { diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 12758af04b..720ab3ba3d 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -19,7 +19,10 @@ import type { SnapshotVisibility, } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; -import type { BackendSnapshotResult } from '@agent-device/contracts/backend'; +import type { + AgentDeviceBackendPlatform, + BackendSnapshotResult, +} from '@agent-device/contracts/backend'; import type { AgentDeviceRuntime, CommandSessionRecord, @@ -122,7 +125,7 @@ export const snapshotCommand: RuntimeCommand< ...(capture.snapshot.presentedSurface ? { presentedSurface: capture.snapshot.presentedSurface } : {}), - ...snapshotAppFields(capture), + ...snapshotAppFields(capture, runtime.backend.platform), }); }; @@ -234,18 +237,42 @@ function nextSnapshotSession( }; } -function snapshotAppFields(capture: SnapshotCapture): { +/** + * The app the response header names. Every Android node carries its window's package, so a capture + * whose app content belongs only to another package, such as the launcher after a failed open, + * names that package instead of the session app. The session keeps its app. + */ +function snapshotAppFields( + capture: SnapshotCapture, + platform: AgentDeviceBackendPlatform, +): { appName?: string; appBundleId?: string; } { - const appName = capture.result.appName ?? capture.session?.appName; - const appBundleId = capture.result.appBundleId ?? capture.session?.appBundleId; + const sessionApp = capture.result.appBundleId ?? capture.session?.appBundleId; + const shownApp = + platform === 'android' && sessionApp + ? capturedAndroidApp(capture.snapshot.nodes, sessionApp) + : sessionApp; + const appName = + shownApp === sessionApp ? (capture.result.appName ?? capture.session?.appName) : shownApp; return { - ...(appName || appBundleId ? { appName: appName ?? appBundleId } : {}), - ...(appBundleId ? { appBundleId } : {}), + ...(appName || shownApp ? { appName: appName ?? shownApp } : {}), + ...(shownApp ? { appBundleId: shownApp } : {}), }; } +/** The session app when any app node belongs to it, else the first other app package. */ +function capturedAndroidApp(nodes: readonly SnapshotNode[], sessionApp: string): string { + let otherApp: string | undefined; + for (const node of nodes) { + if (!node.bundleId || node.systemBar || node.keyboardOwned) continue; + if (node.bundleId === sessionApp) return sessionApp; + otherApp ??= node.bundleId; + } + return otherApp ?? sessionApp; +} + /** * A capture that reported nothing about truncation is only "not truncated" when its producer * actually observes truncation. Producers that do not (Appium page source, the Limrun element diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 9534ad1710..13758c368f 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -80,6 +80,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue now, and hints that point to hidden list content. - Use `snapshot -i` by default in agent loops. - On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm. +- On Android, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the app the snapshot shows. If the session app is not in front, for example after a failed `open`, they name the app that is, such as the launcher. The session keeps its app. - On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway. - The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate. - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`. From 99ab749802c2f81a89792a0d4b75630a0cb9d75c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 18:56:26 +0100 Subject: [PATCH 2/7] test(snapshot): keep the header identity test in its own file The snapshot test file passed its 1000-line lint limit. Co-Authored-By: Claude Opus 5.5 --- .../runtime/snapshot-app-identity.test.ts | 63 +++++++++++++++++++ .../commands/capture/runtime/snapshot.test.ts | 56 ----------------- 2 files changed, 63 insertions(+), 56 deletions(-) create mode 100644 packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts new file mode 100644 index 0000000000..f7139ff6ca --- /dev/null +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; +import { createAgentDevice } from '../../../runtime.ts'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; + +test('an Android snapshot names the app it captured, not a session app that is not in front', async () => { + const captureWith = async (overlay: { type: string; label: string; bundleId: string }) => { + let stored: Parameters[0] | undefined; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async () => ({ + snapshot: makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'TextView', + label: '12:00', + bundleId: 'com.android.systemui', + systemBar: 'status', + }, + { index: 1, depth: 0, ...overlay }, + ], + { backend: 'android' }, + ), + appName: 'Settings', + appBundleId: 'com.android.settings', + }), + }, + artifacts: createLocalArtifactAdapter(), + sessions: { + get: () => undefined, + set: (record) => { + stored = record; + }, + }, + policy: localCommandPolicy(), + }); + const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); + return { result, stored }; + }; + + const launcher = await captureWith({ + type: 'ScrollView', + label: 'Workspace', + bundleId: 'com.google.android.apps.nexuslauncher', + }); + assert.equal(launcher.result.appName, 'com.google.android.apps.nexuslauncher'); + assert.equal(launcher.result.appBundleId, 'com.google.android.apps.nexuslauncher'); + assert.equal(launcher.stored?.appBundleId, 'com.android.settings'); + + const settings = await captureWith({ + type: 'TextView', + label: 'Network & internet', + bundleId: 'com.android.settings', + }); + assert.equal(settings.result.appName, 'Settings'); + assert.equal(settings.result.appBundleId, 'com.android.settings'); +}); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts index d0d2e09b58..a2320dc896 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts @@ -48,62 +48,6 @@ test('runtime snapshot captures nodes and updates the session baseline', async ( assert.equal(stored?.snapshot?.nodes[0]?.label, 'Home'); }); -test('an Android snapshot names the app it captured, not a session app that is not in front', async () => { - const captureWith = async (overlay: { type: string; label: string; bundleId: string }) => { - let stored: Parameters[0] | undefined; - const device = createAgentDevice({ - backend: { - platform: 'android', - captureSnapshot: async () => ({ - snapshot: makeSnapshotState( - [ - { - index: 0, - depth: 0, - type: 'TextView', - label: '12:00', - bundleId: 'com.android.systemui', - systemBar: 'status', - }, - { index: 1, depth: 0, ...overlay }, - ], - { backend: 'android' }, - ), - appName: 'Settings', - appBundleId: 'com.android.settings', - }), - }, - artifacts: createLocalArtifactAdapter(), - sessions: { - get: () => undefined, - set: (record) => { - stored = record; - }, - }, - policy: localCommandPolicy(), - }); - const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); - return { result, stored }; - }; - - const launcher = await captureWith({ - type: 'ScrollView', - label: 'Workspace', - bundleId: 'com.google.android.apps.nexuslauncher', - }); - assert.equal(launcher.result.appName, 'com.google.android.apps.nexuslauncher'); - assert.equal(launcher.result.appBundleId, 'com.google.android.apps.nexuslauncher'); - assert.equal(launcher.stored?.appBundleId, 'com.android.settings'); - - const settings = await captureWith({ - type: 'TextView', - label: 'Network & internet', - bundleId: 'com.android.settings', - }); - assert.equal(settings.result.appName, 'Settings'); - assert.equal(settings.result.appBundleId, 'com.android.settings'); -}); - test('runtime snapshot preserves unknown hierarchy completeness for engine-owned iOS acquisition', async () => { const device = createSnapshotOnlyDevice({ snapshot: { From 80e097eea28be99dc5970090dbc330c189b25882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:17:42 +0100 Subject: [PATCH 3/7] fix(snapshot): name the header app from Android application windows Address review: scanning node packages let system UI, framework dialogs or an IME win the header when the session app was absent. The Android helper classifier already reads window types, so it now lists the captured non-system application windows (androidSnapshot.applicationWindowPackages), and the header names the topmost one only when none belongs to the session app. Co-Authored-By: Claude Opus 5.5 --- .../runtime/snapshot-app-identity.test.ts | 98 +++++++++---------- .../src/commands/capture/runtime/snapshot.ts | 34 ++----- packages/contracts/src/snapshot-types.ts | 5 + .../snapshot-content-recovery.test.ts | 32 ++++++ .../src/snapshot-content-recovery.ts | 40 +++++++- .../platform-android/src/snapshot-types.ts | 5 + packages/platform-android/src/snapshot.ts | 10 +- website/docs/docs/snapshots.md | 2 +- 8 files changed, 142 insertions(+), 84 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts index f7139ff6ca..aa70738ae7 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts @@ -6,58 +6,58 @@ import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -test('an Android snapshot names the app it captured, not a session app that is not in front', async () => { - const captureWith = async (overlay: { type: string; label: string; bundleId: string }) => { - let stored: Parameters[0] | undefined; - const device = createAgentDevice({ - backend: { - platform: 'android', - captureSnapshot: async () => ({ - snapshot: makeSnapshotState( - [ - { - index: 0, - depth: 0, - type: 'TextView', - label: '12:00', - bundleId: 'com.android.systemui', - systemBar: 'status', - }, - { index: 1, depth: 0, ...overlay }, - ], - { backend: 'android' }, - ), - appName: 'Settings', - appBundleId: 'com.android.settings', +async function snapshotWithAppWindows(applicationWindowPackages: string[] | undefined) { + let stored: Parameters[0] | undefined; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async () => ({ + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'TextView', label: 'Gmail' }], { + backend: 'android', }), - }, - artifacts: createLocalArtifactAdapter(), - sessions: { - get: () => undefined, - set: (record) => { - stored = record; + androidSnapshot: { + backend: 'android-helper', + ...(applicationWindowPackages ? { applicationWindowPackages } : {}), }, + appName: 'Settings', + appBundleId: 'com.android.settings', + }), + }, + artifacts: createLocalArtifactAdapter(), + sessions: { + get: () => undefined, + set: (record) => { + stored = record; }, - policy: localCommandPolicy(), - }); - const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); - return { result, stored }; - }; - - const launcher = await captureWith({ - type: 'ScrollView', - label: 'Workspace', - bundleId: 'com.google.android.apps.nexuslauncher', + }, + policy: localCommandPolicy(), }); - assert.equal(launcher.result.appName, 'com.google.android.apps.nexuslauncher'); - assert.equal(launcher.result.appBundleId, 'com.google.android.apps.nexuslauncher'); - assert.equal(launcher.stored?.appBundleId, 'com.android.settings'); + const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); + return { result, stored }; +} - const settings = await captureWith({ - type: 'TextView', - label: 'Network & internet', - bundleId: 'com.android.settings', - }); - assert.equal(settings.result.appName, 'Settings'); - assert.equal(settings.result.appBundleId, 'com.android.settings'); +test('an Android snapshot names the captured app when the session app has no window in it', async () => { + const { result, stored } = await snapshotWithAppWindows([ + 'com.google.android.apps.nexuslauncher', + ]); + + assert.equal(result.appName, 'com.google.android.apps.nexuslauncher'); + assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); + assert.equal(stored?.appBundleId, 'com.android.settings'); +}); + +test('an Android snapshot keeps the session app while one of its windows is captured', async () => { + const { result } = await snapshotWithAppWindows([ + 'com.google.android.permissioncontroller', + 'com.android.settings', + ]); + + assert.equal(result.appName, 'Settings'); + assert.equal(result.appBundleId, 'com.android.settings'); +}); + +test('a snapshot without application window facts names the session app', async () => { + const { result } = await snapshotWithAppWindows(undefined); + + assert.equal(result.appBundleId, 'com.android.settings'); }); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 720ab3ba3d..90218e9220 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -19,10 +19,7 @@ import type { SnapshotVisibility, } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; -import type { - AgentDeviceBackendPlatform, - BackendSnapshotResult, -} from '@agent-device/contracts/backend'; +import type { BackendSnapshotResult } from '@agent-device/contracts/backend'; import type { AgentDeviceRuntime, CommandSessionRecord, @@ -125,7 +122,7 @@ export const snapshotCommand: RuntimeCommand< ...(capture.snapshot.presentedSurface ? { presentedSurface: capture.snapshot.presentedSurface } : {}), - ...snapshotAppFields(capture, runtime.backend.platform), + ...snapshotAppFields(capture), }); }; @@ -238,22 +235,18 @@ function nextSnapshotSession( } /** - * The app the response header names. Every Android node carries its window's package, so a capture - * whose app content belongs only to another package, such as the launcher after a failed open, - * names that package instead of the session app. The session keeps its app. + * The app the response header names. An Android helper capture lists its application windows, so + * one that shows none of the session app, such as the launcher after a failed open, names the + * topmost captured app instead. The session keeps its app. */ -function snapshotAppFields( - capture: SnapshotCapture, - platform: AgentDeviceBackendPlatform, -): { +function snapshotAppFields(capture: SnapshotCapture): { appName?: string; appBundleId?: string; } { const sessionApp = capture.result.appBundleId ?? capture.session?.appBundleId; + const appWindows = capture.annotations.androidSnapshot?.applicationWindowPackages; const shownApp = - platform === 'android' && sessionApp - ? capturedAndroidApp(capture.snapshot.nodes, sessionApp) - : sessionApp; + sessionApp && appWindows?.[0] && !appWindows.includes(sessionApp) ? appWindows[0] : sessionApp; const appName = shownApp === sessionApp ? (capture.result.appName ?? capture.session?.appName) : shownApp; return { @@ -262,17 +255,6 @@ function snapshotAppFields( }; } -/** The session app when any app node belongs to it, else the first other app package. */ -function capturedAndroidApp(nodes: readonly SnapshotNode[], sessionApp: string): string { - let otherApp: string | undefined; - for (const node of nodes) { - if (!node.bundleId || node.systemBar || node.keyboardOwned) continue; - if (node.bundleId === sessionApp) return sessionApp; - otherApp ??= node.bundleId; - } - return otherApp ?? sessionApp; -} - /** * A capture that reported nothing about truncation is only "not truncated" when its producer * actually observes truncation. Producers that do not (Appium page source, the Limrun element diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index 6df1bb1549..390b025023 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -79,6 +79,11 @@ export type AndroidSnapshotBackendMetadata = { cacheInvalidated?: boolean; captureMode?: string; systemSurfaceOnly?: boolean; + /** + * Packages of the captured application windows that are not system-owned, topmost first. Input + * method and system windows are excluded. Absent when the capture read no such window. + */ + applicationWindowPackages?: string[]; windowCount?: number; /** * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because diff --git a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts index c434c98e52..e590123b91 100644 --- a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts @@ -270,6 +270,38 @@ test('rejects system chrome inherited under an empty application window', () => expect(decision?.diagnostics.helperNonSystemMeaningfulNodeCount).toBe(0); }); +test('lists only non-system application windows, topmost first', () => { + const window = (windowType: number, packageName: string, text: string) => + node({ + windowType, + packageName, + className: 'android.widget.FrameLayout', + children: [node({ text, packageName, className: 'android.widget.TextView' })], + }); + const result = classifyAndroidHelperContent( + helperXml([ + window(3, 'com.android.systemui', '12:00'), + window(1, 'android', 'Allow?'), + window(2, 'com.google.android.inputmethod.latin', 'Space'), + window(1, 'com.google.android.apps.nexuslauncher', 'Gmail'), + window(1, 'com.google.android.apps.nexuslauncher', 'Photos'), + window(1, 'com.example.app', 'Home'), + ]), + { + backend: 'android-helper', + nodeCount: 12, + rootPresent: true, + windowCount: 6, + captureMode: 'interactive-windows', + }, + ); + + expect(result.outcome === 'unusable' ? [] : result.applicationWindowPackages).toEqual([ + 'com.google.android.apps.nexuslauncher', + 'com.example.app', + ]); +}); + test('returns a meaningful active system surface (notification shade) faithfully', () => { const result = classifyAndroidHelperContent( helperXml([ diff --git a/packages/platform-android/src/snapshot-content-recovery.ts b/packages/platform-android/src/snapshot-content-recovery.ts index 83269ce198..51f4c13fe5 100644 --- a/packages/platform-android/src/snapshot-content-recovery.ts +++ b/packages/platform-android/src/snapshot-content-recovery.ts @@ -35,19 +35,41 @@ export type AndroidHelperContentRecoveryDecision = { }; }; -export type AndroidHelperContentClassification = +type AndroidHelperContentOutcome = | { outcome: 'ok' } | { outcome: 'system-surface-only' } | { outcome: 'unusable'; decision: AndroidHelperContentRecoveryDecision }; +export type AndroidHelperContentClassification = + | ({ outcome: 'ok' | 'system-surface-only' } & AndroidApplicationWindows) + | { outcome: 'unusable'; decision: AndroidHelperContentRecoveryDecision }; + +type AndroidApplicationWindows = { + /** + * Packages of the captured application windows that are not system-owned, topmost first: the + * app a snapshot header names. Input method and system windows are never application windows. + */ + applicationWindowPackages: string[]; +}; + export function classifyAndroidHelperContent( xml: string, metadata: AndroidSnapshotBackendMetadata, options: { foregroundAppPackage?: string } = {}, ): AndroidHelperContentClassification { - if (metadata.backend !== 'android-helper') return { outcome: 'ok' }; - + if (metadata.backend !== 'android-helper') + return { outcome: 'ok', applicationWindowPackages: [] }; const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage); + const content = classifyAndroidHelperSummary(summary, metadata); + return content.outcome === 'unusable' + ? content + : { ...content, applicationWindowPackages: summary.applicationWindowPackages }; +} + +function classifyAndroidHelperSummary( + summary: AndroidHelperXmlSummary, + metadata: AndroidSnapshotBackendMetadata, +): AndroidHelperContentOutcome { if (isEmptyHelperOutput(summary, metadata)) { return unusable( summary, @@ -97,7 +119,7 @@ function unusable( metadata: AndroidSnapshotBackendMetadata, reason: AndroidHelperContentRecoveryDecision['reason'], failureReason: string, -): AndroidHelperContentClassification { +): AndroidHelperContentOutcome { return { outcome: 'unusable', decision: buildRecoveryDecision(summary, metadata, reason, failureReason), @@ -201,7 +223,7 @@ type AndroidHelperXmlSummary = { foregroundAppPackage?: string; foregroundAppMeaningfulNodeCount?: number; windowTypes: number[]; -}; +} & AndroidApplicationWindows; type AndroidHelperXmlSummaryState = Omit & { currentWindowType?: number; @@ -243,6 +265,7 @@ function createAndroidHelperXmlSummaryState( ? { foregroundAppPackage, foregroundAppMeaningfulNodeCount: 0 } : {}), windowTypes: new Set(), + applicationWindowPackages: [], }; } @@ -280,6 +303,12 @@ function recordAndroidHelperWindowNode( if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) { summary.applicationWindowRootCount += 1; if (summary.currentWindowSystemOwned) summary.systemApplicationWindowRootCount += 1; + else if ( + node.packageName !== null && + !summary.applicationWindowPackages.includes(node.packageName) + ) { + summary.applicationWindowPackages.push(node.packageName); + } } } @@ -370,6 +399,7 @@ function finalizeAndroidHelperXmlSummary( } : {}), windowTypes: [...summary.windowTypes].sort((a, b) => a - b).slice(0, MAX_REPORTED_WINDOW_TYPES), + applicationWindowPackages: summary.applicationWindowPackages, }; } diff --git a/packages/platform-android/src/snapshot-types.ts b/packages/platform-android/src/snapshot-types.ts index 5ca8274e74..f02888f49c 100644 --- a/packages/platform-android/src/snapshot-types.ts +++ b/packages/platform-android/src/snapshot-types.ts @@ -41,6 +41,11 @@ export type AndroidSnapshotBackendMetadata = { cacheInvalidated?: boolean; captureMode?: AndroidSnapshotCaptureMode; systemSurfaceOnly?: boolean; + /** + * Packages of the captured application windows that are not system-owned, topmost first. Input + * method and system windows are excluded. Absent when the capture read no such window. + */ + applicationWindowPackages?: string[]; windowCount?: number; /** * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 8cfb8b56c4..5038981530 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -639,9 +639,13 @@ async function captureAndroidHelperContentAttempt(params: { capture: { xml: helperCapture.xml, helperMetadata: helperCapture.helperMetadata, - metadata: systemSurfaceOnly - ? { ...helperCapture.metadata, systemSurfaceOnly: true } - : helperCapture.metadata, + metadata: { + ...helperCapture.metadata, + ...(systemSurfaceOnly ? { systemSurfaceOnly: true } : {}), + ...(content.applicationWindowPackages.length > 0 + ? { applicationWindowPackages: content.applicationWindowPackages } + : {}), + }, }, }; } diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 13758c368f..6da1553220 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -80,7 +80,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue now, and hints that point to hidden list content. - Use `snapshot -i` by default in agent loops. - On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm. -- On Android, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the app the snapshot shows. If the session app is not in front, for example after a failed `open`, they name the app that is, such as the launcher. The session keeps its app. +- On Android, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the session app while the snapshot holds one of its windows. When it holds none, for example after a failed `open`, they name the topmost captured application window, such as the launcher; system UI and keyboard windows never count. `androidSnapshot.applicationWindowPackages` in `--json` lists those windows. The session keeps its app. - On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway. - The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate. - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`. From 5eccba64b6e8f802108f26594888763ac20f57a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:29:08 +0100 Subject: [PATCH 4/7] fix(snapshot): order Android app windows by layer and name one without a session app Address review: the helper lists windows in getWindows() order, which is not z-order, so the header could name an app behind another. Windows now sort by window layer. A session with no app names the topmost captured app instead of printing no header, and the docs describe the list as deduplicated package names. Co-Authored-By: Claude Opus 5.5 --- .../runtime/snapshot-app-identity.test.ts | 23 +++++++++++-- .../src/commands/capture/runtime/snapshot.ts | 6 ++-- packages/contracts/src/snapshot-types.ts | 5 +-- .../snapshot-content-recovery.test.ts | 16 +++++---- .../src/snapshot-content-recovery.ts | 33 +++++++++++++------ .../platform-android/src/snapshot-types.ts | 5 +-- website/docs/docs/snapshots.md | 2 +- 7 files changed, 63 insertions(+), 27 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts index aa70738ae7..fec0c2ee28 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts @@ -6,7 +6,13 @@ import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -async function snapshotWithAppWindows(applicationWindowPackages: string[] | undefined) { +async function snapshotWithAppWindows( + applicationWindowPackages: string[] | undefined, + sessionApp: { appName: string; appBundleId: string } | undefined = { + appName: 'Settings', + appBundleId: 'com.android.settings', + }, +) { let stored: Parameters[0] | undefined; const device = createAgentDevice({ backend: { @@ -19,8 +25,7 @@ async function snapshotWithAppWindows(applicationWindowPackages: string[] | unde backend: 'android-helper', ...(applicationWindowPackages ? { applicationWindowPackages } : {}), }, - appName: 'Settings', - appBundleId: 'com.android.settings', + ...sessionApp, }), }, artifacts: createLocalArtifactAdapter(), @@ -39,6 +44,7 @@ async function snapshotWithAppWindows(applicationWindowPackages: string[] | unde test('an Android snapshot names the captured app when the session app has no window in it', async () => { const { result, stored } = await snapshotWithAppWindows([ 'com.google.android.apps.nexuslauncher', + 'com.example.behind', ]); assert.equal(result.appName, 'com.google.android.apps.nexuslauncher'); @@ -59,5 +65,16 @@ test('an Android snapshot keeps the session app while one of its windows is capt test('a snapshot without application window facts names the session app', async () => { const { result } = await snapshotWithAppWindows(undefined); + assert.equal(result.appName, 'Settings'); assert.equal(result.appBundleId, 'com.android.settings'); }); + +test('an Android snapshot in a session with no app names the topmost captured app', async () => { + const { result } = await snapshotWithAppWindows( + ['com.google.android.apps.nexuslauncher', 'com.example.behind'], + undefined, + ); + + assert.equal(result.appName, 'com.google.android.apps.nexuslauncher'); + assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); +}); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 90218e9220..75746b7bb7 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -236,8 +236,8 @@ function nextSnapshotSession( /** * The app the response header names. An Android helper capture lists its application windows, so - * one that shows none of the session app, such as the launcher after a failed open, names the - * topmost captured app instead. The session keeps its app. + * one that shows none of the session app, such as the launcher after a failed open, or a session + * with no app, names the topmost captured app instead. The session keeps its app. */ function snapshotAppFields(capture: SnapshotCapture): { appName?: string; @@ -246,7 +246,7 @@ function snapshotAppFields(capture: SnapshotCapture): { const sessionApp = capture.result.appBundleId ?? capture.session?.appBundleId; const appWindows = capture.annotations.androidSnapshot?.applicationWindowPackages; const shownApp = - sessionApp && appWindows?.[0] && !appWindows.includes(sessionApp) ? appWindows[0] : sessionApp; + sessionApp && appWindows?.includes(sessionApp) ? sessionApp : (appWindows?.[0] ?? sessionApp); const appName = shownApp === sessionApp ? (capture.result.appName ?? capture.session?.appName) : shownApp; return { diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index 390b025023..d95c500ae3 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -80,8 +80,9 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: string; systemSurfaceOnly?: boolean; /** - * Packages of the captured application windows that are not system-owned, topmost first. Input - * method and system windows are excluded. Absent when the capture read no such window. + * Package names of the captured application windows that are not system-owned, by window layer + * (topmost first), each listed once. Input method and system windows are excluded. Absent when + * the capture read no such window. */ applicationWindowPackages?: string[]; windowCount?: number; diff --git a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts index e590123b91..8cec5072ab 100644 --- a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts @@ -270,10 +270,11 @@ test('rejects system chrome inherited under an empty application window', () => expect(decision?.diagnostics.helperNonSystemMeaningfulNodeCount).toBe(0); }); -test('lists only non-system application windows, topmost first', () => { - const window = (windowType: number, packageName: string, text: string) => +test('lists only non-system application windows, by layer, topmost first', () => { + const window = (windowType: number, packageName: string, text: string, windowLayer = 0) => node({ windowType, + windowLayer, packageName, className: 'android.widget.FrameLayout', children: [node({ text, packageName, className: 'android.widget.TextView' })], @@ -283,9 +284,10 @@ test('lists only non-system application windows, topmost first', () => { window(3, 'com.android.systemui', '12:00'), window(1, 'android', 'Allow?'), window(2, 'com.google.android.inputmethod.latin', 'Space'), - window(1, 'com.google.android.apps.nexuslauncher', 'Gmail'), - window(1, 'com.google.android.apps.nexuslauncher', 'Photos'), - window(1, 'com.example.app', 'Home'), + // Listed before the app, but below it: getWindows() order is not z-order. + window(1, 'com.google.android.apps.nexuslauncher', 'Gmail', 1), + window(1, 'com.google.android.apps.nexuslauncher', 'Photos', 1), + window(1, 'com.example.app', 'Home', 5), ]), { backend: 'android-helper', @@ -297,8 +299,8 @@ test('lists only non-system application windows, topmost first', () => { ); expect(result.outcome === 'unusable' ? [] : result.applicationWindowPackages).toEqual([ - 'com.google.android.apps.nexuslauncher', 'com.example.app', + 'com.google.android.apps.nexuslauncher', ]); }); @@ -558,12 +560,14 @@ function node(options: { packageName: string; className: string; windowType?: number; + windowLayer?: number; windowActive?: boolean; /** Nested children, so chrome leaves can sit inside their status/nav-bar container as on a device. */ children?: string[]; }): string { const windowAttrs = (options.windowType === undefined ? '' : ` window-type="${options.windowType}"`) + + (options.windowLayer === undefined ? '' : ` window-layer="${options.windowLayer}"`) + (options.windowActive === undefined ? '' : ` window-active="${options.windowActive}"`); const open = ` & { +type AndroidHelperXmlSummaryState = Omit< + AndroidHelperXmlSummary, + 'windowTypes' | 'applicationWindowPackages' +> & { + applicationWindows: Array<{ packageName: string; layer: number }>; currentWindowType?: number; currentWindowActiveOrFocused?: boolean; currentWindowSystemOwned?: boolean; @@ -265,7 +270,7 @@ function createAndroidHelperXmlSummaryState( ? { foregroundAppPackage, foregroundAppMeaningfulNodeCount: 0 } : {}), windowTypes: new Set(), - applicationWindowPackages: [], + applicationWindows: [], }; } @@ -303,11 +308,11 @@ function recordAndroidHelperWindowNode( if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) { summary.applicationWindowRootCount += 1; if (summary.currentWindowSystemOwned) summary.systemApplicationWindowRootCount += 1; - else if ( - node.packageName !== null && - !summary.applicationWindowPackages.includes(node.packageName) - ) { - summary.applicationWindowPackages.push(node.packageName); + else if (node.packageName !== null) { + summary.applicationWindows.push({ + packageName: node.packageName, + layer: node.windowLayer ?? Number.NEGATIVE_INFINITY, + }); } } } @@ -399,7 +404,15 @@ function finalizeAndroidHelperXmlSummary( } : {}), windowTypes: [...summary.windowTypes].sort((a, b) => a - b).slice(0, MAX_REPORTED_WINDOW_TYPES), - applicationWindowPackages: summary.applicationWindowPackages, + // The helper lists windows in getWindows() order, which is not z-order; the layer is. + applicationWindowPackages: [ + ...new Set( + summary.applicationWindows + .slice() + .sort((a, b) => b.layer - a.layer || 0) + .map((window) => window.packageName), + ), + ], }; } diff --git a/packages/platform-android/src/snapshot-types.ts b/packages/platform-android/src/snapshot-types.ts index f02888f49c..3698b5f356 100644 --- a/packages/platform-android/src/snapshot-types.ts +++ b/packages/platform-android/src/snapshot-types.ts @@ -42,8 +42,9 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: AndroidSnapshotCaptureMode; systemSurfaceOnly?: boolean; /** - * Packages of the captured application windows that are not system-owned, topmost first. Input - * method and system windows are excluded. Absent when the capture read no such window. + * Package names of the captured application windows that are not system-owned, by window layer + * (topmost first), each listed once. Input method and system windows are excluded. Absent when + * the capture read no such window. */ applicationWindowPackages?: string[]; windowCount?: number; diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 6da1553220..a5d4f7b224 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -80,7 +80,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue now, and hints that point to hidden list content. - Use `snapshot -i` by default in agent loops. - On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm. -- On Android, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the session app while the snapshot holds one of its windows. When it holds none, for example after a failed `open`, they name the topmost captured application window, such as the launcher; system UI and keyboard windows never count. `androidSnapshot.applicationWindowPackages` in `--json` lists those windows. The session keeps its app. +- On Android, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the session app while the snapshot holds one of its windows. When it holds none, for example after a failed `open`, or the session has no app, they name the app of the topmost captured application window, such as the launcher; system UI and keyboard windows never count. `androidSnapshot.applicationWindowPackages` in `--json` lists those package names, topmost first. The session keeps its app. - On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway. - The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate. - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`. From f1eb37a425379a16602b59fb3618b6653741d118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 21:37:25 +0100 Subject: [PATCH 5/7] fix(snapshot): name the header app from the foreground application window Address review: a list of every application window could keep a stale session-app window that Android leaves behind after a transition. The helper classifier now reports one foreground window package, active or focused at the highest layer, using the presentation's foreground-window predicate, and the header names it. Co-Authored-By: Claude Opus 5.5 --- .../runtime/snapshot-app-identity.test.ts | 31 +++-- .../src/commands/capture/runtime/snapshot.ts | 10 +- packages/contracts/src/snapshot-types.ts | 7 +- .../snapshot-content-recovery.test.ts | 108 +++++++++++++----- .../src/__tests__/snapshot.test.ts | 8 +- .../src/snapshot-content-recovery.ts | 72 +++++------- .../platform-android/src/snapshot-types.ts | 7 +- packages/platform-android/src/snapshot.ts | 4 +- .../src/ui-hierarchy-visibility.ts | 5 +- website/docs/docs/snapshots.md | 2 +- 10 files changed, 139 insertions(+), 115 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts index fec0c2ee28..7e0f281995 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts @@ -6,8 +6,8 @@ import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -async function snapshotWithAppWindows( - applicationWindowPackages: string[] | undefined, +async function snapshotWithForeground( + foregroundWindowPackage: string | undefined, sessionApp: { appName: string; appBundleId: string } | undefined = { appName: 'Settings', appBundleId: 'com.android.settings', @@ -23,7 +23,7 @@ async function snapshotWithAppWindows( }), androidSnapshot: { backend: 'android-helper', - ...(applicationWindowPackages ? { applicationWindowPackages } : {}), + ...(foregroundWindowPackage ? { foregroundWindowPackage } : {}), }, ...sessionApp, }), @@ -41,37 +41,32 @@ async function snapshotWithAppWindows( return { result, stored }; } -test('an Android snapshot names the captured app when the session app has no window in it', async () => { - const { result, stored } = await snapshotWithAppWindows([ - 'com.google.android.apps.nexuslauncher', - 'com.example.behind', - ]); +test('an Android snapshot names the captured foreground app when it is not the session app', async () => { + const { result, stored } = await snapshotWithForeground('com.google.android.apps.nexuslauncher'); + // A foreign app has no label of its own here, so the header shows its package. assert.equal(result.appName, 'com.google.android.apps.nexuslauncher'); assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); assert.equal(stored?.appBundleId, 'com.android.settings'); }); -test('an Android snapshot keeps the session app while one of its windows is captured', async () => { - const { result } = await snapshotWithAppWindows([ - 'com.google.android.permissioncontroller', - 'com.android.settings', - ]); +test('an Android snapshot keeps the session app label when the session app is in front', async () => { + const { result } = await snapshotWithForeground('com.android.settings'); assert.equal(result.appName, 'Settings'); assert.equal(result.appBundleId, 'com.android.settings'); }); -test('a snapshot without application window facts names the session app', async () => { - const { result } = await snapshotWithAppWindows(undefined); +test('a snapshot without a foreground window fact names the session app', async () => { + const { result } = await snapshotWithForeground(undefined); assert.equal(result.appName, 'Settings'); assert.equal(result.appBundleId, 'com.android.settings'); }); -test('an Android snapshot in a session with no app names the topmost captured app', async () => { - const { result } = await snapshotWithAppWindows( - ['com.google.android.apps.nexuslauncher', 'com.example.behind'], +test('an Android snapshot in a session with no app names the captured foreground app', async () => { + const { result } = await snapshotWithForeground( + 'com.google.android.apps.nexuslauncher', undefined, ); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 75746b7bb7..d99479a72c 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -235,18 +235,16 @@ function nextSnapshotSession( } /** - * The app the response header names. An Android helper capture lists its application windows, so - * one that shows none of the session app, such as the launcher after a failed open, or a session - * with no app, names the topmost captured app instead. The session keeps its app. + * The app the response header names: the foreground application window the Android snapshot + * helper captured, such as the launcher after a failed open, else the session app. The session + * keeps its app. */ function snapshotAppFields(capture: SnapshotCapture): { appName?: string; appBundleId?: string; } { const sessionApp = capture.result.appBundleId ?? capture.session?.appBundleId; - const appWindows = capture.annotations.androidSnapshot?.applicationWindowPackages; - const shownApp = - sessionApp && appWindows?.includes(sessionApp) ? sessionApp : (appWindows?.[0] ?? sessionApp); + const shownApp = capture.annotations.androidSnapshot?.foregroundWindowPackage ?? sessionApp; const appName = shownApp === sessionApp ? (capture.result.appName ?? capture.session?.appName) : shownApp; return { diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index d95c500ae3..dd843f06a3 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -80,11 +80,10 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: string; systemSurfaceOnly?: boolean; /** - * Package names of the captured application windows that are not system-owned, by window layer - * (topmost first), each listed once. Input method and system windows are excluded. Absent when - * the capture read no such window. + * Package of the foreground application window the snapshot helper captured: active or focused, + * at the highest layer, and not owned by the system. Absent when the helper saw none. */ - applicationWindowPackages?: string[]; + foregroundWindowPackage?: string; windowCount?: number; /** * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because diff --git a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts index 8cec5072ab..8564c35068 100644 --- a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts @@ -270,38 +270,86 @@ test('rejects system chrome inherited under an empty application window', () => expect(decision?.diagnostics.helperNonSystemMeaningfulNodeCount).toBe(0); }); -test('lists only non-system application windows, by layer, topmost first', () => { - const window = (windowType: number, packageName: string, text: string, windowLayer = 0) => - node({ - windowType, - windowLayer, - packageName, - className: 'android.widget.FrameLayout', - children: [node({ text, packageName, className: 'android.widget.TextView' })], - }); - const result = classifyAndroidHelperContent( - helperXml([ - window(3, 'com.android.systemui', '12:00'), - window(1, 'android', 'Allow?'), - window(2, 'com.google.android.inputmethod.latin', 'Space'), - // Listed before the app, but below it: getWindows() order is not z-order. - window(1, 'com.google.android.apps.nexuslauncher', 'Gmail', 1), - window(1, 'com.google.android.apps.nexuslauncher', 'Photos', 1), - window(1, 'com.example.app', 'Home', 5), +function foregroundWindowPackageOf(windows: string[]): string | undefined { + const result = classifyAndroidHelperContent(helperXml(windows), { + backend: 'android-helper', + nodeCount: windows.length * 2, + rootPresent: true, + windowCount: windows.length, + captureMode: 'interactive-windows', + }); + return result.outcome === 'unusable' ? undefined : result.foregroundWindowPackage; +} + +function window(params: { + windowType: number; + packageName: string; + text: string; + windowLayer: number; + windowActive?: boolean; +}): string { + const { text, ...windowAttributes } = params; + return node({ + ...windowAttributes, + className: 'android.widget.FrameLayout', + children: [ + node({ text, packageName: params.packageName, className: 'android.widget.TextView' }), + ], + }); +} + +test('names the active application window, not a stale one listed first or a system window above it', () => { + expect( + foregroundWindowPackageOf([ + window({ + windowType: 3, + packageName: 'com.android.systemui', + text: '12:00', + windowLayer: 9, + windowActive: true, + }), + window({ + windowType: 2, + packageName: 'com.google.android.inputmethod.latin', + text: 'Space', + windowLayer: 8, + }), + // A stale session-app window Android kept after the transition: inactive, lower layer. + window({ + windowType: 1, + packageName: 'com.android.settings', + text: 'Network & internet', + windowLayer: 1, + }), + window({ + windowType: 1, + packageName: 'com.google.android.apps.nexuslauncher', + text: 'Gmail', + windowLayer: 2, + windowActive: true, + }), ]), - { - backend: 'android-helper', - nodeCount: 12, - rootPresent: true, - windowCount: 6, - captureMode: 'interactive-windows', - }, - ); + ).toBe('com.google.android.apps.nexuslauncher'); +}); - expect(result.outcome === 'unusable' ? [] : result.applicationWindowPackages).toEqual([ - 'com.example.app', - 'com.google.android.apps.nexuslauncher', - ]); +test('names no foreground app when only a framework window is active', () => { + expect( + foregroundWindowPackageOf([ + window({ + windowType: 1, + packageName: 'android', + text: 'Allow?', + windowLayer: 3, + windowActive: true, + }), + window({ + windowType: 1, + packageName: 'com.android.settings', + text: 'Network & internet', + windowLayer: 1, + }), + ]), + ).toBeUndefined(); }); test('returns a meaningful active system surface (notification shade) faithfully', () => { diff --git a/packages/platform-android/src/__tests__/snapshot.test.ts b/packages/platform-android/src/__tests__/snapshot.test.ts index 15d063a768..64d88dff9e 100644 --- a/packages/platform-android/src/__tests__/snapshot.test.ts +++ b/packages/platform-android/src/__tests__/snapshot.test.ts @@ -889,11 +889,9 @@ test('snapshotAndroid keeps helper output when application and system windows ar appBundleId: 'io.example.fabric', }); - assert.equal(result.androidSnapshot.backend, 'android-helper'); - assert.equal( - result.nodes.some((node) => node.label === 'Fabric dashboard'), - true, - ); + const { backend, foregroundWindowPackage } = result.androidSnapshot; + assert.deepEqual([backend, foregroundWindowPackage], ['android-helper', 'io.example.fabric']); + assert.ok(result.nodes.some((node) => node.label === 'Fabric dashboard')); }); test('snapshotAndroid emits helper failure diagnostics', async () => { diff --git a/packages/platform-android/src/snapshot-content-recovery.ts b/packages/platform-android/src/snapshot-content-recovery.ts index 54e24568cb..4e4e29b8e7 100644 --- a/packages/platform-android/src/snapshot-content-recovery.ts +++ b/packages/platform-android/src/snapshot-content-recovery.ts @@ -5,6 +5,7 @@ import { classifyAndroidAlertIdentifier } from './alert-detection.ts'; import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; import { ANDROID_WINDOW_TYPE_APPLICATION } from './ui-hierarchy-node.ts'; +import { isAndroidForegroundWindow } from './ui-hierarchy-visibility.ts'; const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; @@ -35,42 +36,28 @@ export type AndroidHelperContentRecoveryDecision = { }; }; -type AndroidHelperContentOutcome = +export type AndroidHelperContentClassification = | { outcome: 'ok' } | { outcome: 'system-surface-only' } | { outcome: 'unusable'; decision: AndroidHelperContentRecoveryDecision }; -export type AndroidHelperContentClassification = - | ({ outcome: 'ok' | 'system-surface-only' } & AndroidApplicationWindows) - | { outcome: 'unusable'; decision: AndroidHelperContentRecoveryDecision }; - -type AndroidApplicationWindows = { - /** - * Package names of the captured application windows that are not system-owned, by window layer - * (topmost first), each listed once: the app a snapshot header names. Input method and system - * windows are never application windows. - */ - applicationWindowPackages: string[]; -}; - export function classifyAndroidHelperContent( xml: string, metadata: AndroidSnapshotBackendMetadata, options: { foregroundAppPackage?: string } = {}, -): AndroidHelperContentClassification { - if (metadata.backend !== 'android-helper') - return { outcome: 'ok', applicationWindowPackages: [] }; +): AndroidHelperContentClassification & { foregroundWindowPackage?: string } { + if (metadata.backend !== 'android-helper') return { outcome: 'ok' }; const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage); const content = classifyAndroidHelperSummary(summary, metadata); - return content.outcome === 'unusable' - ? content - : { ...content, applicationWindowPackages: summary.applicationWindowPackages }; + return summary.foregroundWindow + ? { ...content, foregroundWindowPackage: summary.foregroundWindow.packageName } + : content; } function classifyAndroidHelperSummary( summary: AndroidHelperXmlSummary, metadata: AndroidSnapshotBackendMetadata, -): AndroidHelperContentOutcome { +): AndroidHelperContentClassification { if (isEmptyHelperOutput(summary, metadata)) { return unusable( summary, @@ -120,7 +107,7 @@ function unusable( metadata: AndroidSnapshotBackendMetadata, reason: AndroidHelperContentRecoveryDecision['reason'], failureReason: string, -): AndroidHelperContentOutcome { +): AndroidHelperContentClassification { return { outcome: 'unusable', decision: buildRecoveryDecision(summary, metadata, reason, failureReason), @@ -224,13 +211,14 @@ type AndroidHelperXmlSummary = { foregroundAppPackage?: string; foregroundAppMeaningfulNodeCount?: number; windowTypes: number[]; -} & AndroidApplicationWindows; + /** + * The non-system application window the presentation keeps as foreground: active or focused, at + * the highest layer. A stale application window under it is not foreground. + */ + foregroundWindow?: { packageName: string; layer: number }; +}; -type AndroidHelperXmlSummaryState = Omit< - AndroidHelperXmlSummary, - 'windowTypes' | 'applicationWindowPackages' -> & { - applicationWindows: Array<{ packageName: string; layer: number }>; +type AndroidHelperXmlSummaryState = Omit & { currentWindowType?: number; currentWindowActiveOrFocused?: boolean; currentWindowSystemOwned?: boolean; @@ -270,7 +258,6 @@ function createAndroidHelperXmlSummaryState( ? { foregroundAppPackage, foregroundAppMeaningfulNodeCount: 0 } : {}), windowTypes: new Set(), - applicationWindows: [], }; } @@ -308,15 +295,20 @@ function recordAndroidHelperWindowNode( if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) { summary.applicationWindowRootCount += 1; if (summary.currentWindowSystemOwned) summary.systemApplicationWindowRootCount += 1; - else if (node.packageName !== null) { - summary.applicationWindows.push({ - packageName: node.packageName, - layer: node.windowLayer ?? Number.NEGATIVE_INFINITY, - }); - } + else recordAndroidForegroundWindow(summary, node); } } +function recordAndroidForegroundWindow( + summary: AndroidHelperXmlSummaryState, + node: AndroidUiNodeMetadata, +): void { + if (node.packageName === null || !isAndroidForegroundWindow(node)) return; + const layer = node.windowLayer ?? -1; + if (summary.foregroundWindow && summary.foregroundWindow.layer >= layer) return; + summary.foregroundWindow = { packageName: node.packageName, layer }; +} + function recordAndroidHelperMeaningfulNode( summary: AndroidHelperXmlSummaryState, node: AndroidUiNodeMetadata, @@ -404,15 +396,7 @@ function finalizeAndroidHelperXmlSummary( } : {}), windowTypes: [...summary.windowTypes].sort((a, b) => a - b).slice(0, MAX_REPORTED_WINDOW_TYPES), - // The helper lists windows in getWindows() order, which is not z-order; the layer is. - applicationWindowPackages: [ - ...new Set( - summary.applicationWindows - .slice() - .sort((a, b) => b.layer - a.layer || 0) - .map((window) => window.packageName), - ), - ], + ...(summary.foregroundWindow ? { foregroundWindow: summary.foregroundWindow } : {}), }; } diff --git a/packages/platform-android/src/snapshot-types.ts b/packages/platform-android/src/snapshot-types.ts index 3698b5f356..ac0163a6cb 100644 --- a/packages/platform-android/src/snapshot-types.ts +++ b/packages/platform-android/src/snapshot-types.ts @@ -42,11 +42,10 @@ export type AndroidSnapshotBackendMetadata = { captureMode?: AndroidSnapshotCaptureMode; systemSurfaceOnly?: boolean; /** - * Package names of the captured application windows that are not system-owned, by window layer - * (topmost first), each listed once. Input method and system windows are excluded. Absent when - * the capture read no such window. + * Package of the foreground application window the snapshot helper captured: active or focused, + * at the highest layer, and not owned by the system. Absent when the helper saw none. */ - applicationWindowPackages?: string[]; + foregroundWindowPackage?: string; windowCount?: number; /** * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 5038981530..c8adf975f3 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -642,8 +642,8 @@ async function captureAndroidHelperContentAttempt(params: { metadata: { ...helperCapture.metadata, ...(systemSurfaceOnly ? { systemSurfaceOnly: true } : {}), - ...(content.applicationWindowPackages.length > 0 - ? { applicationWindowPackages: content.applicationWindowPackages } + ...(content.foregroundWindowPackage + ? { foregroundWindowPackage: content.foregroundWindowPackage } : {}), }, }, diff --git a/packages/platform-android/src/ui-hierarchy-visibility.ts b/packages/platform-android/src/ui-hierarchy-visibility.ts index a93d7152e4..ae6cb082c8 100644 --- a/packages/platform-android/src/ui-hierarchy-visibility.ts +++ b/packages/platform-android/src/ui-hierarchy-visibility.ts @@ -92,6 +92,9 @@ function isAndroidApplicationWindow(node: AndroidNode): boolean { return node.windowType === ANDROID_WINDOW_TYPE_APPLICATION; } -function isAndroidForegroundWindow(node: AndroidNode): boolean { +/** A window the user is in: active or focused. Shared with the snapshot header's foreground app. */ +export function isAndroidForegroundWindow( + node: Pick, +): boolean { return node.windowActive === true || node.windowFocused === true; } diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index a5d4f7b224..a2a19fda31 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -80,7 +80,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue now, and hints that point to hidden list content. - Use `snapshot -i` by default in agent loops. - On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm. -- On Android, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the session app while the snapshot holds one of its windows. When it holds none, for example after a failed `open`, or the session has no app, they name the app of the topmost captured application window, such as the launcher; system UI and keyboard windows never count. `androidSnapshot.applicationWindowPackages` in `--json` lists those package names, topmost first. The session keeps its app. +- When the Android snapshot helper captures the tree, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the foreground application window: active or focused, at the highest layer, and not a system UI or keyboard window. After a failed `open`, for example, they name the launcher. An app other than the session app shows its package name in both lines. `androidSnapshot.foregroundWindowPackage` in `--json` carries it, and the session keeps its app. The UIAutomator fallback names the session app. - On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway. - The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate. - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`. From 53b5f3c4f1bea3a9272b7b313c3b94e5c94eae1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 22:19:42 +0100 Subject: [PATCH 6/7] fix(snapshot): key the unchanged acknowledgement on the header app Address review: the unchanged check compared the session app while the header named the foreground app, so a repeated snapshot of the launcher could read as unchanged against Settings. The check now uses the app the header names. A test fixture no longer writes a raw ampersand into an XML attribute. Co-Authored-By: Claude Opus 5.5 --- .../runtime/snapshot-app-identity.test.ts | 37 +++++++++++++++++++ .../src/commands/capture/runtime/snapshot.ts | 3 +- .../snapshot-content-recovery.test.ts | 4 +- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts index 7e0f281995..dbc34c46db 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts @@ -73,3 +73,40 @@ test('an Android snapshot in a session with no app names the captured foreground assert.equal(result.appName, 'com.google.android.apps.nexuslauncher'); assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); }); + +test('a repeated snapshot of a foreign foreground app is not acknowledged as unchanged', async () => { + const snapshot = makeSnapshotState([{ index: 0, depth: 0, type: 'TextView', label: 'Gmail' }], { + backend: 'android', + }); + let stored: Parameters[0] | undefined = { + name: 'default', + appBundleId: 'com.android.settings', + }; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async () => ({ + snapshot, + androidSnapshot: { + backend: 'android-helper', + foregroundWindowPackage: 'com.google.android.apps.nexuslauncher', + }, + appBundleId: 'com.android.settings', + }), + }, + artifacts: createLocalArtifactAdapter(), + sessions: { + get: () => stored, + set: (record) => { + stored = record; + }, + }, + policy: localCommandPolicy(), + }); + + await device.capture.snapshot({ session: 'default' }); + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); + assert.equal(result.unchanged, undefined); +}); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index d99479a72c..65a6cf5ba1 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -96,7 +96,8 @@ export const snapshotCommand: RuntimeCommand< options, identity: { previousAppBundleId: capture.session?.appBundleId, - currentAppBundleId: capture.result.appBundleId ?? capture.session?.appBundleId, + // The app the header names, so an unchanged acknowledgement never spans a foreground switch. + currentAppBundleId: snapshotAppFields(capture).appBundleId, }, }); await runtime.sessions.set(nextSnapshotSession(options.session, capture)); diff --git a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts index 8564c35068..9a52b3d5f6 100644 --- a/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-content-recovery.test.ts @@ -318,7 +318,7 @@ test('names the active application window, not a stale one listed first or a sys window({ windowType: 1, packageName: 'com.android.settings', - text: 'Network & internet', + text: 'Network and internet', windowLayer: 1, }), window({ @@ -345,7 +345,7 @@ test('names no foreground app when only a framework window is active', () => { window({ windowType: 1, packageName: 'com.android.settings', - text: 'Network & internet', + text: 'Network and internet', windowLayer: 1, }), ]), From 03ef82ac26988fc8f83dad6a7206b6abf1fc2aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 22:31:04 +0100 Subject: [PATCH 7/7] fix(snapshot): keep main's identity for the unchanged acknowledgement Address review: comparing the header app with the session app disabled the unchanged acknowledgement for as long as another app stayed in front. The acknowledgement already requires the same tree as the last stored capture, which is the tree the header describes, so the identity check is back to main's. A test pins the acknowledgement under the foreign app's header. Co-Authored-By: Claude Opus 5.5 --- .../commands/capture/runtime/snapshot-app-identity.test.ts | 5 +++-- .../command-runtime/src/commands/capture/runtime/snapshot.ts | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts index dbc34c46db..7cba3cd10d 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-app-identity.test.ts @@ -74,7 +74,7 @@ test('an Android snapshot in a session with no app names the captured foreground assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); }); -test('a repeated snapshot of a foreign foreground app is not acknowledged as unchanged', async () => { +test('a repeated unchanged snapshot of a foreign foreground app is acknowledged under its header', async () => { const snapshot = makeSnapshotState([{ index: 0, depth: 0, type: 'TextView', label: 'Gmail' }], { backend: 'android', }); @@ -107,6 +107,7 @@ test('a repeated snapshot of a foreign foreground app is not acknowledged as unc await device.capture.snapshot({ session: 'default' }); const result = await device.capture.snapshot({ session: 'default' }); + // The acknowledgement compares against the last stored tree, the launcher's, which the header names. assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher'); - assert.equal(result.unchanged, undefined); + assert.ok(result.unchanged); }); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 65a6cf5ba1..d99479a72c 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -96,8 +96,7 @@ export const snapshotCommand: RuntimeCommand< options, identity: { previousAppBundleId: capture.session?.appBundleId, - // The app the header names, so an unchanged acknowledgement never spans a foreground switch. - currentAppBundleId: snapshotAppFields(capture).appBundleId, + currentAppBundleId: capture.result.appBundleId ?? capture.session?.appBundleId, }, }); await runtime.sessions.set(nextSnapshotSession(options.session, capture));