Skip to content
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter';
import { createAgentDevice } from '../../../runtime.ts';
import { localCommandPolicy } from '@agent-device/contracts/command-policy';
import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract';
import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';

async function snapshotWithForeground(
foregroundWindowPackage: string | undefined,
sessionApp: { appName: string; appBundleId: string } | undefined = {
appName: 'Settings',
appBundleId: 'com.android.settings',
},
) {
let stored: Parameters<CommandSessionStore['set']>[0] | undefined;
const device = createAgentDevice({
backend: {
platform: 'android',
captureSnapshot: async () => ({
snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'TextView', label: 'Gmail' }], {
backend: 'android',
}),
androidSnapshot: {
backend: 'android-helper',
...(foregroundWindowPackage ? { foregroundWindowPackage } : {}),
},
...sessionApp,
}),
},
artifacts: createLocalArtifactAdapter(),
sessions: {
get: () => undefined,
set: (record) => {
stored = record;
},
},
policy: localCommandPolicy(),
});
const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true });
return { result, stored };
}

test('an Android snapshot names the captured foreground app when it is not the session app', async () => {
const { result, stored } = await snapshotWithForeground('com.google.android.apps.nexuslauncher');

// A foreign app has no label of its own here, so the header shows its package.
assert.equal(result.appName, 'com.google.android.apps.nexuslauncher');
assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher');
assert.equal(stored?.appBundleId, 'com.android.settings');
});

test('an Android snapshot keeps the session app label when the session app is in front', async () => {
const { result } = await snapshotWithForeground('com.android.settings');

assert.equal(result.appName, 'Settings');
assert.equal(result.appBundleId, 'com.android.settings');
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
});

test('a snapshot without a foreground window fact names the session app', async () => {
const { result } = await snapshotWithForeground(undefined);

assert.equal(result.appName, 'Settings');
assert.equal(result.appBundleId, 'com.android.settings');
});

test('an Android snapshot in a session with no app names the captured foreground app', async () => {
const { result } = await snapshotWithForeground(
'com.google.android.apps.nexuslauncher',
undefined,
);

assert.equal(result.appName, 'com.google.android.apps.nexuslauncher');
assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher');
});

test('a repeated unchanged snapshot of a foreign foreground app is acknowledged under its header', async () => {
const snapshot = makeSnapshotState([{ index: 0, depth: 0, type: 'TextView', label: 'Gmail' }], {
backend: 'android',
});
let stored: Parameters<CommandSessionStore['set']>[0] | undefined = {
name: 'default',
appBundleId: 'com.android.settings',
};
const device = createAgentDevice({
backend: {
platform: 'android',
captureSnapshot: async () => ({
snapshot,
androidSnapshot: {
backend: 'android-helper',
foregroundWindowPackage: 'com.google.android.apps.nexuslauncher',
},
appBundleId: 'com.android.settings',
}),
},
artifacts: createLocalArtifactAdapter(),
sessions: {
get: () => stored,
set: (record) => {
stored = record;
},
},
policy: localCommandPolicy(),
});

await device.capture.snapshot({ session: 'default' });
const result = await device.capture.snapshot({ session: 'default' });

// The acknowledgement compares against the last stored tree, the launcher's, which the header names.
assert.equal(result.appBundleId, 'com.google.android.apps.nexuslauncher');
assert.ok(result.unchanged);
});
15 changes: 11 additions & 4 deletions packages/command-runtime/src/commands/capture/runtime/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -234,15 +234,22 @@ function nextSnapshotSession(
};
Comment thread
thymikee marked this conversation as resolved.
Resolved
}

/**
* The app the response header names: the foreground application window the Android snapshot
* helper captured, such as the launcher after a failed open, else the session app. The session
* keeps its app.
*/
function snapshotAppFields(capture: SnapshotCapture): {
appName?: string;
appBundleId?: string;
} {
const appName = capture.result.appName ?? capture.session?.appName;
const appBundleId = capture.result.appBundleId ?? capture.session?.appBundleId;
const sessionApp = capture.result.appBundleId ?? capture.session?.appBundleId;
const shownApp = capture.annotations.androidSnapshot?.foregroundWindowPackage ?? sessionApp;
const appName =
shownApp === sessionApp ? (capture.result.appName ?? capture.session?.appName) : shownApp;
Comment thread
thymikee marked this conversation as resolved.
Resolved
return {
...(appName || appBundleId ? { appName: appName ?? appBundleId } : {}),
...(appBundleId ? { appBundleId } : {}),
...(appName || shownApp ? { appName: appName ?? shownApp } : {}),
...(shownApp ? { appBundleId: shownApp } : {}),
};
}

Expand Down
5 changes: 5 additions & 0 deletions packages/contracts/src/snapshot-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ export type AndroidSnapshotBackendMetadata = {
cacheInvalidated?: boolean;
captureMode?: string;
systemSurfaceOnly?: boolean;
/**
* Package of the foreground application window the snapshot helper captured: active or focused,
* at the highest layer, and not owned by the system. Absent when the helper saw none.
*/
foregroundWindowPackage?: string;
windowCount?: number;
/**
* `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,88 @@ test('rejects system chrome inherited under an empty application window', () =>
expect(decision?.diagnostics.helperNonSystemMeaningfulNodeCount).toBe(0);
});

function foregroundWindowPackageOf(windows: string[]): string | undefined {
const result = classifyAndroidHelperContent(helperXml(windows), {
backend: 'android-helper',
nodeCount: windows.length * 2,
rootPresent: true,
windowCount: windows.length,
captureMode: 'interactive-windows',
});
return result.outcome === 'unusable' ? undefined : result.foregroundWindowPackage;
}

function window(params: {
windowType: number;
packageName: string;
text: string;
windowLayer: number;
windowActive?: boolean;
}): string {
const { text, ...windowAttributes } = params;
return node({
...windowAttributes,
className: 'android.widget.FrameLayout',
children: [
node({ text, packageName: params.packageName, className: 'android.widget.TextView' }),
],
});
}

test('names the active application window, not a stale one listed first or a system window above it', () => {
expect(
foregroundWindowPackageOf([
window({
windowType: 3,
packageName: 'com.android.systemui',
text: '12:00',
windowLayer: 9,
windowActive: true,
}),
window({
windowType: 2,
packageName: 'com.google.android.inputmethod.latin',
text: 'Space',
windowLayer: 8,
}),
// A stale session-app window Android kept after the transition: inactive, lower layer.
window({
windowType: 1,
packageName: 'com.android.settings',
text: 'Network and internet',
windowLayer: 1,
}),
window({
windowType: 1,
packageName: 'com.google.android.apps.nexuslauncher',
text: 'Gmail',
windowLayer: 2,
windowActive: true,
}),
]),
).toBe('com.google.android.apps.nexuslauncher');
});

test('names no foreground app when only a framework window is active', () => {
expect(
foregroundWindowPackageOf([
window({
windowType: 1,
packageName: 'android',
text: 'Allow?',
windowLayer: 3,
windowActive: true,
}),
window({
windowType: 1,
packageName: 'com.android.settings',
text: 'Network and internet',
windowLayer: 1,
}),
]),
).toBeUndefined();
});

test('returns a meaningful active system surface (notification shade) faithfully', () => {
const result = classifyAndroidHelperContent(
helperXml([
Expand Down Expand Up @@ -526,12 +608,14 @@ function node(options: {
packageName: string;
className: string;
windowType?: number;
windowLayer?: number;
windowActive?: boolean;
/** Nested children, so chrome leaves can sit inside their status/nav-bar container as on a device. */
children?: string[];
}): string {
const windowAttrs =
(options.windowType === undefined ? '' : ` window-type="${options.windowType}"`) +
(options.windowLayer === undefined ? '' : ` window-layer="${options.windowLayer}"`) +
(options.windowActive === undefined ? '' : ` window-active="${options.windowActive}"`);
const open = `<node index="0"${windowAttrs} text="${options.text ?? ''}" resource-id="${options.resourceId ?? ''}" class="${options.className}" package="${options.packageName}" visible-to-user="true" enabled="true" bounds="[0,0][100,100]"`;
return options.children === undefined
Expand Down
8 changes: 3 additions & 5 deletions packages/platform-android/src/__tests__/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -889,11 +889,9 @@ test('snapshotAndroid keeps helper output when application and system windows ar
appBundleId: 'io.example.fabric',
});

assert.equal(result.androidSnapshot.backend, 'android-helper');
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
const { backend, foregroundWindowPackage } = result.androidSnapshot;
assert.deepEqual([backend, foregroundWindowPackage], ['android-helper', 'io.example.fabric']);
assert.ok(result.nodes.some((node) => node.label === 'Fabric dashboard'));
});

test('snapshotAndroid emits helper failure diagnostics', async () => {
Expand Down
31 changes: 29 additions & 2 deletions packages/platform-android/src/snapshot-content-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { classifyAndroidAlertIdentifier } from './alert-detection.ts';
import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts';
import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts';
import { ANDROID_WINDOW_TYPE_APPLICATION } from './ui-hierarchy-node.ts';
import { isAndroidForegroundWindow } from './ui-hierarchy-visibility.ts';

const MAX_REPORTED_WINDOW_TYPES = 8;
const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2;
Expand Down Expand Up @@ -44,10 +45,19 @@ export function classifyAndroidHelperContent(
xml: string,
metadata: AndroidSnapshotBackendMetadata,
options: { foregroundAppPackage?: string } = {},
): AndroidHelperContentClassification {
): AndroidHelperContentClassification & { foregroundWindowPackage?: string } {
if (metadata.backend !== 'android-helper') return { outcome: 'ok' };

const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage);
const content = classifyAndroidHelperSummary(summary, metadata);
return summary.foregroundWindow
? { ...content, foregroundWindowPackage: summary.foregroundWindow.packageName }
: content;
}

function classifyAndroidHelperSummary(
summary: AndroidHelperXmlSummary,
metadata: AndroidSnapshotBackendMetadata,
): AndroidHelperContentClassification {
if (isEmptyHelperOutput(summary, metadata)) {
return unusable(
summary,
Expand Down Expand Up @@ -201,6 +211,11 @@ type AndroidHelperXmlSummary = {
foregroundAppPackage?: string;
foregroundAppMeaningfulNodeCount?: number;
windowTypes: number[];
/**
* The non-system application window the presentation keeps as foreground: active or focused, at
* the highest layer. A stale application window under it is not foreground.
*/
foregroundWindow?: { packageName: string; layer: number };
};

type AndroidHelperXmlSummaryState = Omit<AndroidHelperXmlSummary, 'windowTypes'> & {
Expand Down Expand Up @@ -280,9 +295,20 @@ function recordAndroidHelperWindowNode(
if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) {
summary.applicationWindowRootCount += 1;
if (summary.currentWindowSystemOwned) summary.systemApplicationWindowRootCount += 1;
else recordAndroidForegroundWindow(summary, node);
}
}

function recordAndroidForegroundWindow(
summary: AndroidHelperXmlSummaryState,
node: AndroidUiNodeMetadata,
): void {
if (node.packageName === null || !isAndroidForegroundWindow(node)) return;
const layer = node.windowLayer ?? -1;
if (summary.foregroundWindow && summary.foregroundWindow.layer >= layer) return;
summary.foregroundWindow = { packageName: node.packageName, layer };
}

function recordAndroidHelperMeaningfulNode(
summary: AndroidHelperXmlSummaryState,
node: AndroidUiNodeMetadata,
Expand Down Expand Up @@ -370,6 +396,7 @@ function finalizeAndroidHelperXmlSummary(
}
: {}),
windowTypes: [...summary.windowTypes].sort((a, b) => a - b).slice(0, MAX_REPORTED_WINDOW_TYPES),
...(summary.foregroundWindow ? { foregroundWindow: summary.foregroundWindow } : {}),
};
}

Expand Down
5 changes: 5 additions & 0 deletions packages/platform-android/src/snapshot-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ export type AndroidSnapshotBackendMetadata = {
cacheInvalidated?: boolean;
captureMode?: AndroidSnapshotCaptureMode;
systemSurfaceOnly?: boolean;
/**
* Package of the foreground application window the snapshot helper captured: active or focused,
* at the highest layer, and not owned by the system. Absent when the helper saw none.
*/
foregroundWindowPackage?: string;
windowCount?: number;
/**
* `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because
Expand Down
10 changes: 7 additions & 3 deletions packages/platform-android/src/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -639,9 +639,13 @@ async function captureAndroidHelperContentAttempt(params: {
capture: {
xml: helperCapture.xml,
helperMetadata: helperCapture.helperMetadata,
metadata: systemSurfaceOnly
? { ...helperCapture.metadata, systemSurfaceOnly: true }
: helperCapture.metadata,
metadata: {
...helperCapture.metadata,
...(systemSurfaceOnly ? { systemSurfaceOnly: true } : {}),
...(content.foregroundWindowPackage
? { foregroundWindowPackage: content.foregroundWindowPackage }
: {}),
},
},
};
}
Expand Down
5 changes: 4 additions & 1 deletion packages/platform-android/src/ui-hierarchy-visibility.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ function isAndroidApplicationWindow(node: AndroidNode): boolean {
return node.windowType === ANDROID_WINDOW_TYPE_APPLICATION;
}

function isAndroidForegroundWindow(node: AndroidNode): boolean {
/** A window the user is in: active or focused. Shared with the snapshot header's foreground app. */
export function isAndroidForegroundWindow(
node: Pick<AndroidNode, 'windowActive' | 'windowFocused'>,
): boolean {
return node.windowActive === true || node.windowFocused === true;
}
1 change: 1 addition & 0 deletions website/docs/docs/snapshots.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue
now, and hints that point to hidden list content.
- Use `snapshot -i` by default in agent loops.
- On Android, `snapshot -i` also keeps the title and message of a standard alert dialog, so you can see what its buttons confirm.
- When the Android snapshot helper captures the tree, the `Page:` and `App:` lines (`appName` and `appBundleId` in `--json`) name the foreground application window: active or focused, at the highest layer, and not a system UI or keyboard window. After a failed `open`, for example, they name the launcher. An app other than the session app shows its package name in both lines. `androidSnapshot.foregroundWindowPackage` in `--json` carries it, and the session keeps its app. The UIAutomator fallback names the session app.
- On Android, repeating an unfiltered snapshot when nothing on screen changed (content or bounds) returns a short acknowledgement instead of the tree. `-i`, `-d`, `-s`, `--json`, and `--raw` always print full output. Add `--force-full` to get the full tree anyway.
- The default text output is a compact view for planning and targeting actions. It lists visible elements first and may collapse helper and accessibility noise. Use `--raw` or `--json` when you need the full provider tree. Keep `--raw` for troubleshooting, such as when you need the full off-screen tree that summaries abbreviate.
- Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`.
Expand Down
Loading