diff --git a/packages/driver-mobile-use/src/driver.ts b/packages/driver-mobile-use/src/driver.ts index f788f36..431e08b 100644 --- a/packages/driver-mobile-use/src/driver.ts +++ b/packages/driver-mobile-use/src/driver.ts @@ -3,6 +3,7 @@ import { stat } from 'node:fs/promises'; import { basename } from 'node:path'; import createDebug from 'debug'; import type { + AnimationScales, AppInfo, ConnectionConfig, DeviceInfo, @@ -399,6 +400,18 @@ export class MobileUseDriver implements MobilewrightDriver { return result; } + async getAnimationScales(): Promise { + return this.call('device.io.animation-scales.get'); + } + + async setAnimationScales(scales: AnimationScales): Promise { + await this.call('device.io.animation-scales.set', { + window: scales.window, + transition: scales.transition, + animator: scales.animator, + }); + } + // ─── Apps ─────────────────────────────────────────────────── async launchApp(bundleId: string, opts?: LaunchOptions): Promise { diff --git a/packages/driver-mobilecli/src/driver.ts b/packages/driver-mobilecli/src/driver.ts index cf03d07..607c291 100644 --- a/packages/driver-mobilecli/src/driver.ts +++ b/packages/driver-mobilecli/src/driver.ts @@ -1,6 +1,7 @@ import createDebug from 'debug'; import { execFileSync } from 'node:child_process'; import type { + AnimationScales, AppInfo, ConnectionConfig, DeviceInfo, @@ -337,6 +338,18 @@ export class MobilecliDriver implements MobilewrightDriver { await this.call('device.io.orientation.set', { orientation }); } + async getAnimationScales(): Promise { + return this.call('device.io.animation-scales.get'); + } + + async setAnimationScales(scales: AnimationScales): Promise { + await this.call('device.io.animation-scales.set', { + window: scales.window, + transition: scales.transition, + animator: scales.animator, + }); + } + // ─── Recording Operations ───────────────────────────────────── async startRecording(opts: RecordingOptions): Promise { diff --git a/packages/mobilewright-core/src/device.ts b/packages/mobilewright-core/src/device.ts index 1929182..f45b294 100644 --- a/packages/mobilewright-core/src/device.ts +++ b/packages/mobilewright-core/src/device.ts @@ -1,4 +1,5 @@ import type { + AnimationScales, AppInfo, ConnectionConfig, LaunchOptions, @@ -132,4 +133,23 @@ export class Device { async stopRecording(): Promise { return this.driver.stopRecording(); } + + // ─── Android animations ───────────────────────────────────────── + // uiautomator dump fails on continuously animated screens (e.g. a + // ride-searching page with a SurfaceView/TextureView animation that + // never settles). Disabling the three system animation scales before + // calling getViewHierarchy() lets the dump succeed. + // + // disableAnimations() saves and returns the current scales so the + // caller can restore the exact originals via enableAnimations(saved). + + async disableAnimations(): Promise { + const saved = await this.driver.getAnimationScales(); + await this.driver.setAnimationScales({ window: 0, transition: 0, animator: 0 }); + return saved; + } + + async enableAnimations(scales: AnimationScales = { window: 1, transition: 1, animator: 1 }): Promise { + await this.driver.setAnimationScales(scales); + } } diff --git a/packages/mobilewright-core/src/expect.test.ts b/packages/mobilewright-core/src/expect.test.ts index a5b258e..af8bd2a 100644 --- a/packages/mobilewright-core/src/expect.test.ts +++ b/packages/mobilewright-core/src/expect.test.ts @@ -84,6 +84,8 @@ function createMockDriver(hierarchy: ViewNode[]): MobilewrightDriver & { _tracke openUrl: async (...args: any[]) => { tracker.openUrlCalls.push(args); }, startRecording: async () => {}, stopRecording: async () => ({}), + getAnimationScales: async () => ({ window: 1, transition: 1, animator: 1 }), + setAnimationScales: async () => {}, }; } diff --git a/packages/mobilewright-core/src/locator.test.ts b/packages/mobilewright-core/src/locator.test.ts index 6ba2ae9..53a1a05 100644 --- a/packages/mobilewright-core/src/locator.test.ts +++ b/packages/mobilewright-core/src/locator.test.ts @@ -83,6 +83,8 @@ function createMockDriver(hierarchy: ViewNode[]): MobilewrightDriver & { _tracke openUrl: async (...args: any[]) => { tracker.openUrlCalls.push(args); }, startRecording: async () => {}, stopRecording: async () => ({}), + getAnimationScales: async () => ({ window: 1, transition: 1, animator: 1 }), + setAnimationScales: async () => {}, }; } diff --git a/packages/mobilewright/src/cli.ts b/packages/mobilewright/src/cli.ts index 41eb6dc..9406af7 100644 --- a/packages/mobilewright/src/cli.ts +++ b/packages/mobilewright/src/cli.ts @@ -13,6 +13,8 @@ import { MobilecliDriver, DEFAULT_URL, resolveMobilecliBinary } from '@mobilewri import { ensureMobilecliReachable } from './server.js'; import { loadConfig } from './config.js'; import { gatherChecks, renderTerminal, renderJSON } from './commands/doctor.js'; +import { runInspect, type InspectOptions } from './commands/inspect.js'; +import { runInspectUI } from './commands/inspect-ui.js'; import { brandReport } from './reporter.js'; import { telemetry } from './telemetry.js'; @@ -248,6 +250,23 @@ program } }); +// ── inspect ─────────────────────────────────────────────────────── +program + .command('inspect') + .description('dump the live accessibility tree of a connected device') + .option('-d, --device ', 'device ID (run "mobilewright devices" to list)') + .option('--url ', 'mobilecli server URL', DEFAULT_URL) + .option('--json', 'output raw ViewNode[] JSON instead of the terminal tree') + .option('--ui', 'open an interactive browser UI with auto-refresh and locator copy') + .option('--disable-animations', 'disable Android system animations before dumping (fixes uiautomator failures on animated screens)') + .action(async (opts: InspectOptions & { ui?: boolean }) => { + if (opts.ui) { + await runInspectUI(opts); + } else { + await runInspect(opts); + } + }); + // ── install ─────────────────────────────────────────────────────── program .command('install') diff --git a/packages/mobilewright/src/commands/inspect-ui.ts b/packages/mobilewright/src/commands/inspect-ui.ts new file mode 100644 index 0000000..f8af9e6 --- /dev/null +++ b/packages/mobilewright/src/commands/inspect-ui.ts @@ -0,0 +1,462 @@ +/** + * mobilewright inspect --ui + * + * Serves an interactive browser UI that renders the live accessibility tree + * from a connected device. Supports manual refresh and auto-refresh with a + * configurable interval. Clicking a node shows its properties and ready-to-use + * locator suggestions with a one-click copy button. + * + * Usage: npx mobilewright inspect --ui + */ +import { createServer } from 'node:http'; +import { exec } from 'node:child_process'; +import { platform as osPlatform } from 'node:os'; +import { MobilecliDriver, DEFAULT_URL } from '@mobilewright/driver-mobilecli'; +import type { AnimationScales } from '@mobilewright/protocol'; +import type { MobilewrightConfig } from '../config.js'; +import { ensureMobilecliReachable } from '../server.js'; +import { loadConfig } from '../config.js'; + +const PORT = 9325; + +export interface InspectUIOptions { + device?: string; + url?: string; + disableAnimations?: boolean; +} + +// ─── Device resolution ──────────────────────────────────────────────────────── + +async function resolveDeviceId( + explicit: string | undefined, + driver: MobilecliDriver, + config: MobilewrightConfig, +): Promise { + if (explicit) return explicit; + if (config.deviceId) return config.deviceId; + + const devices = await driver.listDevices(); + const online = devices.filter(d => d.state === 'online'); + + if (online.length === 0) { + console.error("No online devices found. Specify one with --device , or try 'mobilewright doctor'."); + process.exit(1); + } + + if (config.deviceName) { + const pattern = config.deviceName instanceof RegExp + ? config.deviceName + : new RegExp(config.deviceName); + const matched = online.filter(d => pattern.test(d.name)); + if (matched.length > 0) return matched[0].id; + } + + if (online.length > 1) { + console.error('Multiple devices found. Specify one with --device :'); + for (const d of online) console.error(` ${d.id} ${d.name}`); + process.exit(1); + } + + return online[0].id; +} + +// ─── Persistent device connection ──────────────────────────────────────────── + +function makeConnection(url: string, platform: 'ios' | 'android', deviceId: string) { + let driver: MobilecliDriver | null = null; + let connected = false; + + async function ensureConnected(): Promise { + if (!connected) { + driver = new MobilecliDriver({ url }); + await driver.connect({ platform, deviceId, url }); + connected = true; + } + } + + async function getTree() { + try { + await ensureConnected(); + return await driver!.getViewHierarchy(); + } catch { + connected = false; + await ensureConnected(); + return await driver!.getViewHierarchy(); + } + } + + async function disconnect(): Promise { + if (driver && connected) { + await driver.disconnect().catch(() => {}); + connected = false; + } + } + + return { getTree, disconnect }; +} + +// ─── Browser open ───────────────────────────────────────────────────────────── + +function openBrowser(url: string): void { + const cmd = osPlatform() === 'win32' ? `start ${url}` + : osPlatform() === 'darwin' ? `open ${url}` + : `xdg-open ${url}`; + exec(cmd); +} + +// ─── HTML ───────────────────────────────────────────────────────────────────── + +const HTML = ` + + + + mobilewright inspect + + + +
+ mobilewright inspect +
+ + + + +
+
+
+
+

Click a node to inspect it

+
+ + +`; + +// ─── Main ───────────────────────────────────────────────────────────────────── + +export async function runInspectUI(opts: InspectUIOptions): Promise { + const config = await loadConfig(); + const url = opts.url ?? DEFAULT_URL; + const platform = (config.platform ?? 'android') as 'ios' | 'android'; + + const { serverProcess } = await ensureMobilecliReachable(url, { autoStart: true }); + + const driver = new MobilecliDriver({ url }); + const deviceId = await resolveDeviceId(opts.device, driver, config); + + let savedScales: AnimationScales | undefined; + if (opts.disableAnimations) { + const tmpDriver = new MobilecliDriver({ url }); + await tmpDriver.connect({ platform, deviceId, url }); + savedScales = await tmpDriver.getAnimationScales(); + await tmpDriver.setAnimationScales({ window: 0, transition: 0, animator: 0 }); + await tmpDriver.disconnect(); + console.log('Android animations disabled.'); + } + + const conn = makeConnection(url, platform, deviceId); + + const httpServer = createServer(async (req, res) => { + if (req.url === '/api/tree') { + try { + const tree = await conn.getTree(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(tree)); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: String(err) })); + } + } else { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(HTML); + } + }); + + httpServer.listen(PORT, () => { + const inspectUrl = `http://localhost:${PORT}`; + console.log(`\nInspector running at ${inspectUrl}\n`); + console.log('Press Ctrl+C to stop.\n'); + openBrowser(inspectUrl); + }); + + process.on('SIGINT', async () => { + httpServer.close(); + await conn.disconnect(); + if (savedScales) { + try { + const tmpDriver = new MobilecliDriver({ url }); + await tmpDriver.connect({ platform, deviceId, url }); + await tmpDriver.setAnimationScales(savedScales); + await tmpDriver.disconnect(); + console.log('\nAndroid animations restored.'); + } catch { + // best-effort restore + } + } + if (serverProcess) await serverProcess.kill(); + process.exit(0); + }); +} diff --git a/packages/mobilewright/src/commands/inspect.ts b/packages/mobilewright/src/commands/inspect.ts new file mode 100644 index 0000000..f8fe99f --- /dev/null +++ b/packages/mobilewright/src/commands/inspect.ts @@ -0,0 +1,145 @@ +/** + * mobilewright inspect + * + * Dumps the live accessibility tree from a connected device to the terminal. + * Useful for figuring out which locator to use when writing tests — run it + * with the app open on the screen you want to inspect. + * + * Pass --json to get raw ViewNode[] JSON suitable for piping to jq or saving + * to a file for diffing between two app states. + */ +import { MobilecliDriver, DEFAULT_URL } from '@mobilewright/driver-mobilecli'; +import type { AnimationScales, ViewNode } from '@mobilewright/protocol'; +import { ensureMobilecliReachable } from '../server.js'; +import { loadConfig } from '../config.js'; + +// ─── ANSI helpers ───────────────────────────────────────────────────────────── + +const C = { + reset: '\x1b[0m', + bold: '\x1b[1m', + dim: '\x1b[2m', + cyan: '\x1b[36m', + green: '\x1b[32m', + gray: '\x1b[90m', + white: '\x1b[97m', +} as const; + +// ─── Tree renderer ──────────────────────────────────────────────────────────── + +function renderNode(node: ViewNode, prefix: string, isLast: boolean): void { + const connector = isLast ? '└── ' : '├── '; + const childPrefix = prefix + (isLast ? ' ' : '│ '); + + let line = `${C.gray}${prefix}${connector}${C.reset}`; + line += `${C.cyan}${node.type}${C.reset}`; + + if (node.label) line += ` ${C.white}label=${JSON.stringify(node.label)}${C.reset}`; + if (node.text && node.text !== node.label) line += ` ${C.white}"${node.text}"${C.reset}`; + if (node.placeholder) line += ` ${C.dim}placeholder=${JSON.stringify(node.placeholder)}${C.reset}`; + if (node.identifier) line += ` ${C.dim}id=${JSON.stringify(node.identifier)}${C.reset}`; + + line += node.isVisible + ? ` ${C.green}[visible]${C.reset}` + : ` ${C.gray}[hidden]${C.reset}`; + + process.stdout.write(line + '\n'); + + const children = node.children ?? []; + for (let i = 0; i < children.length; i++) { + renderNode(children[i], childPrefix, i === children.length - 1); + } +} + +function renderTree(nodes: ViewNode[]): void { + for (let i = 0; i < nodes.length; i++) { + renderNode(nodes[i], '', i === nodes.length - 1); + } +} + +function countNodes(nodes: ViewNode[]): number { + return nodes.reduce((acc, n) => acc + 1 + countNodes(n.children ?? []), 0); +} + +// ─── Device resolution ──────────────────────────────────────────────────────── + +async function resolveDeviceId(explicit: string | undefined, driver: MobilecliDriver): Promise { + if (explicit) return explicit; + + const config = await loadConfig(); + if (config.deviceId) return config.deviceId; + + const devices = await driver.listDevices(); + const online = devices.filter(d => d.state === 'online'); + + if (online.length === 0) { + console.error('No online devices found. Specify one with --device , or try \'mobilewright doctor\'.'); + process.exit(1); + } + + if (config.deviceName) { + const pattern = config.deviceName instanceof RegExp + ? config.deviceName + : new RegExp(config.deviceName); + const matched = online.filter(d => pattern.test(d.name)); + if (matched.length > 0) return matched[0].id; + } + + if (online.length > 1) { + console.error('Multiple devices found. Specify one with --device :'); + for (const d of online) console.error(` ${d.id} ${d.name}`); + process.exit(1); + } + + return online[0].id; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +export interface InspectOptions { + device?: string; + url?: string; + json?: boolean; + disableAnimations?: boolean; +} + +export async function runInspect(opts: InspectOptions): Promise { + const config = await loadConfig(); + const url = opts.url ?? DEFAULT_URL; + const platform = config.platform ?? 'android'; + + const { serverProcess } = await ensureMobilecliReachable(url, { autoStart: true }); + try { + const driver = new MobilecliDriver({ url }); + const deviceId = await resolveDeviceId(opts.device, driver); + + await driver.connect({ platform, deviceId, url }); + let savedScales: AnimationScales | undefined; + let tree: Awaited>; + try { + if (opts.disableAnimations) { + savedScales = await driver.getAnimationScales(); + await driver.setAnimationScales({ window: 0, transition: 0, animator: 0 }); + } + tree = await driver.getViewHierarchy(); + } finally { + if (savedScales) { + await driver.setAnimationScales(savedScales).catch(() => {}); + } + await driver.disconnect(); + } + + if (opts.json) { + console.log(JSON.stringify(tree, null, 2)); + return; + } + + const total = countNodes(tree); + const label = config.deviceName?.toString().replace(/^\/|\/$/g, '') ?? deviceId; + console.log(`\n${C.bold}${label}${C.reset} ${C.dim}${total} nodes${C.reset}\n`); + renderTree(tree); + console.log(); + } finally { + if (serverProcess) await serverProcess.kill(); + } +} diff --git a/packages/mobilewright/src/config.ts b/packages/mobilewright/src/config.ts index 7c00ced..2640d65 100644 --- a/packages/mobilewright/src/config.ts +++ b/packages/mobilewright/src/config.ts @@ -69,6 +69,8 @@ export interface MobilewrightConfig { installApps?: string | string[]; /** Automatically launch the app after connecting. Default: true. */ autoAppLaunch?: boolean; + /** Attach the accessibility tree as JSON to the test report on failure. Default: false. */ + saveTreeOnFailure?: boolean; /** mobilecli server URL (use for remote servers). */ url?: string; /** Path to mobilecli binary (if not on PATH). */ diff --git a/packages/protocol/src/driver.ts b/packages/protocol/src/driver.ts index 581ec3d..2717df3 100644 --- a/packages/protocol/src/driver.ts +++ b/packages/protocol/src/driver.ts @@ -1,4 +1,5 @@ import type { + AnimationScales, AppInfo, ConnectionConfig, DeviceInfo, @@ -39,6 +40,8 @@ export interface MobilewrightDriver { getScreenSize(): Promise; getOrientation(): Promise; setOrientation(orientation: Orientation): Promise; + getAnimationScales(): Promise; + setAnimationScales(scales: AnimationScales): Promise; // Apps launchApp(bundleId: string, opts?: LaunchOptions): Promise; diff --git a/packages/protocol/src/types.ts b/packages/protocol/src/types.ts index 60db3bd..b767500 100644 --- a/packages/protocol/src/types.ts +++ b/packages/protocol/src/types.ts @@ -157,6 +157,16 @@ export interface RecordingOptions { timeLimit?: number; } +// ─── Animation Scales ─────────────────────────────────────────── +// Android only. Controls the three global animation scale settings that +// uiautomator uses to determine when a screen is stable enough to dump. + +export interface AnimationScales { + window: number; + transition: number; + animator: number; +} + export interface RecordingResult { /** Operation status */ status?: string; diff --git a/packages/test/src/fixtures.ts b/packages/test/src/fixtures.ts index 56ded3e..bd4af2c 100644 --- a/packages/test/src/fixtures.ts +++ b/packages/test/src/fixtures.ts @@ -33,6 +33,7 @@ type MobilewrightTestFixtures = { bundleId: string | undefined; platform: 'ios' | 'android' | undefined; deviceName: RegExp | undefined; + saveTreeOnFailure: boolean; device: Device; }; @@ -53,6 +54,11 @@ export const test = base.extend({ platform: [undefined, { option: true }], deviceName: [undefined, { option: true }], + saveTreeOnFailure: [async ({}, use, testInfo) => { + const config = await loadConfig(process.cwd(), testInfo.config.configFile); + await use(config.saveTreeOnFailure ?? false); + }, { option: true }], + device: async ({ platform, deviceName, bundleId }, use, testInfo) => { const config = await loadConfig(process.cwd(), testInfo.config.configFile); const merged = { @@ -104,7 +110,7 @@ export const test = base.extend({ } }, - screen: async ({ device, video }, use, testInfo) => { + screen: async ({ device, video, saveTreeOnFailure }, use, testInfo) => { const videoMode = typeof video === 'object' ? video.mode : video; const shouldRecord = videoMode === 'on' || videoMode === 'retain-on-failure'; const videoPath = shouldRecord @@ -145,6 +151,17 @@ export const test = base.extend({ } catch { // device may be disconnected } + if (saveTreeOnFailure) { + try { + const tree = await device.screen.viewTree(); + await testInfo.attach('view-tree-on-failure', { + body: Buffer.from(JSON.stringify(tree, null, 2)), + contentType: 'application/json', + }); + } catch { + // device may be disconnected + } + } } }, });