diff --git a/docs/adr/0014-session-ref-frame-lifetime.md b/docs/adr/0014-session-ref-frame-lifetime.md index d375affb46..4e6f129ff3 100644 --- a/docs/adr/0014-session-ref-frame-lifetime.md +++ b/docs/adr/0014-session-ref-frame-lifetime.md @@ -158,6 +158,7 @@ These outputs are partial external publications: - Snapshot diffs. - Digests that omit the namespace. - `find`. +- `audit` finding refs. - Settled diffs/tails. - Replay divergence screens. @@ -400,7 +401,7 @@ Every other one-target constraint in this decision stays unchanged. ### Partial issuance and MCP -`find`, settled diffs/tails, and replay divergence screens publish only the refs that they actually return. +`find`, `audit`, settled diffs/tails, and replay divergence screens publish only the refs that they actually return. They must not call a whole-frame "refs issued" marker. Read-only `find` may partially publish its returned ref. diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 581081453f..736395ee3e 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -270,6 +270,10 @@ "types": "./src/snapshot/scroll-edge-state.ts", "default": "./src/snapshot/scroll-edge-state.ts" }, + "./snapshot-audit": { + "types": "./src/snapshot/snapshot-audit.ts", + "default": "./src/snapshot/snapshot-audit.ts" + }, "./snapshot-baseline": { "types": "./src/snapshot-baseline.ts", "default": "./src/snapshot-baseline.ts" diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts new file mode 100644 index 0000000000..a5076bf0ef --- /dev/null +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, test } from 'vitest'; +import type { AuditContext, AuditFinding } from '@agent-device/contracts/capture'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { auditSnapshot } from '../snapshot-audit.ts'; + +const IOS: AuditContext = { platform: 'ios', truncated: false }; +const ANDROID: AuditContext = { platform: 'android', pixelDensity: 2, truncated: false }; + +function node(index: number, fields: Partial = {}): SnapshotNode { + return { + index, + ref: `e${index}`, + type: 'Button', + hittable: true, + rect: { x: 0, y: index * 100, width: 100, height: 60 }, + ...fields, + }; +} + +function audit(nodes: SnapshotNode[], context: AuditContext = IOS) { + return auditSnapshot(nodes, context); +} + +function summarize(findings: AuditFinding[]): string[] { + return findings.map((finding) => `${finding.kind} ${finding.ref}`); +} + +describe('small-target', () => { + test('reports a control below 44pt on iOS with its size and the recommendation', () => { + const { findings } = audit([ + node(1, { label: 'Menu', rect: { x: 0, y: 0, width: 30, height: 44 } }), + ]); + expect(findings).toEqual([ + expect.objectContaining({ + kind: 'small-target', + severity: 'warning', + ref: '@e1', + detail: { unit: 'pt', width: 30, height: 44, recommendedSize: 44 }, + }), + ]); + }); + + test('accepts a 44pt control that float layout reports as 43.67', () => { + const { findings } = audit([ + node(1, { label: 'OK', rect: { x: 0, y: 0, width: 43.67, height: 44 } }), + ]); + expect(findings).toEqual([]); + }); + + test('divides Android pixels by the captured density before comparing with 48dp', () => { + const small = { x: 0, y: 0, width: 80, height: 100 }; + expect(summarize(audit([node(1, { label: 'A', rect: small })], ANDROID).findings)).toEqual([ + 'small-target @e1', + ]); + // 100px is 100dp at density 1, so the same rect passes there. + expect( + audit([node(1, { label: 'A', rect: small })], { + platform: 'android', + pixelDensity: 1, + truncated: false, + }).findings, + ).toEqual([]); + }); + + test('skips the check and says why when Android density is unknown', () => { + const result = audit([node(1, { label: 'A', rect: { x: 0, y: 0, width: 10, height: 10 } })], { + platform: 'android', + truncated: false, + }); + expect(result.findings).toEqual([]); + expect(result.limitations).toEqual([expect.stringContaining('did not report display density')]); + }); + + test('skips the check on a TV target', () => { + const result = audit([node(1, { label: 'A', rect: { x: 0, y: 0, width: 10, height: 10 } })], { + platform: 'ios', + target: 'tv', + truncated: false, + }); + expect(result.findings).toEqual([]); + expect(result.limitations).toEqual([expect.stringContaining('tv targets')]); + }); + + test('ignores a disabled control, an offscreen control, and a keyboard key', () => { + const tiny = { x: 0, y: 0, width: 20, height: 20 }; + const { findings } = audit([ + node(1, { label: 'Off', enabled: false, rect: tiny }), + node(2, { label: 'Away', hittable: false, rect: tiny }), + node(3, { label: 'q', keyboardOwned: true, rect: tiny }), + ]); + expect(findings).toEqual([]); + }); + + test('ignores a link inside a text container, but not a link beside text or at the root', () => { + const tiny = { x: 0, y: 0, width: 60, height: 18 }; + const inText = audit([ + node(0, { type: 'StaticText', label: 'Read the terms', hittable: false }), + node(1, { parentIndex: 0, type: 'Link', label: 'terms', rect: tiny }), + ]); + expect(inText.findings).toEqual([]); + + const besideText = audit([ + node(0, { type: 'Other', hittable: false }), + node(1, { parentIndex: 0, type: 'StaticText', label: 'Read the' }), + node(2, { parentIndex: 0, type: 'Link', label: 'terms', rect: tiny }), + ]); + expect(summarize(besideText.findings)).toEqual(['small-target @e2']); + + const atRoot = audit([ + node(0, { type: 'StaticText', label: 'Title' }), + node(1, { type: 'Link', label: 'terms', rect: tiny }), + ]); + expect(summarize(atRoot.findings)).toEqual(['small-target @e1']); + }); +}); + +describe('missing-name', () => { + test('reports a button whose only text is an identifier or a value', () => { + const { findings } = audit([ + node(1, { identifier: 'menu-button' }), + node(2, { value: 'On' }), + node(3, { label: 'com.example:id/menu' }), + ]); + expect(summarize(findings)).toEqual([ + 'missing-name @e1', + 'missing-name @e2', + 'missing-name @e3', + ]); + }); + + test('accepts a label, an Android content description, or a named descendant', () => { + const { findings } = audit([ + node(1, { label: 'Save' }), + node(5, { label: 'Dictate', identifier: 'Dictate' }), + node(2, { contentDescription: 'Close' }), + node(3), + node(4, { parentIndex: 3, type: 'StaticText', label: 'Next' }), + ]); + expect(findings).toEqual([]); + }); + + test('does not treat typed field content as its name, but accepts a placeholder or hint label', () => { + const { findings } = audit([ + node(1, { type: 'TextField', label: 'hello', value: 'hello' }), + node(2, { type: 'TextField', value: 'hello', placeholder: 'Email' }), + node(3, { type: 'SearchField', label: 'Search', value: 'Search', placeholder: 'Search' }), + ]); + expect(summarize(findings)).toEqual(['missing-name @e1']); + }); + + test('reports an unnamed image button but not a decorative image', () => { + const { findings } = audit([ + node(1, { type: 'android.widget.ImageButton' }), + node(2, { type: 'Image' }), + node(3, { type: 'android.widget.ImageView' }), + ]); + expect(summarize(findings)).toEqual(['missing-name @e1']); + }); + + test('does not read a container class as a control', () => { + const { findings } = audit([ + node(1, { type: 'android.widget.RadioGroup' }), + node(2, { type: 'android.widget.TabLayout' }), + ]); + expect(findings).toEqual([]); + }); +}); + +describe('duplicate wrappers', () => { + const small = { x: 0, y: 0, width: 20, height: 20 }; + + test('collapses a control that is the only child of a same-role control that reports the same kinds', () => { + const { findings } = audit([ + node(1, { rect: small }), + node(2, { parentIndex: 1, rect: small }), + ]); + expect(summarize(findings)).toEqual(['small-target @e1', 'missing-name @e1']); + }); + + test('keeps the child when the parent has a sibling child or another role', () => { + const withSibling = audit([ + node(1, { label: 'A', rect: small }), + node(2, { parentIndex: 1, label: 'B', rect: small }), + node(3, { parentIndex: 1, type: 'StaticText', label: 'x' }), + ]); + expect(summarize(withSibling.findings)).toEqual(['small-target @e1', 'small-target @e2']); + + const otherRole = audit([ + node(1, { label: 'A', rect: small }), + node(2, { parentIndex: 1, type: 'Switch', label: 'B', rect: small }), + ]); + expect(summarize(otherRole.findings)).toEqual(['small-target @e1', 'small-target @e2']); + }); + + test('keeps the child when the parent does not report that kind', () => { + const disabledParent = audit([ + node(1, { label: 'A', enabled: false, rect: small }), + node(2, { parentIndex: 1, label: 'B', rect: small }), + ]); + expect(summarize(disabledParent.findings)).toEqual(['small-target @e2']); + + const largeParent = audit([ + node(1, { label: 'A', rect: { x: 0, y: 0, width: 100, height: 100 } }), + node(2, { parentIndex: 1, label: 'B', rect: small }), + ]); + expect(summarize(largeParent.findings)).toEqual(['small-target @e2']); + + const refLessParent = audit([ + node(1, { ref: undefined, label: 'A', rect: small }), + node(2, { parentIndex: 1, label: 'B', rect: small }), + ]); + expect(summarize(refLessParent.findings)).toEqual(['small-target @e2']); + }); + + test('does not collapse unrelated controls that share a rect', () => { + const { findings } = audit([ + node(1, { label: 'A', rect: small }), + node(2, { label: 'B', rect: small }), + ]); + expect(summarize(findings)).toEqual(['small-target @e1', 'small-target @e2']); + }); +}); + +describe('covered-control', () => { + const covered = { interactionBlocked: 'covered', hittable: false } as const; + + test('reports a covered control as an observation', () => { + const { findings } = audit([node(1, { label: 'Pay', ...covered })]); + expect(findings).toEqual([ + expect.objectContaining({ kind: 'covered-control', severity: 'observation', ref: '@e1' }), + ]); + }); + + test('ignores covered images and text, which no touch targets', () => { + const { findings } = audit([ + node(1, { type: 'Image', label: 'Overlay', ...covered }), + node(2, { type: 'StaticText', label: 'Title', ...covered }), + ]); + expect(findings).toEqual([]); + }); + + test('collapses the only child of a covered row, but keeps a child that has siblings', () => { + const wrapped = audit([ + node(1, { type: 'Cell', ...covered }), + node(2, { parentIndex: 1, label: 'Passcode', ...covered }), + ]); + expect(summarize(wrapped.findings)).toEqual(['covered-control @e1']); + + const withSibling = audit([ + node(1, { type: 'Cell', ...covered }), + node(2, { parentIndex: 1, label: 'Passcode', ...covered }), + node(3, { parentIndex: 1, label: 'Info', ...covered }), + ]); + expect(summarize(withSibling.findings)).toEqual([ + 'covered-control @e1', + 'covered-control @e2', + 'covered-control @e3', + ]); + }); + + test('keeps a covered control under a covered container that is neither control nor row', () => { + const { findings } = audit([ + node(1, { type: 'Other', ...covered }), + node(2, { parentIndex: 1, label: 'Pay', ...covered }), + ]); + expect(summarize(findings)).toEqual(['covered-control @e2']); + }); + + test('keeps a covered child whose parent is not covered', () => { + const { findings } = audit([ + node(1, { type: 'Cell' }), + node(2, { parentIndex: 1, label: 'A', ...covered }), + ]); + expect(summarize(findings)).toEqual(['covered-control @e2']); + }); + + test('does not run the other checks on a covered control', () => { + const { findings } = audit([ + node(1, { ...covered, rect: { x: 0, y: 0, width: 10, height: 10 } }), + ]); + expect(summarize(findings)).toEqual(['covered-control @e1']); + }); +}); + +describe('scope', () => { + test('skips system bars and nodes without a ref', () => { + const { findings } = audit([node(1, { systemBar: 'navigation' }), node(2, { ref: undefined })]); + expect(findings).toEqual([]); + }); + + test('marks a truncated capture partial and says controls past the cut were not audited', () => { + const result = audit([node(1, { label: 'A' })], { ...IOS, truncated: true }); + expect(result.checks).toEqual({ snapshot: 'partial' }); + expect(result.limitations).toEqual([expect.stringContaining('truncated')]); + }); + + test('keeps unknown truncation unknown instead of calling the capture complete', () => { + const result = audit([node(1, { label: 'A' })], { platform: 'ios' }); + expect(result.checks).toEqual({ snapshot: 'unknown' }); + expect(result.limitations).toEqual([expect.stringContaining('cannot report truncation')]); + }); + + test('audits web ARIA roles for names and coverage but not size', () => { + const result = audit( + [ + node(1, { type: 'button' }), + node(2, { type: 'textbox', label: 'Email' }), + node(3, { type: 'link', label: 'Home', interactionBlocked: 'covered', hittable: false }), + ], + { platform: 'web', truncated: false }, + ); + expect(summarize(result.findings)).toEqual(['missing-name @e1', 'covered-control @e3']); + expect(result.limitations).toEqual([expect.stringContaining('web rects')]); + }); +}); diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts new file mode 100644 index 0000000000..51870e4834 --- /dev/null +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -0,0 +1,288 @@ +import type { + AuditChecks, + AuditContext, + AuditFinding, + AuditResult, + AuditSizeUnit, +} from '@agent-device/contracts/capture'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + buildChildrenByParent, + collectDescendants, + type ChildrenByParent, +} from './snapshot-tree.ts'; + +const IOS_TARGET_PT = 44; +const ANDROID_TARGET_DP = 48; +/** Rects round-trip through float layout, so a 44pt control can read 43.67. */ +const SIZE_TOLERANCE = 0.5; +const GENERIC_RESOURCE_ID = /^[\w.]+:id\/[\w.-]+$/i; + +// Longest suffix first: `radiobutton` also ends with `button`. A suffix match keeps +// containers such as `RadioGroup` and `TabLayout` out of the control set. +const CONTROL_ROLES: ReadonlyArray = [ + ['radiobutton', 'radio'], + ['segmentedcontrol', 'segmented-control'], + ['button', 'button'], + ['link', 'link'], + ['switch', 'switch'], + ['checkbox', 'checkbox'], + ['slider', 'slider'], + ['seekbar', 'slider'], + ['menuitem', 'menu-item'], + ['searchfield', 'search-field'], + ['searchbox', 'search-field'], + ['textfield', 'text-field'], + ['edittext', 'text-field'], + ['textbox', 'text-field'], + ['combobox', 'combobox'], +]; + +const EDITABLE_ROLES = new Set(['text-field', 'search-field', 'combobox']); + +type SizeRule = + | { kind: 'checked'; unit: AuditSizeUnit; recommended: number; scale: number } + | { kind: 'unchecked'; limitation: string }; + +type AuditScan = { + byIndex: ReadonlyMap; + children: ChildrenByParent; +}; + +/** + * Advisory accessibility findings for one captured screen. Runs on the semantic nodes as + * captured, before display label dedup, and reads only facts the capture already published. + */ +export function auditSnapshot(nodes: readonly SnapshotNode[], context: AuditContext): AuditResult { + const scan: AuditScan = { + byIndex: new Map(nodes.map((node) => [node.index, node])), + children: buildChildrenByParent(nodes), + }; + const rule = resolveSizeRule(context); + const findings = nodes.flatMap((node) => auditNode(node, scan, rule)); + return { + findings, + checks: { snapshot: snapshotCompleteness(context) }, + limitations: collectLimitations(context, rule), + }; +} + +function snapshotCompleteness(context: AuditContext): AuditChecks['snapshot'] { + if (context.truncated === undefined) return 'unknown'; + return context.truncated ? 'partial' : 'complete'; +} + +function auditNode(node: SnapshotNode, scan: AuditScan, rule: SizeRule): AuditFinding[] { + if (node.ref === undefined) return []; + const role = controlRole(node); + const control = + role === undefined + ? [] + : collapseIntoParent(node, role, controlFindings(node, role, scan, rule), scan, rule); + const covered = coveredFinding(node, scan); + return covered ? [...control, covered] : control; +} + +/** Findings for a control the app owns and the user can see; none for any other node. */ +function controlFindings( + node: SnapshotNode, + role: string, + scan: AuditScan, + rule: SizeRule, +): AuditFinding[] { + if (node.ref === undefined || !isActionable(node)) return []; + const small = isInlineLink(node, role, scan) ? undefined : smallTargetFinding(node, role, rule); + const unnamed = hasAccessibleName(node, role, scan) ? undefined : missingNameFinding(node, role); + return [small, unnamed].filter((finding) => finding !== undefined); +} + +/** + * One control exposed twice: the only child of a control of the same role. The child's finding + * drops only when that parent reports the same kind, so a collapse never leaves a screen silent. + */ +function collapseIntoParent( + node: SnapshotNode, + role: string, + findings: AuditFinding[], + scan: AuditScan, + rule: SizeRule, +): AuditFinding[] { + const parent = parentOf(node, scan); + if (!parent || controlRole(parent) !== role) return findings; + if (scan.children.get(parent.index)?.length !== 1) return findings; + const parentKinds = new Set(controlFindings(parent, role, scan, rule).map(({ kind }) => kind)); + return findings.filter(({ kind }) => !parentKinds.has(kind)); +} + +function controlRole(node: SnapshotNode): string | undefined { + const type = normalizeType(node.type ?? ''); + return CONTROL_ROLES.find(([suffix]) => type.endsWith(suffix))?.[1]; +} + +/** + * Visible, enabled, not already proven unreachable by position, and owned by the app: system bars + * and keyboard keys are the platform's controls. + */ +function isActionable(node: SnapshotNode): boolean { + return ( + isPositiveFiniteRect(node.rect) && + node.systemBar === undefined && + node.keyboardOwned !== true && + node.enabled !== false && + node.visibleToUser !== false && + node.hittable !== false && + node.interactionBlocked === undefined + ); +} + +function parentOf(node: SnapshotNode, scan: AuditScan): SnapshotNode | undefined { + return node.parentIndex === undefined ? undefined : scan.byIndex.get(node.parentIndex); +} + +/** A link inside a text container keeps the text's own line height. */ +function isInlineLink(node: SnapshotNode, role: string, scan: AuditScan): boolean { + const parent = parentOf(node, scan); + return role === 'link' && parent !== undefined && isTextNode(parent); +} + +function isTextNode(node: SnapshotNode): boolean { + const role = formatRole(node.type ?? ''); + return role === 'text' || role === 'text-view'; +} + +function resolveSizeRule(context: AuditContext): SizeRule { + if (context.target !== undefined && context.target !== 'mobile') { + return unchecked( + `${context.target} targets are focus-driven, so touch target size does not apply.`, + ); + } + if (context.platform === 'ios') { + return { kind: 'checked', unit: 'pt', recommended: IOS_TARGET_PT, scale: 1 }; + } + if (context.platform !== 'android') { + return unchecked(`${context.platform} rects are not in points or dp.`); + } + if (!(context.pixelDensity && context.pixelDensity > 0)) { + return unchecked( + 'this Android capture did not report display density, so pixels cannot become dp.', + ); + } + return { + kind: 'checked', + unit: 'dp', + recommended: ANDROID_TARGET_DP, + scale: 1 / context.pixelDensity, + }; +} + +function unchecked(reason: string): SizeRule { + return { kind: 'unchecked', limitation: `Target size was not checked: ${reason}` }; +} + +function collectLimitations(context: AuditContext, rule: SizeRule): string[] { + const limitations = rule.kind === 'unchecked' ? [rule.limitation] : []; + if (context.truncated === true) { + limitations.push('The snapshot was truncated, so controls past the cut were not audited.'); + } + if (context.truncated === undefined) { + limitations.push('This capture cannot report truncation, so controls may be missing.'); + } + return limitations; +} + +function smallTargetFinding( + node: SnapshotNode, + role: string, + rule: SizeRule, +): AuditFinding | undefined { + if (rule.kind !== 'checked' || !isPositiveFiniteRect(node.rect)) return undefined; + const width = node.rect.width * rule.scale; + const height = node.rect.height * rule.scale; + const floor = rule.recommended - SIZE_TOLERANCE; + if (width >= floor && height >= floor) return undefined; + return { + ...findingBase(node, role), + kind: 'small-target', + severity: 'warning', + detail: { + unit: rule.unit, + width: roundTenth(width), + height: roundTenth(height), + recommendedSize: rule.recommended, + }, + }; +} + +function missingNameFinding(node: SnapshotNode, role: string): AuditFinding { + return { ...findingBase(node, role), kind: 'missing-name', severity: 'warning' }; +} + +/** + * The occlusion pass already decided this node is covered. Only controls and rows are reported, + * and the sole child of a covered control or row collapses into that parent, which reports. + */ +function coveredFinding(node: SnapshotNode, scan: AuditScan): AuditFinding | undefined { + const role = coveredRole(node); + if (node.interactionBlocked !== 'covered' || role === undefined) return undefined; + if (isOnlyChildOfReportedCover(node, scan)) return undefined; + return { ...findingBase(node, role), kind: 'covered-control', severity: 'observation' }; +} + +function coveredRole(node: SnapshotNode): string | undefined { + if (node.interactionBlocked !== 'covered') return undefined; + return controlRole(node) ?? (isRow(node) ? 'cell' : undefined); +} + +function isRow(node: SnapshotNode): boolean { + return normalizeType(node.type ?? '').endsWith('cell'); +} + +function isOnlyChildOfReportedCover(node: SnapshotNode, scan: AuditScan): boolean { + const parent = parentOf(node, scan); + if (!parent || parent.ref === undefined || coveredRole(parent) === undefined) return false; + return scan.children.get(parent.index)?.length === 1; +} + +function findingBase(node: SnapshotNode, role: string) { + return { + ref: `@${node.ref}`, + ...(node.rect ? { rect: node.rect } : {}), + role, + }; +} + +/** + * The control's own label or content description, a hint for a field, or a named descendant + * such as the text inside a button. An identifier is a test handle and a value is content, so + * neither names a control; a resource id is never a label. + */ +function hasAccessibleName(node: SnapshotNode, role: string, scan: AuditScan): boolean { + if (ownName(node, EDITABLE_ROLES.has(role))) return true; + if (EDITABLE_ROLES.has(role)) return false; + return collectDescendants(scan.children, node.index).some((child) => ownName(child, false)); +} + +function ownName(node: SnapshotNode, editable: boolean): string | undefined { + const typed = editable ? typedContent(node) : undefined; + const candidates = [node.label, node.contentDescription, node.text]; + if (editable) candidates.push(node.placeholder); + return candidates + .map((candidate) => candidate?.trim()) + .find( + (name) => + name !== undefined && name !== '' && name !== typed && !GENERIC_RESOURCE_ID.test(name), + ); +} + +/** A field value is content the user typed unless it is the placeholder showing. */ +function typedContent(node: SnapshotNode): string | undefined { + const value = node.value?.trim(); + if (!value || node.hintShowing === true || value === node.placeholder?.trim()) return undefined; + return value; +} + +function roundTenth(value: number): number { + return Math.round(value * 10) / 10; +} diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index b00e671665..67eab6bf3b 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -468,6 +468,8 @@ export function createAgentDeviceClient( return normalizeScreenshotCaptureResult(data, session); }, diff: async (options) => await executeCommand>('diff', options), + audit: async (options) => + await executeCommand>('audit', options ?? {}), }, interactions: { gesture: async (options) => await executeCommand('gesture', options), diff --git a/packages/command-output/src/capture/audit-output.test.ts b/packages/command-output/src/capture/audit-output.test.ts new file mode 100644 index 0000000000..240575819b --- /dev/null +++ b/packages/command-output/src/capture/audit-output.test.ts @@ -0,0 +1,76 @@ +import { expect, test } from 'vitest'; +import type { AuditSnapshotCommandResult } from '@agent-device/contracts/capture'; +import { auditCliOutput } from './audit-output.ts'; + +const CHECKS = { snapshot: 'complete' } as const; + +test('renders warnings and observations as aligned lines with the advisory wording', () => { + const result: AuditSnapshotCommandResult = { + findings: [ + { + kind: 'small-target', + severity: 'warning', + ref: '@e7', + role: 'button', + detail: { unit: 'pt', width: 22, height: 22.5, recommendedSize: 44 }, + }, + { + kind: 'missing-name', + severity: 'warning', + ref: '@e9', + role: 'text-field', + }, + { + kind: 'covered-control', + severity: 'observation', + ref: '@e12', + role: 'button', + }, + ], + checks: CHECKS, + limitations: [], + refsGeneration: 3, + }; + expect(auditCliOutput(result).text).toBe( + [ + 'agent-device audit', + '2 warnings, 1 observation — current screen', + '@e7~s3 small-target 22×22.5pt; review touch area', + '@e9~s3 missing-name Text field has no accessible name', + '@e12~s3 covered-control Accessible action is unavailable to touch', + '', + ].join('\n'), + ); +}); + +test('prints plain refs when the result carries no generation to pin them to', () => { + const output = auditCliOutput({ + findings: [{ kind: 'missing-name', severity: 'warning', ref: '@e9', role: 'button' }], + checks: CHECKS, + limitations: [], + }); + expect(output.text).toContain('@e9 missing-name'); + expect(output.text).not.toContain('~s'); +}); + +test('says no findings and lists each limitation', () => { + const output = auditCliOutput({ + findings: [], + checks: CHECKS, + limitations: ['Target size was not checked: tv targets are focus-driven.'], + }); + expect(output.text).toBe( + [ + 'agent-device audit', + 'No findings — current screen', + '', + 'Limitation: Target size was not checked: tv targets are focus-driven.', + '', + ].join('\n'), + ); +}); + +test('keeps the structured result as the JSON payload', () => { + const result: AuditSnapshotCommandResult = { findings: [], checks: CHECKS, limitations: [] }; + expect(auditCliOutput(result).data).toBe(result); +}); diff --git a/packages/command-output/src/capture/audit-output.ts b/packages/command-output/src/capture/audit-output.ts new file mode 100644 index 0000000000..3e4971e6d7 --- /dev/null +++ b/packages/command-output/src/capture/audit-output.ts @@ -0,0 +1,68 @@ +import type { + AuditFinding, + AuditSizeDetail, + AuditSnapshotCommandResult, +} from '@agent-device/contracts/capture'; +import { pinnedRefText, type CliOutput } from '../output-common.ts'; + +export function auditCliOutput(result: AuditSnapshotCommandResult): CliOutput { + return { data: result, text: formatAuditText(result) }; +} + +function formatAuditText(result: AuditSnapshotCommandResult): string { + const lines = [ + 'agent-device audit', + summaryLine(result.findings), + ...findingLines(result.findings, result.refsGeneration), + ]; + if (result.limitations.length > 0) { + lines.push('', ...result.limitations.map((limitation) => `Limitation: ${limitation}`)); + } + return `${lines.join('\n')}\n`; +} + +function summaryLine(findings: readonly AuditFinding[]): string { + const warnings = findings.filter((finding) => finding.severity === 'warning').length; + const observations = findings.length - warnings; + const counts = [plural(warnings, 'warning'), plural(observations, 'observation')].filter( + (count) => count !== undefined, + ); + return `${counts.length > 0 ? counts.join(', ') : 'No findings'} — current screen`; +} + +function plural(count: number, noun: string): string | undefined { + if (count === 0) return undefined; + return `${count} ${noun}${count === 1 ? '' : 's'}`; +} + +function findingLines( + findings: readonly AuditFinding[], + refsGeneration: number | undefined, +): string[] { + const refs = findings.map((finding) => pinnedRefText(finding.ref, refsGeneration) ?? finding.ref); + const refWidth = Math.max(0, ...refs.map((ref) => ref.length)); + const kindWidth = Math.max(0, ...findings.map((finding) => finding.kind.length)); + return findings.map( + (finding, index) => + `${refs[index]!.padEnd(refWidth)} ${finding.kind.padEnd(kindWidth)} ${findingText(finding)}`, + ); +} + +function findingText(finding: AuditFinding): string { + switch (finding.kind) { + case 'small-target': + return `${sizeText(finding.detail)}; review touch area`; + case 'missing-name': + return `${capitalize(finding.role.replaceAll('-', ' '))} has no accessible name`; + case 'covered-control': + return 'Accessible action is unavailable to touch'; + } +} + +function sizeText({ width, height, unit }: AuditSizeDetail): string { + return `${width}×${height}${unit}`; +} + +function capitalize(text: string): string { + return `${text.charAt(0).toUpperCase()}${text.slice(1)}`; +} diff --git a/packages/command-output/src/capture/output.ts b/packages/command-output/src/capture/output.ts index e8ed35709a..f63350d55c 100644 --- a/packages/command-output/src/capture/output.ts +++ b/packages/command-output/src/capture/output.ts @@ -1,6 +1,8 @@ import type { CommandName } from '@agent-device/command-surface'; +import type { AuditSnapshotCommandResult } from '@agent-device/contracts/capture'; import type { CaptureSnapshotResult } from '@agent-device/contracts/client'; import { dedupeInheritedSnapshotLabels } from '@agent-device/capture-kit/snapshot-label-dedup'; +import { auditCliOutput } from './audit-output.ts'; import { formatSnapshotText } from '../output/snapshot.ts'; import { messageOutput, type CliOutputFormatter, type CliOutput } from '../output-common.ts'; @@ -47,6 +49,7 @@ export const captureCliOutputFormatters = { scope: input.scope as string | undefined, depth: input.depth as number | undefined, }), + audit: async ({ result }) => auditCliOutput(result as AuditSnapshotCommandResult), wait: messageOutput, alert: messageOutput, // Android permission revokes append a relaunch warning (#1796); render it for humans too. diff --git a/packages/command-output/src/cli-output.test.ts b/packages/command-output/src/cli-output.test.ts index faf510b8ae..49bfc1ca4a 100644 --- a/packages/command-output/src/cli-output.test.ts +++ b/packages/command-output/src/cli-output.test.ts @@ -33,6 +33,7 @@ const FORMATTER_COMMANDS = [ 'appstate', 'artifacts', 'audio', + 'audit', 'back', 'batch', 'boot', @@ -100,6 +101,11 @@ async function render(name: CommandName, result: unknown, input = {}) { const RESPONSE_FIXTURES: Partial> = { apps: ['com.example.app'], artifacts: { source: 'daemon', artifacts: [], message: 'No daemon artifacts.' }, + audit: { + findings: [], + checks: { snapshot: 'complete' }, + limitations: [], + }, batch: { executed: 0, total: 0, totalDurationMs: 5, results: [] }, capabilities: { device: { name: 'iPhone' }, availableCommands: [] }, debug: { diff --git a/packages/command-registry/src/__tests__/command-result.test.ts b/packages/command-registry/src/__tests__/command-result.test.ts index 6d97d6887f..f514adaec8 100644 --- a/packages/command-registry/src/__tests__/command-result.test.ts +++ b/packages/command-registry/src/__tests__/command-result.test.ts @@ -1,4 +1,5 @@ import type { + AuditSnapshotCommandResult, DiffSnapshotCommandResult, ViewportCommandResult, } from '@agent-device/contracts/capture'; @@ -70,6 +71,7 @@ test('seeded CommandResult entries resolve to their existing contract result typ TriggerAppEventCommandResult > = true; const doctor: Equal, DoctorCommandResult> = true; + const audit: Equal, AuditSnapshotCommandResult> = true; const diff: Equal, DiffSnapshotCommandResult> = true; const replay: Equal, ReplayCommandResult> = true; const replayTest: Equal, ReplaySuiteResult> = true; @@ -98,6 +100,7 @@ test('seeded CommandResult entries resolve to their existing contract result typ push, triggerAppEvent, doctor, + audit, diff, replay, replayTest, @@ -143,6 +146,7 @@ test('CommandResultMap is seeded only from already-existing contract result type | 'push' | 'trigger-app-event' | 'doctor' + | 'audit' | 'diff' | 'replay' | 'test' diff --git a/packages/command-registry/src/command-result.ts b/packages/command-registry/src/command-result.ts index 6e0a9aeb1b..421fef0df2 100644 --- a/packages/command-registry/src/command-result.ts +++ b/packages/command-registry/src/command-result.ts @@ -1,4 +1,5 @@ import type { + AuditSnapshotCommandResult, DiffSnapshotCommandResult, ViewportCommandResult, } from '@agent-device/contracts/capture'; @@ -64,6 +65,7 @@ export interface CommandResultMap { 'action-button': ActionButtonCommandResult; 'app-switcher': AppSwitcherCommandResult; appstate: AppStateCommandResult; + audit: AuditSnapshotCommandResult; back: BackCommandResult; boot: BootCommandResult; click: ClickCommandResponseData; diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index 3230fb2d02..971ec64dae 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -478,6 +478,20 @@ export const RAW_COMMAND_DESCRIPTORS = [ selectUses: selectSnapshotStepUses, }, }, + { + name: 'audit', + deviceClaimPolicy: 'require-owner', + ...(ownerFilesEnabled + ? { ownerFiles: ['packages/command-surface/src/capture/snapshot.ts'] as const } + : {}), + catalog: { group: 'public' }, + frameworkTier: 'extended', + recordsSessionAction: false, + daemon: { route: 'snapshot', refFrameEffect: 'preserve', appLease: 'allowed' }, + timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + batchable: false, + platformExecution: { kind: 'device-runtime', uses: snapshotRuntimePlanUses }, + }, ...DEVICE_OPERATION_COMMAND_DESCRIPTORS.captureAndSystem, { name: 'find', diff --git a/packages/command-runtime/src/commands/capture/runtime/index.ts b/packages/command-runtime/src/commands/capture/runtime/index.ts index 13188f55fc..ee728f8b43 100644 --- a/packages/command-runtime/src/commands/capture/runtime/index.ts +++ b/packages/command-runtime/src/commands/capture/runtime/index.ts @@ -1,6 +1,10 @@ -import type { DiffSnapshotCommandResult } from '@agent-device/contracts/capture'; +import type { + AuditSnapshotCommandResult, + DiffSnapshotCommandResult, +} from '@agent-device/contracts/capture'; import { bindRuntimeCommands } from '../../runtime-types.ts'; import type { + AuditSnapshotCommandOptions, BoundOf, DiffSnapshotCommandOptions, RuntimeCommand, @@ -14,13 +18,19 @@ import { type DiffScreenshotCommandResult, } from './diff-screenshot.ts'; import { screenshotCommand, type ScreenshotCommandResult } from './screenshot.ts'; -import { diffSnapshotCommand, snapshotCommand, type SnapshotCommandResult } from './snapshot.ts'; +import { + auditSnapshotCommand, + diffSnapshotCommand, + snapshotCommand, + type SnapshotCommandResult, +} from './snapshot.ts'; export type CaptureCommands = { screenshot: RuntimeCommand; diffScreenshot: RuntimeCommand; snapshot: RuntimeCommand; diffSnapshot: RuntimeCommand; + auditSnapshot: RuntimeCommand; }; export type BoundCaptureCommands = BoundOf; @@ -30,6 +40,7 @@ const captureCommands: CaptureCommands = { diffScreenshot: diffScreenshotCommand, snapshot: snapshotCommand, diffSnapshot: diffSnapshotCommand, + auditSnapshot: auditSnapshotCommand, }; export function bindCaptureCommands(runtime: AgentDeviceRuntime): BoundCaptureCommands { diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 73abc8e6b5..d1c76bc7a4 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -2,6 +2,7 @@ import { copySnapshotClickabilityEvidence, publicSnapshotCaptureAnnotations, snapshotCaptureAnnotationsFrom, + type AuditSnapshotCommandResult, type DiffSnapshotCommandResult, type PublicSnapshotCaptureAnnotations, type SnapshotCaptureAnnotations, @@ -40,6 +41,7 @@ import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/andro import { formatReactNativeOverlayWarning } from '@agent-device/capture-kit/react-native-overlay'; import { now } from '../../runtime-common.ts'; import type { + AuditSnapshotCommandOptions, DiffSnapshotCommandOptions, RuntimeCommand, SnapshotCommandOptions, @@ -182,6 +184,27 @@ export const diffSnapshotCommand: RuntimeCommand< }; }; +/** + * One fresh full-tree capture, audited on the captured nodes. The capture replaces the session + * snapshot like `snapshot` does, so the refs in the findings address the stored tree. The audit + * module loads on first use: no other command pays for it. + */ +export const auditSnapshotCommand: RuntimeCommand< + AuditSnapshotCommandOptions, + AuditSnapshotCommandResult +> = async (runtime, options): Promise => { + const { auditSnapshot } = await import('@agent-device/capture-kit/snapshot-audit'); + const capture = await captureRuntimeSnapshot(runtime, options); + const audit = auditSnapshot(capture.snapshot.nodes, { + platform: runtime.backend.platform, + target: options.target, + pixelDensity: capture.annotations.androidSnapshot?.pixelDensity, + truncated: snapshotTruncationForResult(capture.snapshot), + }); + await runtime.sessions.set(nextSnapshotSession(options.session, capture)); + return { ...audit, ...(capture.warnings.length > 0 ? { warnings: capture.warnings } : {}) }; +}; + async function captureRuntimeSnapshot( runtime: AgentDeviceRuntime, options: SnapshotCommandOptions, diff --git a/packages/command-runtime/src/commands/runtime-types.ts b/packages/command-runtime/src/commands/runtime-types.ts index ace1450125..7bbfa87e1f 100644 --- a/packages/command-runtime/src/commands/runtime-types.ts +++ b/packages/command-runtime/src/commands/runtime-types.ts @@ -2,6 +2,7 @@ import type { ScreenshotScreenMask } from '@agent-device/contracts/capture'; import type { FileOutputRef } from '@agent-device/contracts/artifact-adapter'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import type { SessionSurface } from '@agent-device/contracts/session'; +import type { DeviceTarget } from '@agent-device/kernel/device'; import type { SnapshotCommandOptionFields } from '@agent-device/kernel/snapshot'; export type CommandResult = Record; @@ -61,3 +62,9 @@ export type ScreenshotCommandOptions = CommandContext & { export type SnapshotCommandOptions = CommandContext & SnapshotCommandOptionFields; export type DiffSnapshotCommandOptions = SnapshotCommandOptions; + +/** `audit` always reads the whole screen, so it takes no snapshot scoping fields. */ +export type AuditSnapshotCommandOptions = CommandContext & { + /** Target the capture came from; absent means mobile. */ + target?: DeviceTarget; +}; diff --git a/packages/command-surface/src/capture/index.ts b/packages/command-surface/src/capture/index.ts index c908be42f6..c1b2b746bc 100644 --- a/packages/command-surface/src/capture/index.ts +++ b/packages/command-surface/src/capture/index.ts @@ -3,13 +3,14 @@ import { alertCommandFacet } from './alert.ts'; import { diffCommandFacet } from './diff.ts'; import { screenshotCommandFacet } from './screenshot.ts'; import { settingsCommandFacet } from './settings.ts'; -import { snapshotCommandFacet } from './snapshot.ts'; +import { auditCommandFacet, snapshotCommandFacet } from './snapshot.ts'; import { waitCommandFacet } from './wait.ts'; const captureCommandFacets = [ snapshotCommandFacet, screenshotCommandFacet, diffCommandFacet, + auditCommandFacet, waitCommandFacet, alertCommandFacet, settingsCommandFacet, diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index 2f5eb5f561..bd456dde9e 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -5,14 +5,26 @@ import { snapshotOptionsFromFlags, } from '@agent-device/kernel/snapshot'; import { SNAPSHOT_FLAGS } from '@agent-device/command-registry/flag-groups'; -import { booleanField, integerField, optionField, stringField } from '../command-input.ts'; +import type { CommandResultMap } from '@agent-device/command-registry/command-result'; +import { + booleanField, + enumSchema, + integerField, + numberSchema, + objectSchema, + optionField, + stringArraySchema, + stringField, + stringSchema, +} from '../command-input.ts'; +import type { JsonSchema } from '../command-contract.ts'; import { commonInputFromFlags, direct, observationRecordInputFromFlags, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; -import { defineCommandFacet } from '../family/types.ts'; +import { defineCommandFacet, defineParameterlessCommandFacet } from '../family/types.ts'; import { defineFieldCommandMetadata } from '../field-command-contract.ts'; const SNAPSHOT_COMMAND_NAME = 'snapshot'; @@ -85,3 +97,79 @@ export const snapshotCommandFacet = defineCommandFacet({ cliReader: snapshotCliReader, daemonWriter: snapshotDaemonWriter, }); + +const AUDIT_COMMAND_NAME = 'audit'; + +/** + * This family's advertised MCP `outputSchema`, keyed by daemon command name and projected into + * the command map by `command-output-schemas.ts`. Non-strict: additive response fields keep + * validating. + */ +export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { + // packages/contracts/src/audit.ts + audit: objectSchema( + { + findings: { + type: 'array', + items: objectSchema( + { + kind: enumSchema(['small-target', 'missing-name', 'covered-control']), + severity: enumSchema(['warning', 'observation']), + ref: stringSchema('Plain ref, usable in the frame this audit published.'), + rect: objectSchema( + { + x: numberSchema(), + y: numberSchema(), + width: numberSchema(), + height: numberSchema(), + }, + ['x', 'y', 'width', 'height'], + ), + role: stringSchema(), + detail: objectSchema( + { + unit: enumSchema(['pt', 'dp']), + width: numberSchema(), + height: numberSchema(), + recommendedSize: numberSchema(), + }, + ['unit', 'width', 'height', 'recommendedSize'], + ), + }, + ['kind', 'severity', 'ref', 'role'], + ), + }, + checks: objectSchema( + { + snapshot: enumSchema( + ['complete', 'partial', 'unknown'], + 'unknown: the capture cannot report truncation, so controls may be missing.', + ), + }, + ['snapshot'], + ), + limitations: stringArraySchema(), + refsGeneration: numberSchema( + 'Present only with findings: pair each ref with it (@eN~sG) before a mutation.', + ), + warnings: stringArraySchema(), + }, + ['findings', 'checks', 'limitations'], + ), +} satisfies Pick, 'audit'>; + +const auditCommandDescription = + 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and controls the snapshot already marks covered. Findings are review prompts, not failures, and the audit changes nothing on the device.'; + +export const auditCommandFacet = defineParameterlessCommandFacet({ + name: AUDIT_COMMAND_NAME, + effect: 'write', + description: auditCommandDescription, + text: { + summary: 'Report advisory accessibility findings', + searchHint: 'a11y lint voiceover talkback', + cliDetail: + 'Findings are a partial ref publication, like find: the text prints each ref pinned (@eN~sG), and a plain ref from before the audit stays refused after the frame expires. An audit with no findings changes no ref state. Target size is checked on iOS in points (44) and on Android in dp (48, from the captured display density); other targets report a limitation instead. Duplicate wrapper controls collapse only when a control is the sole child of a control with the same role.', + }, + run: (client, input) => client.capture.audit(input), +}); diff --git a/packages/command-surface/src/client-port.ts b/packages/command-surface/src/client-port.ts index f35507f2d5..34428f36f3 100644 --- a/packages/command-surface/src/client-port.ts +++ b/packages/command-surface/src/client-port.ts @@ -29,6 +29,7 @@ import type { AudioOptions, BackCommandOptions, BatchRunOptions, + CaptureAuditOptions, CaptureDiffOptions, CaptureScreenshotOptions, CaptureScreenshotResult, @@ -203,6 +204,7 @@ export type AgentDeviceClient = { snapshot: (options?: CaptureSnapshotOptions) => Promise; screenshot: (options?: CaptureScreenshotOptions) => Promise; diff: (options: CaptureDiffOptions) => Promise>; + audit: (options?: CaptureAuditOptions) => Promise>; }; interactions: { gesture: (options: GestureOptions) => Promise; diff --git a/packages/command-surface/src/command-output-schemas.ts b/packages/command-surface/src/command-output-schemas.ts index ae8de1288f..ffbf25c11a 100644 --- a/packages/command-surface/src/command-output-schemas.ts +++ b/packages/command-surface/src/command-output-schemas.ts @@ -9,6 +9,7 @@ import { stringSchema, } from './command-input.ts'; import { DIFF_COMMAND_OUTPUT_SCHEMAS } from './capture/diff.ts'; +import { AUDIT_COMMAND_OUTPUT_SCHEMAS } from './capture/snapshot.ts'; import { WAIT_COMMAND_OUTPUT_SCHEMAS } from './capture/wait.ts'; import { PREPARE_COMMAND_OUTPUT_SCHEMAS } from './management/prepare.ts'; import { DOCTOR_COMMAND_OUTPUT_SCHEMAS } from './management/doctor.ts'; @@ -190,6 +191,7 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { // command surface and projects it here. This spread stays last: a hand-written entry for // a projected command then fails as TS2783 instead of quietly overriding the family's, // and this map's `satisfies` still refuses a missing `CommandResultMap` key. + ...AUDIT_COMMAND_OUTPUT_SCHEMAS, ...DIFF_COMMAND_OUTPUT_SCHEMAS, ...WAIT_COMMAND_OUTPUT_SCHEMAS, ...PREPARE_COMMAND_OUTPUT_SCHEMAS, diff --git a/packages/contracts/src/audit.ts b/packages/contracts/src/audit.ts new file mode 100644 index 0000000000..73e0fedfa8 --- /dev/null +++ b/packages/contracts/src/audit.ts @@ -0,0 +1,65 @@ +import type { DeviceTarget, PublicPlatform } from '@agent-device/kernel/device'; +import type { Rect } from '@agent-device/kernel/snapshot'; + +export type AuditFindingKind = 'small-target' | 'missing-name' | 'covered-control'; + +/** `warning` asks for review; `observation` reports a fact the screen already proved. */ +export type AuditSeverity = 'warning' | 'observation'; + +export type AuditSizeUnit = 'pt' | 'dp'; + +/** A `small-target` finding's measured size and the recommendation, all in `unit`. */ +export type AuditSizeDetail = { + unit: AuditSizeUnit; + width: number; + height: number; + recommendedSize: number; +}; + +type AuditFindingBase = { + /** Plain ref (`@e7`) of the node. It is valid only with the response's `refsGeneration`. */ + ref: string; + /** The node's rect in snapshot coordinates (points on iOS, physical pixels on Android). */ + rect?: Rect; + /** Control role as the snapshot presents it, such as `button`. */ + role: string; +}; + +export type AuditFinding = + | (AuditFindingBase & { kind: 'small-target'; severity: 'warning'; detail: AuditSizeDetail }) + | (AuditFindingBase & { kind: 'missing-name'; severity: 'warning' }) + | (AuditFindingBase & { kind: 'covered-control'; severity: 'observation' }); + +export type AuditContext = { + platform: PublicPlatform; + /** Absent means a mobile target. */ + target?: DeviceTarget; + /** Physical pixels per dp, from the Android capture. */ + pixelDensity?: number; + /** + * Whether the capture cut the tree short. Absent means the producer cannot say, which is not + * the same as `false`. + */ + truncated?: boolean; +}; + +export type AuditChecks = { + /** `unknown`: the producer cannot report truncation, so the tree may be partial. */ + snapshot: 'complete' | 'partial' | 'unknown'; +}; + +export type AuditResult = { + findings: AuditFinding[]; + checks: AuditChecks; + /** Checks the audit could not run or could only partly run, with the reason. */ + limitations: string[]; +}; + +export type AuditSnapshotCommandResult = AuditResult & { + /** + * Ref-frame epoch the finding refs were minted from (ADR 0014). Present only when the audit + * returned a finding; pair each ref with it (`@e7~s`) before a mutation. + */ + refsGeneration?: number; + warnings?: string[]; +}; diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index db9f48ad04..047f3c9bce 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -148,6 +148,8 @@ export type CaptureDiffOptions = DeviceCommandBaseOptions & out?: string; }; +export type CaptureAuditOptions = DeviceCommandBaseOptions; + export type SelectorSnapshotCommandOptions = Pick< CaptureSnapshotOptions, 'depth' | 'scope' | 'raw' diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts index 85e86560bc..e945f9f97a 100644 --- a/packages/contracts/src/daemon-command-surface.ts +++ b/packages/contracts/src/daemon-command-surface.ts @@ -3,10 +3,12 @@ // composition root assigns `createCommandSurfaceAgentDevice` to this type, so the compiler // checks every signature against the implementation on each build. +import type { DeviceTarget } from '@agent-device/kernel/device'; import type { SessionSurface } from './session-surface.ts'; import type { ArtifactDescriptor, FileOutputRef } from './artifact-adapter.ts'; import type { DeviceRotation } from './device-rotation.ts'; import type { ClickButton } from './click-button.ts'; +import type { AuditSnapshotCommandResult } from './audit.ts'; import type { DiffSnapshotCommandResult } from './diff.ts'; import type { PostGestureOutcome } from './snapshot-model.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; @@ -63,6 +65,9 @@ export type DaemonSelectorSnapshotOptions = Readonly<{ /** The options of the daemon's `snapshot` / `diff snapshot` calls. */ export type DaemonSnapshotOptions = CommandContext & SnapshotCommandOptionFields; +/** The options of the daemon's `audit` call: one full-screen capture, no snapshot scoping. */ +export type DaemonAuditOptions = CommandContext & Readonly<{ target?: DeviceTarget }>; + /** What the daemon's `screenshot` route and its two observation paths receive. */ export type DaemonScreenshotResult = Readonly<{ path: string; @@ -312,6 +317,7 @@ export type DaemonCommandSurfaceCapture = Readonly<{ screenshot: (options: DaemonScreenshotCommandOptions) => Promise; snapshot: (options: DaemonSnapshotOptions) => Promise; diffSnapshot: (options: DaemonSnapshotOptions) => Promise; + auditSnapshot: (options: DaemonAuditOptions) => Promise; }>; /** The four selector reads the daemon's selector and wait routes serve. Function-typed diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index e69e431077..5e22aa156b 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -1,3 +1,14 @@ +export type { + AuditChecks, + AuditContext, + AuditFinding, + AuditFindingKind, + AuditResult, + AuditSeverity, + AuditSizeDetail, + AuditSizeUnit, + AuditSnapshotCommandResult, +} from '../audit.ts'; export type { DiffSnapshotCommandResult, SnapshotDiffLine, SnapshotDiffSummary } from '../diff.ts'; export { RETIRED_SCREENSHOT_MAX_SIZE, diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index 4e526be19c..13fc8b1f15 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -19,6 +19,7 @@ export type { MaterializationReleaseResult, } from '../client-app.ts'; export type { + CaptureAuditOptions, CaptureDiffOptions, CaptureScreenshotOptions, CaptureScreenshotResult, diff --git a/packages/daemon/src/__tests__/support/command-surface-fake.ts b/packages/daemon/src/__tests__/support/command-surface-fake.ts index ea17cd1f46..b26ddebade 100644 --- a/packages/daemon/src/__tests__/support/command-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/command-surface-fake.ts @@ -25,6 +25,11 @@ const fakeCommandSurface: DaemonCommandSurface = { summary: { additions: 0, removals: 0, unchanged: 0 }, lines: [], })), + auditSnapshot: vi.fn(async () => ({ + findings: [], + checks: { snapshot: 'complete' as const, platform: 'not-requested' as const }, + limitations: [], + })), }, selectors: { find: vi.fn(async () => { diff --git a/packages/daemon/src/core/snapshot-runtime-binding.ts b/packages/daemon/src/core/snapshot-runtime-binding.ts index 7d341ecc24..8f33d09d76 100644 --- a/packages/daemon/src/core/snapshot-runtime-binding.ts +++ b/packages/daemon/src/core/snapshot-runtime-binding.ts @@ -164,7 +164,7 @@ function canUseShortAndroidSnapshotIdleWait(session: SessionState | undefined): /** Resolves one plan, inspects its owner facts once, then returns one bound capture closure. */ export async function resolveBoundSnapshotCaptureRuntime( params: SnapshotRuntimeRouteParams, - command: 'snapshot' | 'diff', + command: 'snapshot' | 'diff' | 'audit', ): Promise { const { req, sessionName, sessionStore } = params; const { ref, session, device } = await resolveSessionDevice( diff --git a/packages/daemon/src/execution/android-snapshot-timeout-evidence.ts b/packages/daemon/src/execution/android-snapshot-timeout-evidence.ts index de6d8e38b6..09d79f4092 100644 --- a/packages/daemon/src/execution/android-snapshot-timeout-evidence.ts +++ b/packages/daemon/src/execution/android-snapshot-timeout-evidence.ts @@ -36,13 +36,13 @@ import { export async function maybeBuildAndroidSnapshotTimeoutFailure( params: { error: unknown; - command: 'snapshot' | 'diff'; + command: 'snapshot' | 'diff' | 'audit'; logPath: string; session: SessionState | undefined; device: SessionState['device']; } & ScreenshotRuntimeBindings, ): Promise | undefined> { - if (params.command !== 'snapshot') return undefined; + if (params.command === 'diff') return undefined; if (params.device.platform !== 'android') return undefined; const normalized = normalizeError(params.error); diff --git a/packages/daemon/src/sessions/handlers/snapshot.ts b/packages/daemon/src/sessions/handlers/snapshot.ts index 82f2e6ed8b..7d762544d1 100644 --- a/packages/daemon/src/sessions/handlers/snapshot.ts +++ b/packages/daemon/src/sessions/handlers/snapshot.ts @@ -7,6 +7,7 @@ import { handleSettingsCommand, parseSettingsArgs, } from '../../execution/handlers/snapshot-settings.ts'; +import { dispatchSnapshotAuditViaRuntime } from '../snapshot-audit-runtime.ts'; import { dispatchSnapshotDiffViaRuntime } from '../snapshot-diff-runtime.ts'; import { dispatchSnapshotViaRuntime } from '../snapshot-runtime.ts'; import { dispatchWaitViaRuntime } from '../../execution/wait-runtime.ts'; @@ -82,6 +83,28 @@ const SNAPSHOT_COMMAND_HANDLER_IMPLS = { platformResourceCleanup, }); }, + audit: async ({ + req, + createCommandSurface, + sessionName, + logPath, + sessionStore, + inspectFacts, + bindDevice, + platformServices, + platformResourceCleanup, + }) => + await dispatchSnapshotAuditViaRuntime({ + createCommandSurface, + req, + sessionName, + logPath, + sessionStore, + inspectFacts, + bindDevice, + platformServices, + platformResourceCleanup, + }), wait: async ({ req, createCommandSurface, diff --git a/packages/daemon/src/sessions/snapshot-audit-runtime.ts b/packages/daemon/src/sessions/snapshot-audit-runtime.ts new file mode 100644 index 0000000000..ba7e09570a --- /dev/null +++ b/packages/daemon/src/sessions/snapshot-audit-runtime.ts @@ -0,0 +1,33 @@ +import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import { dispatchSnapshotRuntimeCommand } from './snapshot-command-runtime.ts'; +import type { SnapshotRuntimeRouteParams } from '../core/snapshot-runtime-binding.ts'; +import { markSessionPartialRefsIssued } from '../execution/session-snapshot.ts'; + +/** + * Canonical `audit` route: one fresh capture over the request-bound capture runtime. The response + * returns only its finding refs, so it is a partial publication (ADR 0014): the refs are issued + * through the partial-frame primitive and ride with the frame epoch. A response without findings + * issues nothing and leaves the frame as it was. + */ +export async function dispatchSnapshotAuditViaRuntime( + params: SnapshotRuntimeRouteParams, +): Promise { + return await dispatchSnapshotRuntimeCommand({ + ...params, + command: 'audit', + execute: async ({ runtime: agentRuntime, sessionName, device, getSession }) => { + const result = await agentRuntime.capture.auditSnapshot({ + session: sessionName, + target: device.target, + }); + const session = getSession(); + if (!session || result.findings.length === 0) return { data: result }; + markSessionPartialRefsIssued( + session, + result.findings.map((finding) => finding.ref), + ); + const refsGeneration = session.snapshotGeneration; + return { data: refsGeneration === undefined ? result : { ...result, refsGeneration } }; + }, + }); +} diff --git a/packages/daemon/src/sessions/snapshot-command-runtime.ts b/packages/daemon/src/sessions/snapshot-command-runtime.ts index c159254a7d..de91a41ba0 100644 --- a/packages/daemon/src/sessions/snapshot-command-runtime.ts +++ b/packages/daemon/src/sessions/snapshot-command-runtime.ts @@ -43,14 +43,15 @@ export type SnapshotRuntimeRecord = }; type SnapshotRuntimeCommandParams = SnapshotRuntimeRouteParams & { - command: 'snapshot' | 'diff'; + command: 'snapshot' | 'diff' | 'audit'; execute(params: { runtime: Awaited>['runtime']; sessionName: string; req: DaemonRequest; + device: SessionState['device']; snapshotScope: string | undefined; getSession(): SessionState | undefined; - }): Promise<{ data: DaemonResponseData; record: SnapshotRuntimeRecord }>; + }): Promise<{ data: DaemonResponseData; record?: SnapshotRuntimeRecord }>; }; /** Shared snapshot/diff command construction after one request-bound capture has been admitted. */ @@ -86,7 +87,14 @@ export async function dispatchSnapshotRuntimeCommand( }; let result: Awaited>; try { - result = await params.execute({ runtime, sessionName, req, snapshotScope, getSession }); + result = await params.execute({ + runtime, + sessionName, + req, + device, + snapshotScope, + getSession, + }); } catch (error) { const timeoutResponse = await maybeBuildAndroidSnapshotTimeoutFailure({ error, @@ -102,13 +110,15 @@ export async function dispatchSnapshotRuntimeCommand( return timeoutResponse; } const current = getSession(); - recordSnapshotRuntimeAction({ - req, - sessionName, - sessionStore, - ref: sessions.getRef(), - result: result.record, - }); + if (result.record) { + recordSnapshotRuntimeAction({ + req, + sessionName, + sessionStore, + ref: sessions.getRef(), + result: result.record, + }); + } const data = applyRecoveredWarningLatch({ session: current, data: result.data, @@ -174,7 +184,7 @@ async function createSnapshotRuntime(params: { reactivateCompleteFrameIfIssuing( nextSession, keepCurrentSnapshot, - req.command === 'snapshot' && req.internal?.observationOnly !== true, + issuesCompleteRefFrame(req), ); return ref ?? sessionStore.publish(sessionName, nextSession); }, @@ -211,6 +221,10 @@ function shouldKeepCurrentSnapshot( } // ADR 0014: only a snapshot command hands the client the complete ref namespace. +function issuesCompleteRefFrame(req: DaemonRequest): boolean { + return req.command === 'snapshot' && req.internal?.observationOnly !== true; +} + function reactivateCompleteFrameIfIssuing( session: SessionState, keepCurrentSnapshot: boolean, diff --git a/scripts/capability-matrix/annotations.ts b/scripts/capability-matrix/annotations.ts index fb846fbc27..495d341a08 100644 --- a/scripts/capability-matrix/annotations.ts +++ b/scripts/capability-matrix/annotations.ts @@ -80,8 +80,8 @@ export const MATRIX_SECTIONS: readonly MatrixSection[] = [ rows: [ { id: 'snapshot-tree', - label: 'Accessibility snapshot and snapshot diff', - commands: ['snapshot', 'diff'], + label: 'Accessibility snapshot, snapshot diff, and audit', + commands: ['snapshot', 'diff', 'audit'], link: docs('snapshots'), advertises: ['captureSnapshot'], note: 'A capture returns a tree only when the app or desktop exposes an accessibility tree.', diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index cee6d6dc56..ee07f2fb67 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -537,6 +537,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/screenshot-diff-regions', '@agent-device/capture-kit/screenshot-overlay', '@agent-device/capture-kit/scroll-edge-state', + '@agent-device/capture-kit/snapshot-audit', '@agent-device/capture-kit/snapshot-baseline', '@agent-device/capture-kit/snapshot-chrome', '@agent-device/capture-kit/snapshot-chrome-fixtures', diff --git a/scripts/perf/suite.ts b/scripts/perf/suite.ts index 3d110c29ce..33a219af36 100644 --- a/scripts/perf/suite.ts +++ b/scripts/perf/suite.ts @@ -203,6 +203,7 @@ export const SUITE: readonly SuiteScenario[] = [ }), step('snapshot-diff', C.snapshot, ['snapshot', '--diff']), step('diff-snapshot', C.diff, ['diff', 'snapshot']), + step('audit', C.audit, ['audit']), step('wait-text', C.wait, ['wait', 'text', 'Agent Device Tester']), step('wait-selector', C.wait, ['wait', 'id="gesture-canary-ready"']), step('wait-absent', C.wait, ['wait', 'absent', 'id="heavy-feed-title"']), diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index a14897f360..e6a9fe4d00 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1191,6 +1191,7 @@ function createStubClient(params: { identifiers: { session: 'default' }, })), diff: unexpectedCommandCall, + audit: unexpectedCommandCall, }, interactions: createThrowingMethodGroup(), replay: createThrowingMethodGroup(), diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index e71f11fb40..aee2e26c7a 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -80,6 +80,7 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { .filter((descriptor) => descriptor.timeoutPolicy.onTimeout === 'preserve-daemon') .map((descriptor) => descriptor.name); assert.deepEqual(preserving.sort(), [ + 'audit', 'back', 'click', 'fill', diff --git a/src/__tests__/test-utils/android-snapshot-helper.ts b/src/__tests__/test-utils/android-snapshot-helper.ts index 155d763701..55d3885897 100644 --- a/src/__tests__/test-utils/android-snapshot-helper.ts +++ b/src/__tests__/test-utils/android-snapshot-helper.ts @@ -25,7 +25,10 @@ export const ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT: AndroidSnapshotHelperArti }, }; -export function androidSnapshotHelperOutput(xml: string): string { +export function androidSnapshotHelperOutput( + xml: string, + options: { pixelDensity?: number } = {}, +): string { return [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', 'INSTRUMENTATION_STATUS: helperApiVersion=1', @@ -37,6 +40,9 @@ export function androidSnapshotHelperOutput(xml: string): string { 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-snapshot-helper-v1', 'INSTRUMENTATION_RESULT: helperApiVersion=1', 'INSTRUMENTATION_RESULT: ok=true', + ...(options.pixelDensity === undefined + ? [] + : [`INSTRUMENTATION_RESULT: pixelDensity=${options.pixelDensity}`]), 'INSTRUMENTATION_CODE: 0', ].join('\n'); } diff --git a/src/commands/replay/replay-script.fixtures.ts b/src/commands/replay/replay-script.fixtures.ts index a89a4c254e..f433e68df2 100644 --- a/src/commands/replay/replay-script.fixtures.ts +++ b/src/commands/replay/replay-script.fixtures.ts @@ -183,6 +183,7 @@ const REPLAY_SCRIPT_LINE_PLANS = { 'hardware-button': GENERIC_REPLAY_LINE, uninstall: GENERIC_REPLAY_LINE, viewport: GENERIC_REPLAY_LINE, + audit: { waived: 'audit reports one observation and is never recorded in replay scripts' }, human_control: { waived: 'host-local control commands are never recorded in replay scripts' }, install_source: GENERIC_REPLAY_LINE, lease_allocate: GENERIC_REPLAY_LINE, diff --git a/src/mcp/__tests__/tool-ref-pins.test.ts b/src/mcp/__tests__/tool-ref-pins.test.ts index e7399824b6..9f8f7900eb 100644 --- a/src/mcp/__tests__/tool-ref-pins.test.ts +++ b/src/mcp/__tests__/tool-ref-pins.test.ts @@ -465,6 +465,60 @@ test('ref-pin store leaves existing pins untouched for a mutating find without r assert.deepEqual(pinned, { session: 'demo', ref: '@e2~s7' }); }); +test('ref-pin store pins audit finding refs to the audit generation and keeps other pins', () => { + const pins = makeStore(); + const checks = { snapshot: 'complete' } as const; + + pins.mergeCommandResult( + 'snapshot', + { nodes: [{ ref: 'e2' }, { ref: 'e37' }], truncated: false, refsGeneration: 7 }, + undefined, + 'demo', + ); + pins.mergeCommandResult( + 'audit', + { + findings: [{ kind: 'missing-name', severity: 'warning', ref: '@e2', role: 'button' }], + checks, + limitations: [], + refsGeneration: 9, + }, + undefined, + 'demo', + ); + + assert.deepEqual(pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined), { + session: 'demo', + ref: '@e2~s9', + }); + assert.deepEqual(pins.pinInput('press', { session: 'demo', ref: '@e37' }, undefined), { + session: 'demo', + ref: '@e37~s7', + }); +}); + +test('ref-pin store leaves pins untouched for an audit without findings', () => { + const pins = makeStore(); + + pins.mergeCommandResult( + 'snapshot', + { nodes: [{ ref: 'e2' }], truncated: false, refsGeneration: 7 }, + undefined, + 'demo', + ); + pins.mergeCommandResult( + 'audit', + { findings: [], checks: { snapshot: 'complete' }, limitations: [] }, + undefined, + 'demo', + ); + + assert.deepEqual(pins.pinInput('press', { session: 'demo', ref: '@e2' }, undefined), { + session: 'demo', + ref: '@e2~s7', + }); +}); + // --- #1638: the generic-route settle commands pin on identical terms --- test('ref-pin store merges per-ref pins from a scroll settle diff', () => { diff --git a/src/mcp/tool-ref-pins.ts b/src/mcp/tool-ref-pins.ts index 98fe7a7800..c2aa37feb3 100644 --- a/src/mcp/tool-ref-pins.ts +++ b/src/mcp/tool-ref-pins.ts @@ -76,7 +76,7 @@ function mergeErrorCandidateRefPins( * (the coarse #1093 warning is the floor). The model never sees or types * suffixes. */ -const REF_ISSUING_TOOLS: ReadonlySet = new Set(['snapshot', 'find'] as const); +const REF_ISSUING_TOOLS: ReadonlySet = new Set(['snapshot', 'find', 'audit'] as const); /** * `--settle` (#1101) makes a response CONDITIONALLY ref-issuing: when it @@ -150,6 +150,8 @@ function mergeCommandResult( if (!REF_ISSUING_TOOLS.has(name)) return; if (name === 'find') { mergeFindRefPins(refPinsByScope, scopeKey, result as CommandExecutionResult<'find'>); + } else if (name === 'audit') { + mergeAuditRefPins(refPinsByScope, scopeKey, result as CommandExecutionResult<'audit'>); } else { mergeSnapshotRefPins(refPinsByScope, scopeKey, result as CommandExecutionResult<'snapshot'>); } @@ -216,6 +218,21 @@ function mergeFindRefPins( mergeIntoScopedPins(refPinsByScope, scopeKey, issued, refsGeneration); } +/** + * `audit` publishes a partial frame for exactly its finding refs (ADR 0014), like a read-only + * `find`. A result without findings carries no generation, issues nothing, and leaves pins alone. + */ +function mergeAuditRefPins( + refPinsByScope: Map>, + scopeKey: string, + result: CommandExecutionResult<'audit'>, +): void { + const refsGeneration = result.refsGeneration; + if (typeof refsGeneration !== 'number') return; + const issued = result.findings.map((finding) => finding.ref.replace(/^@/, '')); + mergeIntoScopedPins(refPinsByScope, scopeKey, issued, refsGeneration); +} + /** * MERGE-ONLY, like the snapshot/find rule: refs on the settled diff's added * lines (plus the unchanged-interactive `tail`, when present) move to the diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index 6ccff8c886..c306bcc0f9 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -13,6 +13,7 @@ import { import { ANDROID_ACTION_BUTTON_RUNTIME_CONTRACT_EVIDENCE, ANDROID_AUDIO_CONTRACT_EVIDENCE, + ANDROID_AUDIT_CONTRACT_EVIDENCE, ANDROID_APPLICATION_LIFECYCLE_CONTRACT_EVIDENCE, ANDROID_FOLD_RUNTIME_CONTRACT_EVIDENCE, ANDROID_HARDWARE_BUTTON_RUNTIME_CONTRACT_EVIDENCE, @@ -749,6 +750,28 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'the command-evidence lane observes a non-empty calculator snapshot mutation', ), }, + [C.audit]: { + androidEmulator: androidEmulator.contract( + ANDROID_AUDIT_CONTRACT_EVIDENCE, + 'audit converts captured pixels to dp with the captured display density', + ), + iosSimulator: iosSimulator.contract( + 'test/integration/provider-scenarios/ios-audit.test.ts', + 'Provider-backed integration audit publishes pinned finding refs and keeps stale plain refs refused', + 'audit publishes pinned finding refs the next press accepts and keeps stale plain refs refused', + ), + macos: macos.gap('No macOS audit evidence exists yet'), + tvos: tvos.gap('No tvOS audit evidence exists yet'), + web: web.contract( + 'packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts', + 'audits web ARIA roles for names and coverage but not size', + 'audit reads web ARIA roles for names and coverage and reports a size limitation', + ), + linux: linux.accepted( + 'No Linux-specific audit evidence exists', + 'Linux rects carry no point or dp unit, so audit would check only names and coverage there, and no lane observes Linux audit findings.', + ), + }, [C.wait]: { androidEmulator: androidEmulator.live( 'smoke:automation-system', diff --git a/test/integration/command-coverage/evidence.ts b/test/integration/command-coverage/evidence.ts index 4b79b73bc8..9b8fb8d831 100644 --- a/test/integration/command-coverage/evidence.ts +++ b/test/integration/command-coverage/evidence.ts @@ -25,6 +25,12 @@ export const ANDROID_LIFECYCLE_CONTRACT_EVIDENCE: AndroidContractEvidence = [C.boot, C.clipboard, C.network, C.reactNative, C.reinstall, C.shutdown, C.triggerAppEvent], 'Provider-backed integration Android Settings flow uses scripted ADB provider', ); +export const ANDROID_AUDIT_CONTRACT_EVIDENCE: AndroidContractEvidence = + defineAndroidContractEvidence( + 'test/integration/provider-scenarios/android-audit.test.ts', + [C.audit], + 'Provider-backed integration Android audit converts captured pixels to dp before judging target size', + ); export const ANDROID_INSTALL_SOURCE_CONTRACT_EVIDENCE: AndroidContractEvidence = defineAndroidContractEvidence( 'src/__tests__/install-source.test.ts', diff --git a/test/integration/provider-scenarios/android-audit.test.ts b/test/integration/provider-scenarios/android-audit.test.ts new file mode 100644 index 0000000000..256bb36058 --- /dev/null +++ b/test/integration/provider-scenarios/android-audit.test.ts @@ -0,0 +1,87 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { ANDROID_IME_HELPER_FIXTURE_ARTIFACT } from '../../../src/__tests__/test-utils/android-ime-helper.ts'; +import { + ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + androidSnapshotHelperOutput, +} from '../../../src/__tests__/test-utils/android-snapshot-helper.ts'; +import type { AndroidAdbProvider } from '@agent-device/platform-android/mechanics'; +import { assertRpcOk } from './assertions.ts'; +import { + androidImeLifecycleAdbResult, + createAndroidProviderShellState, +} from './android-ime-lifecycle-world.ts'; +import { PROVIDER_SCENARIO_ANDROID } from './fixtures.ts'; +import { createProviderScenarioHarness } from './harness.ts'; + +const PIXEL_DENSITY = 2.625; + +// 100px is 38dp at this density: below the 48dp recommendation. 160px is 61dp. +const AUDIT_SCREEN_XML = [ + '', + '', + ' ', + ' ', + ' ', + ' ', + ' ', + '', +].join('\n'); + +test('Provider-backed integration Android audit converts captured pixels to dp before judging target size', async () => { + const ime = createAndroidProviderShellState(); + const adbProvider: AndroidAdbProvider = { + snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + imeHelperArtifact: ANDROID_IME_HELPER_FIXTURE_ARTIFACT, + exec: async (args) => { + const key = args.join(' '); + const imeResult = androidImeLifecycleAdbResult(key, [...args], ime); + if (imeResult) return imeResult; + if (key === 'shell getprop sys.boot_completed') { + return { stdout: '1\n', stderr: '', exitCode: 0 }; + } + if (key === 'shell dumpsys window windows') { + return { + stdout: 'mCurrentFocus=Window{42 u0 com.android.settings/.Settings}\n', + stderr: '', + exitCode: 0, + }; + } + if (key.startsWith('shell am instrument ')) { + return { + stdout: androidSnapshotHelperOutput(AUDIT_SCREEN_XML, { pixelDensity: PIXEL_DENSITY }), + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + }; + const daemon = await createProviderScenarioHarness({ + androidAdbProvider: () => adbProvider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID], + }); + try { + const selection = { platform: 'android' as const, serial: PROVIDER_SCENARIO_ANDROID.id }; + assertRpcOk(await daemon.callCommand('open', ['settings'], selection)); + + const audit = assertRpcOk(await daemon.callCommand('audit', [], selection)); + const findings = audit.findings as Array<{ + kind: string; + detail?: { unit: string; width: number; recommendedSize: number }; + }>; + assert.deepEqual( + findings.map((finding) => finding.kind), + ['small-target', 'missing-name'], + ); + assert.deepEqual(findings[0]?.detail, { + unit: 'dp', + width: 38.1, + height: 38.1, + recommendedSize: 48, + }); + assert.deepEqual(audit.limitations, []); + } finally { + await daemon.close(); + } +}); diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index 7ce7d3cc10..dd816114ed 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -102,6 +102,7 @@ const DRIVEN_COMMANDS: Record = { // -- snapshot-scoped reads -- [PUBLIC_COMMANDS.diff]: () => one(), + [PUBLIC_COMMANDS.audit]: () => one(), [PUBLIC_COMMANDS.wait]: () => one(['text', 'General', '50']), [PUBLIC_COMMANDS.find]: () => one(['label', 'General', 'get', 'attrs']), [PUBLIC_COMMANDS.get]: () => one(['attrs', 'label=General']), diff --git a/test/integration/provider-scenarios/ios-audit.test.ts b/test/integration/provider-scenarios/ios-audit.test.ts new file mode 100644 index 0000000000..bf2459a425 --- /dev/null +++ b/test/integration/provider-scenarios/ios-audit.test.ts @@ -0,0 +1,194 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { assertRpcOk } from './assertions.ts'; +import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; +import type { ProviderScenarioRpcResult } from './harness.ts'; +import { + createAppleRunnerProviderFromTranscript, + createRecordingAppleToolProvider, + simctlDeviceLifecycleHandler, +} from './providers.ts'; +import { createProviderTranscript, type ProviderScenarioProviderEntry } from './transcript.ts'; + +const APP = 'com.example.app'; +const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; + +const SCREEN = { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, +}; + +const FINDING_NODES = [ + SCREEN, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Menu', + hittable: true, + rect: { x: 10, y: 60, width: 30, height: 30 }, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + hittable: true, + rect: { x: 100, y: 200, width: 200, height: 44 }, + }, + { + index: 3, + parentIndex: 0, + type: 'Image', + hittable: true, + rect: { x: 20, y: 300, width: 20, height: 20 }, + }, + { + index: 4, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 400, width: 200, height: 44 }, + }, +]; + +// The same screen with every finding resolved: no small target, every control named. +const CLEAN_NODES = [ + SCREEN, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Menu', + hittable: true, + rect: { x: 10, y: 60, width: 60, height: 60 }, + }, + ...FINDING_NODES.slice(2).map((node) => ({ + ...node, + ...(node.type === 'Button' ? { label: 'Continue' } : {}), + })), +]; + +function snapshotEntry(nodes: unknown[]): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + result: { nodes, truncated: false }, + }; +} + +const TAP_ENTRY: ProviderScenarioProviderEntry = { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + result: {}, +}; + +function refusalReason(response: ProviderScenarioRpcResult): unknown { + const data = response.json?.error?.data as { details?: { reason?: unknown } } | undefined; + return data?.details?.reason; +} + +async function withAuditScenario( + entries: ProviderScenarioProviderEntry[], + run: ( + daemon: Awaited>, + tapRequests: () => Array<{ x?: number; y?: number }>, + ) => Promise, +): Promise { + const runnerTranscript = createProviderTranscript(entries); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlDeviceLifecycleHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => + createAppleRunnerProviderFromTranscript(runnerTranscript, 'ios.runner'), + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + assertRpcOk(await daemon.callCommand('open', [APP], { platform: 'ios', udid: DEVICE_ID })); + await run(daemon, () => + runnerTranscript.calls + .filter((call) => call.command === 'ios.runner.tap') + .map((call) => call.request as { x?: number; y?: number }), + ); + }, + ); +} + +test('Provider-backed integration audit publishes pinned finding refs and keeps stale plain refs refused', async () => { + await withAuditScenario( + [ + // The first capture after open settles over a quiet pair of reads of the same screen. + snapshotEntry(FINDING_NODES), + snapshotEntry(FINDING_NODES), + TAP_ENTRY, + snapshotEntry(FINDING_NODES), + TAP_ENTRY, + ], + async (daemon, tapRequests) => { + assertRpcOk(await daemon.callCommand('snapshot')); + // The press expires the complete frame the snapshot issued. + assertRpcOk(await daemon.callCommand('press', ['@e4'])); + + const audit = assertRpcOk(await daemon.callCommand('audit')); + const findings = audit.findings as Array<{ kind: string; ref: string }>; + assert.deepEqual( + findings.map(({ kind, ref }) => [kind, ref]), + [ + ['small-target', '@e2'], + ['missing-name', '@e3'], + ], + ); + const generation = audit.refsGeneration; + assert.equal(typeof generation, 'number'); + + // Audit publishes a partial frame: a plain finding ref and a ref it never listed stay refused. + assert.equal( + refusalReason(await daemon.callCommand('press', ['@e2'])), + 'plain_ref_requires_complete_frame', + ); + assert.equal( + refusalReason(await daemon.callCommand('press', [`@e4~s${generation}`])), + 'ref_not_issued', + ); + + // The pinned finding ref presses the Menu button, the node the finding names. + assertRpcOk(await daemon.callCommand('press', [`@e2~s${generation}`])); + const taps = tapRequests(); + assert.equal(taps.length, 2); + assert.deepEqual([taps[1]?.x, taps[1]?.y], [25, 75]); + }, + ); +}); + +test('Provider-backed integration an audit without findings leaves an expired frame expired', async () => { + await withAuditScenario( + [ + snapshotEntry(FINDING_NODES), + snapshotEntry(FINDING_NODES), + TAP_ENTRY, + snapshotEntry(CLEAN_NODES), + ], + async (daemon) => { + assertRpcOk(await daemon.callCommand('snapshot')); + assertRpcOk(await daemon.callCommand('press', ['@e4'])); + + const audit = assertRpcOk(await daemon.callCommand('audit')); + assert.deepEqual(audit.findings, []); + assert.equal(audit.refsGeneration, undefined); + + // A plain ref remembered from the snapshot is still refused: the audit issued nothing. + assert.equal(refusalReason(await daemon.callCommand('press', ['@e2'])), 'ref_frame_expired'); + }, + ); +}); diff --git a/test/integration/smoke-linux-coverage.test.ts b/test/integration/smoke-linux-coverage.test.ts index 273de99542..cb960c9ab3 100644 --- a/test/integration/smoke-linux-coverage.test.ts +++ b/test/integration/smoke-linux-coverage.test.ts @@ -120,6 +120,7 @@ test('Linux accepted-gap rows are exactly the pinned command list', () => { .sort(); assert.deepEqual(acceptedCommands, [ 'artifacts', + 'audit', 'install', 'install-from-source', 'logs', diff --git a/website/docs/docs/capabilities.md b/website/docs/docs/capabilities.md index f380545e69..95a4fc99c9 100644 --- a/website/docs/docs/capabilities.md +++ b/website/docs/docs/capabilities.md @@ -26,7 +26,7 @@ A provider column answers for that provider in its default configuration. Provid | Capability and reference | iOS Simulator | iOS device (CoreDevice) | Android emulator | Android device | HarmonyOS emulator | HarmonyOS device | | --- | --- | --- | --- | --- | --- | --- | -| [Accessibility snapshot and snapshot diff](/docs/snapshots) | Yes | Yes | Yes | Yes | Yes | Yes | +| [Accessibility snapshot, snapshot diff, and audit](/docs/snapshots) | Yes | Yes | Yes | Yes | Yes | Yes | | [Refs, selectors, element lookup, and attribute reads](/docs/selectors) | Yes | Yes | Yes | Yes | Yes | Yes | | [Screenshots](/docs/commands#media-and-logs) | Yes | Yes | Yes | Yes | Yes | Yes | | [Wait for text, selectors, and timeouts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | Yes | Yes | @@ -35,7 +35,7 @@ A provider column answers for that provider in its default configuration. Provid | Capability and reference | Android TV emulator | Android TV / Fire OS device | tvOS Simulator | tvOS device | Vega OS (VVD) | | --- | --- | --- | --- | --- | --- | -| [Accessibility snapshot and snapshot diff](/docs/snapshots) | Yes | Yes | Yes | Yes | No | +| [Accessibility snapshot, snapshot diff, and audit](/docs/snapshots) | Yes | Yes | Yes | Yes | No | | [Refs, selectors, element lookup, and attribute reads](/docs/selectors) | Yes | Yes | Yes | Yes | No | | [Screenshots](/docs/commands#media-and-logs) | Yes | Yes | Yes | Yes | No | | [Wait for text, selectors, and timeouts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | No | @@ -44,7 +44,7 @@ A provider column answers for that provider in its default configuration. Provid | Capability and reference | macOS desktop | macOS app | Linux desktop | Web browser | | --- | --- | --- | --- | --- | -| [Accessibility snapshot and snapshot diff](/docs/snapshots) | Yes | Yes | Yes | Yes | +| [Accessibility snapshot, snapshot diff, and audit](/docs/snapshots) | Yes | Yes | Yes | Yes | | [Refs, selectors, element lookup, and attribute reads](/docs/selectors) | Yes | Yes | Yes | Yes | | [Screenshots](/docs/commands#media-and-logs) | Yes | Yes | Yes | Yes | | [Wait for text, selectors, and timeouts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | @@ -53,14 +53,14 @@ A provider column answers for that provider in its default configuration. Provid | Capability and reference | BrowserStack Android / TestMu AI Android | BrowserStack iOS / TestMu AI iOS | AWS Device Farm Android | AWS Device Farm iOS | Limrun iOS | Limrun Android | Revyl iOS | Revyl Android | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| [Accessibility snapshot and snapshot diff](/docs/snapshots) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | +| [Accessibility snapshot, snapshot diff, and audit](/docs/snapshots) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | [Refs, selectors, element lookup, and attribute reads](/docs/selectors) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | [Screenshots](/docs/commands#media-and-logs) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | [Wait for text, selectors, and timeouts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | What a **Yes** does not claim: -- **Accessibility snapshot and snapshot diff**: A capture returns a tree only when the app or desktop exposes an accessibility tree. +- **Accessibility snapshot, snapshot diff, and audit**: A capture returns a tree only when the app or desktop exposes an accessibility tree. ## Interact with the app diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index da4461f90d..e2bfbd76f5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -422,6 +422,22 @@ agent-device get attrs @e1 - `--actions` lists custom accessibility actions an element merged away (iOS simulator only, rejected with `--raw`); see [Snapshots](/docs/snapshots). - `diff snapshot` (alias `snapshot --diff`) compares with the previous session baseline, then updates it. It compares only trees read with the same `-i`, `-d`, `-s`, `--raw` and `--actions`: when the stored snapshot was read in another view, the capture starts a new baseline (`baselineInitialized: true`) instead of listing the view change as changes. +### Audit the current screen + +```bash +agent-device audit +``` + +`audit` captures the current screen once and reports advisory accessibility findings. Findings are review prompts, not failures, and `audit` changes nothing on the device. + +- `small-target`: an enabled, visible control below 44pt on iOS or 48dp on Android, with a 0.5-unit tolerance for layout rounding. Android pixels convert to dp with the captured display density; without it the check reports a limitation instead. TV and desktop targets skip the check. A link inside a text container is exempt. +- `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. +- `covered-control`: a control or row the snapshot already marks `[covered]`. + +Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars and keyboard keys are skipped. A control that is the only child of a same-role control reports only through the parent. + +Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. `--json` returns `findings`, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. + ## Wait and alerts ```bash