diff --git a/.changeset/revyl-ios-screen-origin.md b/.changeset/revyl-ios-screen-origin.md new file mode 100644 index 0000000000..4febde6db1 --- /dev/null +++ b/.changeset/revyl-ios-screen-origin.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +Revyl iOS taps by ref work again while the interface is in the orientation Revyl reported its screen in. Since 0.21.37 they failed with `window-origin-unknown`, because the viewport of a Revyl iOS snapshot came from the tree's root frame and was never reported. A root frame that is exactly the reported screen, at the origin, now establishes the screen origin. Any other root, including one with width and height swapped after a rotation, still refuses coordinate-derived actions. diff --git a/packages/capture-kit/src/ios-ax-tree.test.ts b/packages/capture-kit/src/ios-ax-tree.test.ts index 450a015f0b..0105c9806f 100644 --- a/packages/capture-kit/src/ios-ax-tree.test.ts +++ b/packages/capture-kit/src/ios-ax-tree.test.ts @@ -99,6 +99,27 @@ test('without a usable root the device screen is the viewport, and a bad screen ).toEqual({ kind: 'derived', rect: { x: 0, y: 0, width: 320, height: 240 } }); }); +test('a root frame is reported only when it is the provider screen at the origin', () => { + const screen = { width: 440, height: 956 }; + const root = (frame: Record) => [{ type: 'Application', frame }]; + expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 440, height: 956 }), screen)).toEqual({ + kind: 'reported', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); + expect(resolveIosAxTreeViewport(root({ x: 0, y: 20, width: 440, height: 956 }), screen)).toEqual({ + kind: 'derived', + rect: { x: 0, y: 20, width: 440, height: 956 }, + }); + expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 956, height: 440 }), screen)).toEqual({ + kind: 'derived', + rect: { x: 0, y: 0, width: 956, height: 440 }, + }); + expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 440, height: 956 }))).toEqual({ + kind: 'derived', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); +}); + test('a screen box with one dimension is invalid, and a null role description is omitted', () => { expect(resolveIosAxTreeViewport([{ type: 'Application' }], { width: 440 })).toEqual({ kind: 'missing', diff --git a/packages/capture-kit/src/ios-ax-tree.ts b/packages/capture-kit/src/ios-ax-tree.ts index b8d55e4557..faecbe75e9 100644 --- a/packages/capture-kit/src/ios-ax-tree.ts +++ b/packages/capture-kit/src/ios-ax-tree.ts @@ -144,8 +144,9 @@ function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] { /** * The viewport an AX tree acquisition uses: the tree's own application or window root when it reports a * usable box, because every node rect in the tree was measured in it (`derived`); otherwise the screen - * size the provider reported for the device (`reported`), in the same points. Without either, the - * tree's own reason the box is missing. + * size the provider reported for the device (`reported`), in the same points. A root that is exactly that + * screen at the origin is `reported`: two independent reads agree that window space is screen space. + * Without either box, the tree's own reason the box is missing. */ export function resolveIosAxTreeViewport( tree: IosTreeNode | IosTreeNode[], @@ -154,7 +155,11 @@ export function resolveIosAxTreeViewport( const evidence = resolveIosViewportEvidenceFromRoots( (Array.isArray(tree) ? tree : [tree]).map(viewportRoot), ); - if (evidence?.kind === 'reported') return { kind: 'derived', rect: evidence.rect }; + if (evidence?.kind === 'reported') { + return isScreenAtOrigin(evidence.rect, screen) + ? { kind: 'reported', rect: evidence.rect } + : { kind: 'derived', rect: evidence.rect }; + } if (screen !== undefined && (screen.width !== undefined || screen.height !== undefined)) { return isFiniteNumber(screen.width) && isFiniteNumber(screen.height) && @@ -166,6 +171,18 @@ export function resolveIosAxTreeViewport( return evidence ?? { kind: 'missing', reason: 'not-provided' }; } +// CONSERVATIVE: a root whose size is the screen's with width and height swapped stays `derived`. The +// provider reads its screen size once, and no live landscape run has confirmed which coordinate space +// taps use after a rotation; revisit once one has. +function isScreenAtOrigin( + rect: Readonly<{ x: number; y: number; width: number; height: number }>, + screen: Readonly<{ width?: number; height?: number }> | undefined, +): boolean { + return ( + rect.x === 0 && rect.y === 0 && rect.width === screen?.width && rect.height === screen.height + ); +} + /** * The root as `resolveIosViewportEvidenceFromRoots` reads it: its type, its usable rect when it has * one, and whether the box it reports is absent, reported, or unusable. A frame that is present but diff --git a/packages/provider-revyl/src/interactor.test.ts b/packages/provider-revyl/src/interactor.test.ts index bc442bb50b..b92b68bcf4 100644 --- a/packages/provider-revyl/src/interactor.test.ts +++ b/packages/provider-revyl/src/interactor.test.ts @@ -454,6 +454,20 @@ test('the iOS snapshot reads the AX node tree Revyl answers with, in the root fr assert.equal(acquisition.nodes[0]?.type, 'Application'); }); +test('an iOS root frame that is exactly the health screen establishes the screen origin', async () => { + const { interactor } = await liveInteractor('ios', { + screen: { width: 440, height: 956 }, + hierarchy: REVYL_IOS_HIERARCHY, + }); + const result = await interactor.snapshot({}); + if (!('stage' in result) || result.stage !== 'acquired') + throw new Error('Expected an iOS acquisition'); + assert.deepEqual(result.acquisition.viewport, { + kind: 'reported', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); +}); + for (const [name, hierarchy] of [ ['XML', ''], ['a null node', '[null]'],