From ba088e3fd1761cb6438f1f5cc6f66fdcd7a3cc9e Mon Sep 17 00:00:00 2001 From: KayaanT Date: Fri, 13 Nov 2026 15:12:57 -0500 Subject: [PATCH] fix(revyl): establish the iOS screen origin when the root frame matches the screen 0.21.37 refuses a tap placed from snapshot rects unless the capture's viewport is reported. resolveIosAxTreeViewport returned derived for any usable tree root, so on Revyl, its only caller, every iOS ref tap failed with window-origin-unknown even when the root was exactly the health screen at the origin. A root that is the provider's screen at the origin is now reported; any other root, including a rotated one, stays derived. --- .changeset/revyl-ios-screen-origin.md | 5 ++++ packages/capture-kit/src/ios-ax-tree.test.ts | 21 +++++++++++++++++ packages/capture-kit/src/ios-ax-tree.ts | 23 ++++++++++++++++--- .../provider-revyl/src/interactor.test.ts | 14 +++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 .changeset/revyl-ios-screen-origin.md diff --git a/.changeset/revyl-ios-screen-origin.md b/.changeset/revyl-ios-screen-origin.md new file mode 100644 index 0000000000..4febde6db1 --- /dev/null +++ b/.changeset/revyl-ios-screen-origin.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +Revyl iOS taps by ref work again while the interface is in the orientation Revyl reported its screen in. Since 0.21.37 they failed with `window-origin-unknown`, because the viewport of a Revyl iOS snapshot came from the tree's root frame and was never reported. A root frame that is exactly the reported screen, at the origin, now establishes the screen origin. Any other root, including one with width and height swapped after a rotation, still refuses coordinate-derived actions. diff --git a/packages/capture-kit/src/ios-ax-tree.test.ts b/packages/capture-kit/src/ios-ax-tree.test.ts index 450a015f0b..0105c9806f 100644 --- a/packages/capture-kit/src/ios-ax-tree.test.ts +++ b/packages/capture-kit/src/ios-ax-tree.test.ts @@ -99,6 +99,27 @@ test('without a usable root the device screen is the viewport, and a bad screen ).toEqual({ kind: 'derived', rect: { x: 0, y: 0, width: 320, height: 240 } }); }); +test('a root frame is reported only when it is the provider screen at the origin', () => { + const screen = { width: 440, height: 956 }; + const root = (frame: Record) => [{ type: 'Application', frame }]; + expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 440, height: 956 }), screen)).toEqual({ + kind: 'reported', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); + expect(resolveIosAxTreeViewport(root({ x: 0, y: 20, width: 440, height: 956 }), screen)).toEqual({ + kind: 'derived', + rect: { x: 0, y: 20, width: 440, height: 956 }, + }); + expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 956, height: 440 }), screen)).toEqual({ + kind: 'derived', + rect: { x: 0, y: 0, width: 956, height: 440 }, + }); + expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 440, height: 956 }))).toEqual({ + kind: 'derived', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); +}); + test('a screen box with one dimension is invalid, and a null role description is omitted', () => { expect(resolveIosAxTreeViewport([{ type: 'Application' }], { width: 440 })).toEqual({ kind: 'missing', diff --git a/packages/capture-kit/src/ios-ax-tree.ts b/packages/capture-kit/src/ios-ax-tree.ts index b8d55e4557..faecbe75e9 100644 --- a/packages/capture-kit/src/ios-ax-tree.ts +++ b/packages/capture-kit/src/ios-ax-tree.ts @@ -144,8 +144,9 @@ function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] { /** * The viewport an AX tree acquisition uses: the tree's own application or window root when it reports a * usable box, because every node rect in the tree was measured in it (`derived`); otherwise the screen - * size the provider reported for the device (`reported`), in the same points. Without either, the - * tree's own reason the box is missing. + * size the provider reported for the device (`reported`), in the same points. A root that is exactly that + * screen at the origin is `reported`: two independent reads agree that window space is screen space. + * Without either box, the tree's own reason the box is missing. */ export function resolveIosAxTreeViewport( tree: IosTreeNode | IosTreeNode[], @@ -154,7 +155,11 @@ export function resolveIosAxTreeViewport( const evidence = resolveIosViewportEvidenceFromRoots( (Array.isArray(tree) ? tree : [tree]).map(viewportRoot), ); - if (evidence?.kind === 'reported') return { kind: 'derived', rect: evidence.rect }; + if (evidence?.kind === 'reported') { + return isScreenAtOrigin(evidence.rect, screen) + ? { kind: 'reported', rect: evidence.rect } + : { kind: 'derived', rect: evidence.rect }; + } if (screen !== undefined && (screen.width !== undefined || screen.height !== undefined)) { return isFiniteNumber(screen.width) && isFiniteNumber(screen.height) && @@ -166,6 +171,18 @@ export function resolveIosAxTreeViewport( return evidence ?? { kind: 'missing', reason: 'not-provided' }; } +// CONSERVATIVE: a root whose size is the screen's with width and height swapped stays `derived`. The +// provider reads its screen size once, and no live landscape run has confirmed which coordinate space +// taps use after a rotation; revisit once one has. +function isScreenAtOrigin( + rect: Readonly<{ x: number; y: number; width: number; height: number }>, + screen: Readonly<{ width?: number; height?: number }> | undefined, +): boolean { + return ( + rect.x === 0 && rect.y === 0 && rect.width === screen?.width && rect.height === screen.height + ); +} + /** * The root as `resolveIosViewportEvidenceFromRoots` reads it: its type, its usable rect when it has * one, and whether the box it reports is absent, reported, or unusable. A frame that is present but diff --git a/packages/provider-revyl/src/interactor.test.ts b/packages/provider-revyl/src/interactor.test.ts index bc442bb50b..b92b68bcf4 100644 --- a/packages/provider-revyl/src/interactor.test.ts +++ b/packages/provider-revyl/src/interactor.test.ts @@ -454,6 +454,20 @@ test('the iOS snapshot reads the AX node tree Revyl answers with, in the root fr assert.equal(acquisition.nodes[0]?.type, 'Application'); }); +test('an iOS root frame that is exactly the health screen establishes the screen origin', async () => { + const { interactor } = await liveInteractor('ios', { + screen: { width: 440, height: 956 }, + hierarchy: REVYL_IOS_HIERARCHY, + }); + const result = await interactor.snapshot({}); + if (!('stage' in result) || result.stage !== 'acquired') + throw new Error('Expected an iOS acquisition'); + assert.deepEqual(result.acquisition.viewport, { + kind: 'reported', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); +}); + for (const [name, hierarchy] of [ ['XML', ''], ['a null node', '[null]'],