Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/revyl-ios-screen-origin.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'agent-device': patch
---

Revyl iOS taps by ref work again while the interface is in the orientation Revyl reported its screen in. Since 0.21.37 they failed with `window-origin-unknown`, because the viewport of a Revyl iOS snapshot came from the tree's root frame and was never reported. A root frame that is exactly the reported screen, at the origin, now establishes the screen origin. Any other root, including one with width and height swapped after a rotation, still refuses coordinate-derived actions.
21 changes: 21 additions & 0 deletions packages/capture-kit/src/ios-ax-tree.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,27 @@ test('without a usable root the device screen is the viewport, and a bad screen
).toEqual({ kind: 'derived', rect: { x: 0, y: 0, width: 320, height: 240 } });
});

test('a root frame is reported only when it is the provider screen at the origin', () => {
const screen = { width: 440, height: 956 };
const root = (frame: Record<string, number>) => [{ type: 'Application', frame }];
expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 440, height: 956 }), screen)).toEqual({
kind: 'reported',
rect: { x: 0, y: 0, width: 440, height: 956 },
});
expect(resolveIosAxTreeViewport(root({ x: 0, y: 20, width: 440, height: 956 }), screen)).toEqual({
kind: 'derived',
rect: { x: 0, y: 20, width: 440, height: 956 },
});
expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 956, height: 440 }), screen)).toEqual({
kind: 'derived',
rect: { x: 0, y: 0, width: 956, height: 440 },
});
expect(resolveIosAxTreeViewport(root({ x: 0, y: 0, width: 440, height: 956 }))).toEqual({
kind: 'derived',
rect: { x: 0, y: 0, width: 440, height: 956 },
});
});

test('a screen box with one dimension is invalid, and a null role description is omitted', () => {
expect(resolveIosAxTreeViewport([{ type: 'Application' }], { width: 440 })).toEqual({
kind: 'missing',
Expand Down
23 changes: 20 additions & 3 deletions packages/capture-kit/src/ios-ax-tree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,9 @@ function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] {
/**
* The viewport an AX tree acquisition uses: the tree's own application or window root when it reports a
* usable box, because every node rect in the tree was measured in it (`derived`); otherwise the screen
* size the provider reported for the device (`reported`), in the same points. Without either, the
* tree's own reason the box is missing.
* size the provider reported for the device (`reported`), in the same points. A root that is exactly that
* screen at the origin is `reported`: two independent reads agree that window space is screen space.
* Without either box, the tree's own reason the box is missing.
*/
export function resolveIosAxTreeViewport(
tree: IosTreeNode | IosTreeNode[],
Expand All @@ -154,7 +155,11 @@ export function resolveIosAxTreeViewport(
const evidence = resolveIosViewportEvidenceFromRoots(
(Array.isArray(tree) ? tree : [tree]).map(viewportRoot),
);
if (evidence?.kind === 'reported') return { kind: 'derived', rect: evidence.rect };
if (evidence?.kind === 'reported') {
return isScreenAtOrigin(evidence.rect, screen)
? { kind: 'reported', rect: evidence.rect }
: { kind: 'derived', rect: evidence.rect };
}
if (screen !== undefined && (screen.width !== undefined || screen.height !== undefined)) {
return isFiniteNumber(screen.width) &&
isFiniteNumber(screen.height) &&
Expand All @@ -166,6 +171,18 @@ export function resolveIosAxTreeViewport(
return evidence ?? { kind: 'missing', reason: 'not-provided' };
}

// CONSERVATIVE: a root whose size is the screen's with width and height swapped stays `derived`. The
// provider reads its screen size once, and no live landscape run has confirmed which coordinate space
// taps use after a rotation; revisit once one has.
function isScreenAtOrigin(
rect: Readonly<{ x: number; y: number; width: number; height: number }>,
screen: Readonly<{ width?: number; height?: number }> | undefined,
): boolean {
return (
rect.x === 0 && rect.y === 0 && rect.width === screen?.width && rect.height === screen.height
);
}

/**
* The root as `resolveIosViewportEvidenceFromRoots` reads it: its type, its usable rect when it has
* one, and whether the box it reports is absent, reported, or unusable. A frame that is present but
Expand Down
14 changes: 14 additions & 0 deletions packages/provider-revyl/src/interactor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,20 @@ test('the iOS snapshot reads the AX node tree Revyl answers with, in the root fr
assert.equal(acquisition.nodes[0]?.type, 'Application');
});

test('an iOS root frame that is exactly the health screen establishes the screen origin', async () => {
const { interactor } = await liveInteractor('ios', {
screen: { width: 440, height: 956 },
hierarchy: REVYL_IOS_HIERARCHY,
});
const result = await interactor.snapshot({});
if (!('stage' in result) || result.stage !== 'acquired')
throw new Error('Expected an iOS acquisition');
assert.deepEqual(result.acquisition.viewport, {
kind: 'reported',
rect: { x: 0, y: 0, width: 440, height: 956 },
});
});

for (const [name, hierarchy] of [
['XML', '<AppiumAUT><XCUIElementTypeApplication/></AppiumAUT>'],
['a null node', '[null]'],
Expand Down
Loading