Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 158 additions & 0 deletions packages/capture-kit/src/snapshot-occlusion-bottom-tabs.fixture.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
[
{
"index": 0,
"type": "Application",
"role": "UIApplication",
"label": "e2e Benchmark",
"rect": {
"x": 0,
"y": 0,
"width": 402,
"height": 874
},
"enabled": true,
"depth": 0,
"hittable": false
},
{
"index": 1,
"parentIndex": 0,
"type": "StaticText",
"role": "RCTParagraphComponentView",
"subrole": "UIView",
"label": "Home",
"rect": {
"x": 177.3333282470703,
"y": 71,
"width": 47.333343505859375,
"height": 20.333335876464844
},
"enabled": true,
"depth": 1,
"hittable": true
},
{
"index": 2,
"parentIndex": 0,
"type": "StaticText",
"role": "RCTParagraphComponentView",
"subrole": "UIView",
"label": "Home",
"identifier": "home-tab-content",
"rect": {
"x": 173.3333282470703,
"y": 422.00000762939453,
"width": 55,
"height": 24
},
"enabled": true,
"depth": 1,
"hittable": true
},
{
"index": 3,
"parentIndex": 0,
"type": "StaticText",
"role": "RCTParagraphComponentView",
"subrole": "UIView",
"label": "Go to the Actions tab and press the button there.",
"rect": {
"x": 52.66666793823242,
"y": 454.00000762939453,
"width": 296.66668701171875,
"height": 15.666656494140625
},
"enabled": true,
"depth": 1,
"hittable": true
},
{
"index": 4,
"parentIndex": 0,
"type": "Toolbar",
"role": "_UIFloatingBarContainerView",
"label": "Toolbar",
"identifier": "Toolbar",
"rect": {
"x": 0,
"y": 0,
"width": 402,
"height": 791
},
"enabled": true,
"depth": 1,
"hittable": true
},
{
"index": 5,
"parentIndex": 0,
"type": "Button",
"role": "RCTViewComponentView",
"subrole": "UIView",
"label": "Home, tab, 1 of 3",
"identifier": "tab-home",
"rect": {
"x": 0,
"y": 791.3333333432674,
"width": 134,
"height": 48.66668701171875
},
"enabled": true,
"selected": true,
"depth": 1,
"hittable": true
},
{
"index": 6,
"parentIndex": 0,
"type": "Button",
"role": "RCTViewComponentView",
"subrole": "UIView",
"label": "Feed, tab, 2 of 3",
"identifier": "tab-feed",
"rect": {
"x": 134,
"y": 791.3333333432674,
"width": 134,
"height": 48.66668701171875
},
"enabled": true,
"depth": 1,
"hittable": true
},
{
"index": 7,
"parentIndex": 0,
"type": "Button",
"role": "RCTViewComponentView",
"subrole": "UIView",
"label": "Actions, tab, 3 of 3",
"identifier": "tab-actions",
"rect": {
"x": 268,
"y": 791.3333333432674,
"width": 134,
"height": 48.66668701171875
},
"enabled": true,
"depth": 1,
"hittable": true
},
{
"index": 8,
"parentIndex": 0,
"type": "Toolbar",
"role": "_UIFloatingBarContainerView",
"label": "Toolbar",
"identifier": "Toolbar",
"rect": {
"x": 0,
"y": 0,
"width": 402,
"height": 874
},
"enabled": true,
"depth": 1,
"hittable": true
}
]
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { test } from 'vitest';
import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
import { annotateCoveredSnapshotNodes } from './snapshot-occlusion.ts';
Expand Down Expand Up @@ -134,10 +135,6 @@ function targetVerdict(nodes: RawSnapshotNode[]): RawSnapshotNode | undefined {
}

test("a chrome container sized to its content sibling branch does not cover that branch's targets (#2996)", () => {
// The reporter's capture: the container is 791pt tall, matching the content branch beside it and
// stopping 83pt short of the 874pt viewport the existing exemption keys on. The four publications
// of the chrome kind are pinned by the table below, so this pins the geometry and the target's own
// readback.
const annotated = targetVerdict(
chromeHostTree({ branchRect: BRANCH, chromeRole: '_UIFloatingBarContainerView' }),
);
Expand All @@ -147,9 +144,6 @@ test("a chrome container sized to its content sibling branch does not cover that
});

test('a control the exempt container really hosts still covers what it overlaps (#2996)', () => {
// The exemption excuses the container's OWN box, not its subtree: a hosted strip with its own
// smaller rect is unrelated to the target and differs from the branch frame, so it keeps covering
// the content beneath it — the fix cannot trade false refusals for missed real occlusion.
const nodes = chromeHostTree({
targetRect: { x: 151, y: 700, width: 100, height: 32 },
chromeRole: '_UIFloatingBarContainerView',
Expand All @@ -171,9 +165,6 @@ test('a control the exempt container really hosts still covers what it overlaps
});

test('a chrome candidate whose frame differs from the content branch beside it still covers (#2996 non-vacuity)', () => {
// Isolates frame-EQUALITY from mere overlap: the container stops 60pt short of the branch, so it
// matches neither the branch nor the viewport, and its box is real covering evidence over the
// target in the region they share.
const nodes = chromeHostTree({
branchRect: BRANCH,
chromeRect: { x: 0, y: 0, width: 402, height: 731 },
Expand All @@ -184,9 +175,6 @@ test('a chrome candidate whose frame differs from the content branch beside it s
});

test('a chrome candidate taller than the content branch beside it still covers, so equality is not overlap (#2996 containment)', () => {
// The mirror must be the SAME frame, not merely containing it. The container reaches 91pt below
// the branch while stopping short of the viewport, so it paints over the branch's lower edge and
// keeps covering the target inside that edge.
const nodes = chromeHostTree({
branchRect: { x: 0, y: 0, width: 402, height: 700 },
chromeRect: BRANCH,
Expand All @@ -198,8 +186,6 @@ test('a chrome candidate taller than the content branch beside it still covers,
});

test('a full-screen dialog sized to the content branch beside it still covers, because only chrome kinds are exempt (#2996 kind gate)', () => {
// A `Dialog` sibling that fills the branch is a real modal presentation over that branch. Reading
// the exemption from geometry alone would let it through, so it stays kind-gated.
const nodes = chromeHostTree({
chromeType: 'Dialog',
chromeRole: 'UIAlertControllerView',
Expand All @@ -210,9 +196,6 @@ test('a full-screen dialog sized to the content branch beside it still covers, b
});

test('a chrome candidate in another window keeps covering content sized like its own frame (#2996 sibling scope)', () => {
// The exemption compares a candidate against the siblings listed under ITS OWN parent. This
// container's only sibling is its own full-window host, so nothing matches and it keeps covering
// — a same-size rect elsewhere in the tree cannot excuse it.
const nodes: RawSnapshotNode[] = [
node({ index: 0, depth: 0, type: 'Application', role: 'UIApplication', rect: VIEWPORT }),
node({
Expand Down Expand Up @@ -925,3 +908,75 @@ test('a control the exempt toolbar host really hosts still covers the row it ove

assert.deepEqual(coveredLabels(nodes), ['Barless row 12']);
});

function projectedTabsTree(): RawSnapshotNode[] {
return JSON.parse(
readFileSync(new URL('./snapshot-occlusion-bottom-tabs.fixture.json', import.meta.url), 'utf8'),
) as RawSnapshotNode[];
}

test('projected tab content remains uncovered above the adjoining bottom tab buttons (#4365)', () => {
for (const type of ['Button', 'TabItem']) {
const nodes = projectedTabsTree();
for (const index of [5, 6, 7]) nodes[index]!.type = type;
const annotated = annotateCoveredSnapshotNodes(nodes);
for (const index of [1, 2, 3, 5, 6, 7]) {
assert.equal(
annotated.find((candidate) => candidate.index === index)?.interactionBlocked,
undefined,
);
}
}
});

test('a real toolbar over content remains covering despite an adjoining control (#4365)', () => {
const nodes = projectedTabsTree();
nodes[4]!.rect = { x: 0, y: 0, width: 402, height: 300 };
nodes[5]!.rect = { x: 0, y: 300, width: 134, height: 49 };
assert.equal(annotateCoveredSnapshotNodes(nodes)[1]?.interactionBlocked, 'covered');
});

test('a control hosted by the projected floating container still covers content (#4365)', () => {
const nodes = projectedTabsTree();
nodes.splice(
5,
0,
node({
index: 9,
parentIndex: 4,
depth: 2,
type: 'Toolbar',
label: 'Action',
rect: { x: 160, y: 60, width: 100, height: 50 },
hittable: true,
}),
);
const target = annotateCoveredSnapshotNodes(nodes).find((candidate) => candidate.index === 1);
assert.equal(target?.interactionBlocked, 'covered');
assert.equal(target?.coveredBy?.label, 'Action');
});

test('a sheet with the same footprint and adjoining tab buttons still covers content (#4365)', () => {
const nodes = projectedTabsTree();
nodes[4]!.type = 'Sheet';
assert.equal(annotateCoveredSnapshotNodes(nodes)[1]?.interactionBlocked, 'covered');
});

test('a single bottom CTA does not exempt a covering toolbar (#4365)', () => {
const nodes = projectedTabsTree();
nodes.splice(6, 2);
nodes[5]!.label = 'Continue';
nodes[5]!.rect = { x: 0, y: 791, width: 402, height: 49 };
assert.equal(annotateCoveredSnapshotNodes(nodes)[1]?.interactionBlocked, 'covered');
});

test('an ordinary toolbar above a two-button action row still covers content (#4365)', () => {
const nodes = projectedTabsTree();
nodes[4]!.role = '';
nodes.splice(7, 1);
nodes[5]!.rect = { x: 0, y: 791, width: 201, height: 49 };
nodes[6]!.rect = { x: 201, y: 791, width: 201, height: 49 };
nodes[5]!.label = 'Cancel';
nodes[6]!.label = 'Confirm';
assert.equal(annotateCoveredSnapshotNodes(nodes)[1]?.interactionBlocked, 'covered');
});
54 changes: 51 additions & 3 deletions packages/capture-kit/src/snapshot-occlusion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const OVERLAY_KIND_FRAGMENTS = [
'menu',
];
const VIEWPORT_CHROME_KIND_FRAGMENTS = ['tabbar', 'toolbar', 'navigationbar'];
const FLOATING_BAR_CONTAINER_ROLE = '_UIFloatingBarContainerView';
/**
* How far below the viewport top a full-viewport chrome container may start, before the 0.5pt rect
* tolerance. On iOS 27 an app that draws a 2pt bar at the top gets a `_UIFloatingBarContainerView`
Expand Down Expand Up @@ -334,7 +335,7 @@ function isOverlayLikeNode(
): boolean {
if (!positiveRect(node.rect)) return false;
if (isViewportRootNode(node)) return false;
if (isFullViewportChromeContainer(node, neighbourhood.byIndex)) return false;
if (isFullViewportChromeContainer(node, neighbourhood)) return false;
if (isSiblingSizedChromeContainer(node, neighbourhood)) return false;
if (isOwnPresentationToolbarHost(node, neighbourhood)) return false;
// This is a presentation-order heuristic: only known floating UI chrome should cover
Expand Down Expand Up @@ -501,18 +502,23 @@ function isEnclosedHostedRegion(rect: Rect, node: RawSnapshotNode): boolean {

function isFullViewportChromeContainer(
node: RawSnapshotNode,
byIndex: Map<number, RawSnapshotNode>,
neighbourhood: NeighbourhoodIndex,
): boolean {
if (!nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)) return false;
const rect = positiveRect(node.rect);
if (!rect) return false;

const byIndex = neighbourhood.byIndex;
let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined;
const visited = new Set<number>();
while (current && !visited.has(current.index)) {
if (isViewportRootNode(current)) {
const viewportRect = positiveRect(current.rect);
return Boolean(viewportRect && isViewportFootprint(rect, viewportRect));
return Boolean(
viewportRect &&
(isViewportFootprint(rect, viewportRect) ||
isViewportAboveBottomBar(node, rect, viewportRect, neighbourhood)),
);
}
visited.add(current.index);
current =
Expand All @@ -521,6 +527,48 @@ function isFullViewportChromeContainer(
return false;
}

// The floating host can end at a complete bottom tab row after projection removes its content sibling.
function isViewportAboveBottomBar(
node: RawSnapshotNode,
rect: Rect,
viewport: Rect,
neighbourhood: NeighbourhoodIndex,
): boolean {
if (node.role !== FLOATING_BAR_CONTAINER_ROLE || typeof node.parentIndex !== 'number')
return false;
const bottom = rect.y + rect.height;
if (bottom >= viewport.y + viewport.height) return false;
if (!isViewportFootprint(rect, { ...viewport, height: bottom - viewport.y })) return false;
const siblings = neighbourhood.childrenByParent.get(node.parentIndex) ?? [];
const buttons = siblings.flatMap((sibling) => {
if (!nodeKindIncludesAny(sibling, ['button', 'tabitem'])) return [];
const bar = positiveRect(sibling.rect);
if (!bar) return [];
return Math.abs(bar.y - bottom) <= 0.5 &&
bar.x >= viewport.x &&
bar.x + bar.width <= viewport.x + viewport.width &&
bar.height <= viewport.height / 4 &&
bar.y + bar.height >= viewport.y + viewport.height * 0.9 &&
bar.y + bar.height <= viewport.y + viewport.height + 0.5
? [bar]
: [];
});
Comment thread
thymikee marked this conversation as resolved.
Resolved
buttons.sort((a, b) => a.x - b.x);
if (buttons.length < 2 || Math.abs(buttons[0]!.x - viewport.x) > 0.5) return false;
let edge = viewport.x;
const first = buttons[0]!;
for (const button of buttons) {
if (
Math.abs(button.x - edge) > 0.5 ||
Math.abs(button.y - first.y) > 0.5 ||
Math.abs(button.height - first.height) > 0.5
)
return false;
edge = button.x + button.width;
}
return Math.abs(edge - (viewport.x + viewport.width)) <= 0.5;
}
Comment thread
thymikee marked this conversation as resolved.
Resolved

function isAdditionalOverlayRootNode(
node: RawSnapshotNode,
byIndex: Map<number, RawSnapshotNode>,
Expand Down
Loading