diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.test.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.test.ts new file mode 100644 index 0000000000..08da127149 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.test.ts @@ -0,0 +1,96 @@ +import { expect, test } from 'vitest'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts'; + +const FULL_SCREEN = { x: 0, y: 0, width: 402, height: 874 }; +const SETTINGS_BACK_BUTTON = { + type: 'Button', + label: 'Settings', + rect: { x: 16, y: 62, width: 44, height: 44 }, +}; + +function presented(nodes: RawSnapshotNode[]) { + return buildIosInteractiveSnapshotPresentation(nodes).nodes.map((node) => [ + node.type, + node.label, + node.parentIndex, + ]); +} + +test('a SwiftUI app root that shares the back button label stays the root', () => { + const nodes = [ + { + index: 0, + depth: 0, + type: 'Other', + subrole: 'UIApplication', + label: 'Settings', + rect: FULL_SCREEN, + }, + { index: 1, depth: 1, parentIndex: 0, ...SETTINGS_BACK_BUTTON }, + ]; + + expect(presented(nodes)).toEqual([ + ['Other', 'Settings', undefined], + ['Button', 'Settings', 0], + ]); +}); + +test('a SwiftUI app root that shares its list label stays the root', () => { + const nodes = [ + { + index: 0, + depth: 0, + type: 'Other', + subrole: 'UIApplication', + label: 'Notes', + rect: FULL_SCREEN, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'CollectionView', + label: 'Notes', + rect: FULL_SCREEN, + }, + ]; + + expect(presented(nodes)).toEqual([ + ['Other', 'Notes', undefined], + ['CollectionView', 'Notes', 0], + ]); +}); + +test('a wrapper that shares its list label still collapses', () => { + const nodes = [ + { index: 0, depth: 0, type: 'Application', label: 'App', rect: FULL_SCREEN }, + { index: 1, depth: 1, parentIndex: 0, type: 'Other', label: 'Notes', rect: FULL_SCREEN }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'CollectionView', + label: 'Notes', + rect: FULL_SCREEN, + }, + ]; + + expect(presented(nodes)).toEqual([ + ['Application', 'App', undefined], + ['CollectionView', 'Notes', 0], + ]); +}); + +test('a full-screen wrapper that shares its button label still collapses', () => { + const nodes = [ + { index: 0, depth: 0, type: 'Application', label: 'App', rect: FULL_SCREEN }, + { index: 1, depth: 1, parentIndex: 0, type: 'Other', label: 'Settings', rect: FULL_SCREEN }, + { index: 2, depth: 2, parentIndex: 1, ...SETTINGS_BACK_BUTTON }, + ]; + + expect(presented(nodes)).toEqual([ + ['Application', 'App', undefined], + ['Button', 'Settings', 0], + ]); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.ts index 6b196fa901..b0b5adc9ca 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.ts @@ -1,5 +1,5 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isViewportRootNode, normalizeType } from '@agent-device/contracts/snapshot'; import { areRectsApproximatelyEqual, findDescendant, @@ -41,7 +41,8 @@ function collectRepeatedStaticSuppressionForNode( if (type !== 'other') { return; } - const semanticDescendant = findEquivalentSemanticDescendant(nodes, position, nodeLabel); + const semanticDescendant = + !isViewportRootNode(node) && findEquivalentSemanticDescendant(nodes, position, nodeLabel); if (semanticDescendant) { context.suppressNode(node, [semanticDescendant]); return; @@ -85,6 +86,7 @@ export function collectIosActionWrapperSuppression( context: SnapshotTreeRuleContext, ): void { forEachOtherNodeWithLabel(nodes, (node, nodeLabel, position) => { + if (isViewportRootNode(node)) return; const semanticDescendant = findDescendant(nodes, position, (descendant) => { return ( isSemanticActionNode(descendant) && diff --git a/packages/capture-kit/src/snapshot-occlusion-ios27-system-toolbars.fixture.json b/packages/capture-kit/src/snapshot-occlusion-ios27-system-toolbars.fixture.json new file mode 100644 index 0000000000..6b623a1301 --- /dev/null +++ b/packages/capture-kit/src/snapshot-occlusion-ios27-system-toolbars.fixture.json @@ -0,0 +1,997 @@ +{ + "contactsNewContactInteractive": [ + { + "index": 0, + "type": "Other", + "role": "CNApplication", + "subrole": "UIApplication", + "label": "Contacts", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 0, + "hittable": false + }, + { + "index": 1, + "parentIndex": 0, + "type": "Button", + "role": "_UIGrabber", + "label": "Sheet Grabber", + "value": "Expanded", + "rect": { "x": 151, "y": 58, "width": 100, "height": 24 }, + "enabled": true, + "depth": 1, + "hittable": true + }, + { + "index": 2, + "parentIndex": 0, + "type": "NavigationBar", + "role": "UINavigationBar", + "identifier": "New Contact", + "rect": { "x": 0, "y": 78, "width": 402, "height": 54 }, + "enabled": true, + "depth": 1, + "hittable": true + }, + { + "index": 3, + "parentIndex": 2, + "type": "Button", + "role": "_UIButtonBarButton", + "label": "close", + "rect": { "x": 20, "y": 82, "width": 36, "height": 36 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 4, + "parentIndex": 2, + "type": "StaticText", + "role": "UILabel", + "label": "New Contact", + "rect": { + "x": 149.33333333333334, + "y": 89.66666666666666, + "width": 103, + "height": 20.66666666666667 + }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 5, + "parentIndex": 2, + "type": "Button", + "role": "_UIButtonBarButton", + "label": "Done", + "rect": { "x": 346, "y": 82, "width": 36, "height": 36 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 6, + "parentIndex": 0, + "type": "Table", + "role": "CNContactView", + "subrole": "UITableView", + "rect": { "x": 0, "y": 132, "width": 402, "height": 680 }, + "enabled": true, + "depth": 1, + "hittable": true + }, + { + "index": 7, + "parentIndex": 6, + "type": "Button", + "role": "CNContactPhotoView", + "subrole": "UIControl", + "label": "Add photo", + "rect": { "x": 113.66666666666669, "y": 132, "width": 175, "height": 175 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 8, + "parentIndex": 6, + "type": "Button", + "role": "UIButton", + "label": "Add Photo", + "rect": { "x": 137.66666666666666, "y": 319, "width": 126.99999999999997, "height": 48 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 9, + "parentIndex": 6, + "type": "Button", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "label": "Remove photo", + "identifier": "RemoveAvatarPosterButton", + "rect": { "x": 235, "y": 132, "width": 44, "height": 41 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 10, + "parentIndex": 6, + "type": "Cell", + "role": "UITableViewCellAccessibilityElement", + "subrole": "UIAccessibilityElement", + "rect": { "x": 20, "y": 390.3333333333333, "width": 362, "height": 53 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 11, + "parentIndex": 10, + "type": "TextField", + "role": "CNTextField", + "subrole": "UITextField", + "label": "First name", + "identifier": "First name", + "value": "Ann", + "rect": { "x": 36, "y": 406, "width": 330, "height": 22 }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 12, + "parentIndex": 11, + "type": "Button", + "role": "_UITextFieldClearButton", + "label": "Clear text", + "rect": { "x": 341, "y": 407, "width": 19.666666666666686, "height": 19.666666666666686 }, + "enabled": true, + "depth": 4, + "hittable": true + }, + { + "index": 13, + "parentIndex": 6, + "type": "Cell", + "role": "UITableViewCellAccessibilityElement", + "subrole": "UIAccessibilityElement", + "rect": { "x": 20, "y": 443.3333333333333, "width": 362, "height": 53 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 14, + "parentIndex": 13, + "type": "TextField", + "role": "CNTextField", + "subrole": "UITextField", + "label": "Last name", + "identifier": "Last name", + "value": "Last name", + "rect": { "x": 36, "y": 459, "width": 330, "height": 22 }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 15, + "parentIndex": 6, + "type": "Cell", + "role": "UITableViewCellAccessibilityElement", + "subrole": "UIAccessibilityElement", + "rect": { "x": 20, "y": 496.3333333333333, "width": 362, "height": 52.99999999999994 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 16, + "parentIndex": 15, + "type": "TextField", + "role": "CNTextField", + "subrole": "UITextField", + "label": "Company", + "identifier": "Company", + "value": "Company", + "rect": { "x": 36, "y": 512, "width": 330, "height": 22 }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 17, + "parentIndex": 6, + "type": "Cell", + "role": "UITableViewCellAccessibilityElement", + "subrole": "UIAccessibilityElement", + "label": "add phone", + "rect": { "x": 20, "y": 565.3333339691162, "width": 362, "height": 51.33333206176758 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 18, + "parentIndex": 17, + "type": "Button", + "role": "UITableViewCellEditControl", + "subrole": "UIAccessibilityElement", + "label": "Insert add phone", + "rect": { + "x": 39.166666666666664, + "y": 577.0000006357828, + "width": 25.666666666666664, + "height": 25.66666666666663 + }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 19, + "parentIndex": 17, + "type": "StaticText", + "role": "UILabel", + "label": "add phone", + "rect": { "x": 80, "y": 580.3333339691162, "width": 286, "height": 21.33333333333337 }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 20, + "parentIndex": 6, + "type": "Cell", + "role": "UITableViewCellAccessibilityElement", + "subrole": "UIAccessibilityElement", + "label": "add email", + "rect": { "x": 20, "y": 632.6666666666666, "width": 362, "height": 51.33333206176758 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 21, + "parentIndex": 20, + "type": "Button", + "role": "UITableViewCellEditControl", + "subrole": "UIAccessibilityElement", + "label": "Insert add email", + "rect": { + "x": 39.166666666666664, + "y": 644.3333333333333, + "width": 25.666666666666664, + "height": 25.66666666666663 + }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 22, + "parentIndex": 20, + "type": "StaticText", + "role": "UILabel", + "label": "add email", + "rect": { "x": 80, "y": 647.6666666666666, "width": 286, "height": 21.33333333333337 }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 23, + "parentIndex": 6, + "type": "Cell", + "role": "UITableViewCellAccessibilityElement", + "subrole": "UIAccessibilityElement", + "label": "add pronouns", + "rect": { "x": 20, "y": 699.999999364217, "width": 362, "height": 51.33333206176758 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 24, + "parentIndex": 23, + "type": "Button", + "role": "UITableViewCellEditControl", + "subrole": "UIAccessibilityElement", + "label": "Insert add pronouns", + "rect": { + "x": 39.166666666666664, + "y": 711.6666660308837, + "width": 25.666666666666664, + "height": 25.66666666666663 + }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 25, + "parentIndex": 23, + "type": "StaticText", + "role": "UILabel", + "label": "add pronouns", + "rect": { "x": 80, "y": 714.999999364217, "width": 286, "height": 21.33333333333337 }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 26, + "parentIndex": 6, + "type": "Cell", + "role": "UITableViewCellAccessibilityElement", + "subrole": "UIAccessibilityElement", + "label": "Ringtone", + "value": "Default", + "rect": { "x": 20, "y": 767.3333320617675, "width": 362, "height": 44.666667938232536 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 27, + "parentIndex": 26, + "type": "StaticText", + "role": "UILabel", + "label": "Ringtone", + "rect": { + "x": 36, + "y": 783.3333320617675, + "width": 61.33333333333334, + "height": 19.66666666666663 + }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 28, + "parentIndex": 26, + "type": "StaticText", + "role": "UILabel", + "label": "Default", + "rect": { + "x": 113.33333333333333, + "y": 782.9999987284341, + "width": 54.999999999999986, + "height": 20.33333333333337 + }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 29, + "parentIndex": 26, + "type": "Image", + "role": "UIImageView", + "label": "Forward", + "identifier": "chevron.forward", + "rect": { + "x": 355.3333333333333, + "y": 785.9999987284341, + "width": 10.666666666666686, + "height": 14.333333333333371 + }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 30, + "parentIndex": 6, + "type": "StaticText", + "role": "UILabel", + "label": "Add Photo", + "rect": { + "x": 161.66666666666666, + "y": 333.3333333333333, + "width": 79, + "height": 19.333333333333314 + }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 31, + "parentIndex": 0, + "type": "Toolbar", + "role": "CNPassthroughToolbar", + "subrole": "UIToolbar", + "label": "Toolbar", + "identifier": "Toolbar", + "rect": { "x": 0, "y": 83, "width": 402, "height": 48 }, + "enabled": true, + "depth": 1, + "hittable": true + }, + { + "index": 32, + "parentIndex": 0, + "type": "Toolbar", + "role": "_UIFloatingBarContainerView", + "label": "Toolbar", + "identifier": "Toolbar", + "rect": { "x": 0, "y": 62, "width": 402, "height": 812 }, + "enabled": true, + "depth": 1, + "hittable": true + }, + { + "index": 33, + "parentIndex": 0, + "type": "Key", + "role": "UIAccessibilityElementKBKey", + "subrole": "UIAccessibilityElement", + "label": "Padding-Left", + "rect": { "x": 0, "y": 918, "width": 0, "height": 0 }, + "enabled": true, + "depth": 1, + "hittable": false + }, + { + "index": 34, + "parentIndex": 0, + "type": "Key", + "role": "UIAccessibilityElementKBKey", + "subrole": "UIAccessibilityElement", + "label": "Padding-Right", + "rect": { "x": 0, "y": 918, "width": 0, "height": 0 }, + "enabled": true, + "depth": 1, + "hittable": false + }, + { + "index": 35, + "parentIndex": 0, + "type": "Key", + "role": "UIAccessibilityElementKBKey", + "subrole": "UIAccessibilityElement", + "label": "Padding-Left", + "rect": { "x": 0, "y": 918, "width": 0, "height": 0 }, + "enabled": true, + "depth": 1, + "hittable": false + }, + { + "index": 36, + "parentIndex": 0, + "type": "Key", + "role": "UIAccessibilityElementKBKey", + "subrole": "UIAccessibilityElement", + "label": "Padding-Right", + "rect": { "x": 0, "y": 918, "width": 0, "height": 0 }, + "enabled": true, + "depth": 1, + "hittable": false + }, + { + "index": 37, + "parentIndex": 0, + "type": "Key", + "role": "UIAccessibilityElementKBKey", + "subrole": "UIAccessibilityElement", + "label": "Padding-Left", + "rect": { "x": 0, "y": 918, "width": 0, "height": 0 }, + "enabled": true, + "depth": 1, + "hittable": false + }, + { + "index": 38, + "parentIndex": 0, + "type": "Key", + "role": "UIAccessibilityElementKBKey", + "subrole": "UIAccessibilityElement", + "label": "Padding-Right", + "rect": { "x": 0, "y": 918, "width": 0, "height": 0 }, + "enabled": true, + "depth": 1, + "hittable": false + } + ], + "settingsGeneralRaw": [ + { + "index": 0, + "type": "Other", + "role": "_TtC7SwiftUIP33_ACC2C5639A7D76F611E170E831FCA49118SwiftUIApplication", + "subrole": "UIApplication", + "label": "Settings", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 0, + "hittable": false + }, + { + "index": 1, + "parentIndex": 0, + "type": "Window", + "role": "UIWindow", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 1, + "hittable": true + }, + { + "index": 2, + "parentIndex": 1, + "type": "Other", + "role": "UITransitionView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 2, + "hittable": true + }, + { + "index": 3, + "parentIndex": 2, + "type": "Other", + "role": "UIDropShadowView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 3, + "hittable": true + }, + { + "index": 4, + "parentIndex": 3, + "type": "Other", + "role": "_TtGC7SwiftUI14_UIHostingViewGVS_15ModifiedContentGVS_8LazyViewGVS_P10$1ca6179c417SceneBridgeReaderGVS_16SubscriptionViewGC7Combine18PassthroughSubjectP_Os5Never_V11SettingsApp27SettingsNavigationSplitView___VS_12RootModifier__", + "subrole": "UIView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 4, + "hittable": true + }, + { + "index": 5, + "parentIndex": 4, + "type": "Other", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 5, + "hittable": true + }, + { + "index": 6, + "parentIndex": 5, + "type": "Other", + "role": "UIView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 6, + "hittable": true + }, + { + "index": 7, + "parentIndex": 6, + "type": "Other", + "role": "_UISplitViewControllerAdaptiveColumnContainerView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 7, + "hittable": true + }, + { + "index": 8, + "parentIndex": 7, + "type": "Other", + "role": "_UISplitViewControllerAdaptiveColumnContainerView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 8, + "hittable": true + }, + { + "index": 9, + "parentIndex": 8, + "type": "Other", + "role": "_UISplitViewControllerAdaptiveColumnView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 9, + "hittable": true + }, + { + "index": 10, + "parentIndex": 9, + "type": "Other", + "role": "UILayoutContainerView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 10, + "hittable": true + }, + { + "index": 11, + "parentIndex": 10, + "type": "NavigationBar", + "role": "SwiftUI.UIKitNavigationBar", + "subrole": "UINavigationBar", + "identifier": "General", + "rect": { "x": 0, "y": 62, "width": 402, "height": 54 }, + "enabled": true, + "depth": 11, + "hittable": true + }, + { + "index": 12, + "parentIndex": 11, + "type": "Button", + "role": "UIAccessibilityBackButtonElement", + "subrole": "UIAccessibilityElement", + "label": "Settings", + "identifier": "BackButton", + "rect": { "x": 16, "y": 62, "width": 44, "height": 44 }, + "enabled": true, + "depth": 12, + "hittable": true + }, + { + "index": 13, + "parentIndex": 11, + "type": "StaticText", + "role": "_TtGC7SwiftUI16UIKitBarItemHostVS_11BarItemView_", + "subrole": "UIView", + "label": "General", + "rect": { "x": 170, "y": 73.66666666666666, "width": 62, "height": 20.66666666666667 }, + "enabled": true, + "depth": 12, + "hittable": true + }, + { + "index": 14, + "parentIndex": 13, + "type": "Other", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "rect": { "x": 170, "y": 73.66666666666666, "width": 62, "height": 20.66666666666667 }, + "enabled": true, + "depth": 13, + "hittable": true + }, + { + "index": 15, + "parentIndex": 14, + "type": "StaticText", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "label": "General", + "rect": { "x": 170, "y": 73.83333333333333, "width": 62, "height": 20.33333333333333 }, + "enabled": true, + "depth": 14, + "hittable": true + }, + { + "index": 16, + "parentIndex": 10, + "type": "Other", + "role": "UINavigationTransitionView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 11, + "hittable": true + }, + { + "index": 17, + "parentIndex": 16, + "type": "Other", + "role": "UIViewControllerWrapperView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 12, + "hittable": true + }, + { + "index": 18, + "parentIndex": 17, + "type": "Other", + "role": "_TtCGC7SwiftUI32NavigationStackHostingControllerVS_7AnyView_P10$1ca66e02c11HostingView", + "subrole": "UIView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 13, + "hittable": true + }, + { + "index": 19, + "parentIndex": 18, + "type": "Other", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 14, + "hittable": true + }, + { + "index": 20, + "parentIndex": 19, + "type": "CollectionView", + "role": "SwiftUI.UpdateCoalescingCollectionView", + "subrole": "UICollectionView", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 15, + "hittable": true + }, + { + "index": 21, + "parentIndex": 20, + "type": "Cell", + "role": "SwiftUI.ListCollectionViewCell", + "subrole": "UICollectionViewListCell", + "rect": { "x": 16, "y": 151, "width": 370, "height": 221.66666666666663 }, + "enabled": true, + "depth": 16, + "hittable": true + }, + { + "index": 22, + "parentIndex": 21, + "type": "Other", + "role": "_UISystemBackgroundView", + "rect": { "x": 16, "y": 151, "width": 370, "height": 221.66666666666663 }, + "enabled": true, + "depth": 17, + "hittable": true + }, + { + "index": 23, + "parentIndex": 22, + "type": "Other", + "role": "UIView", + "rect": { "x": 16, "y": 151, "width": 370, "height": 221.66666666666663 }, + "enabled": true, + "depth": 18, + "hittable": true + }, + { + "index": 24, + "parentIndex": 21, + "type": "Other", + "role": "_UICollectionViewListCellContentView", + "rect": { "x": 16, "y": 151, "width": 370, "height": 221.66666666666663 }, + "enabled": true, + "depth": 17, + "hittable": true + }, + { + "index": 25, + "parentIndex": 24, + "type": "Other", + "role": "_TtGC7SwiftUI15CellHostingViewGVS_15ModifiedContentVS_14_ViewList_ViewVS_26CollectionViewCellModifier__", + "subrole": "UIView", + "rect": { "x": 16, "y": 151, "width": 370, "height": 221.66666666666663 }, + "enabled": true, + "depth": 18, + "hittable": true + }, + { + "index": 26, + "parentIndex": 25, + "type": "Other", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "rect": { "x": 16, "y": 151, "width": 370, "height": 221.66666666666663 }, + "enabled": true, + "depth": 19, + "hittable": true + }, + { + "index": 27, + "parentIndex": 26, + "type": "Button", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "identifier": "com.apple.settings.general.featureDescription", + "rect": { "x": 32, "y": 166, "width": 333, "height": 191.66668701171875 }, + "enabled": true, + "depth": 20, + "hittable": true + }, + { + "index": 28, + "parentIndex": 27, + "type": "Image", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "rect": { "x": 32, "y": 166, "width": 60, "height": 60 }, + "enabled": true, + "depth": 21, + "hittable": true + }, + { + "index": 29, + "parentIndex": 27, + "type": "StaticText", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "label": "General", + "rect": { "x": 32, "y": 241, "width": 80, "height": 26.333333333333314 }, + "enabled": true, + "depth": 21, + "hittable": true + }, + { + "index": 30, + "parentIndex": 27, + "type": "StaticText", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "label": "Manage your overall setup and preferences for iPhone, such as software updates, device language, CarPlay, AirDrop, and more.", + "rect": { "x": 32, "y": 271.3333333333333, "width": 333, "height": 86.33333333333331 }, + "enabled": true, + "depth": 21, + "hittable": true + }, + { + "index": 31, + "parentIndex": 20, + "type": "Cell", + "role": "SwiftUI.ListCollectionViewCell", + "subrole": "UICollectionViewListCell", + "rect": { "x": 16, "y": 407.6666666666667, "width": 370, "height": 52 }, + "enabled": true, + "depth": 16, + "hittable": true + }, + { + "index": 32, + "parentIndex": 31, + "type": "Other", + "role": "_UISystemBackgroundView", + "rect": { "x": 16, "y": 407.6666666666667, "width": 370, "height": 52 }, + "enabled": true, + "depth": 17, + "hittable": true + }, + { + "index": 33, + "parentIndex": 32, + "type": "Other", + "role": "UIView", + "rect": { "x": 16, "y": 407.6666666666667, "width": 370, "height": 52 }, + "enabled": true, + "depth": 18, + "hittable": true + }, + { + "index": 34, + "parentIndex": 31, + "type": "Other", + "role": "_UICollectionViewListCellContentView", + "rect": { "x": 16, "y": 407.6666666666667, "width": 370, "height": 52 }, + "enabled": true, + "depth": 17, + "hittable": true + }, + { + "index": 35, + "parentIndex": 34, + "type": "Other", + "role": "_TtGC7SwiftUI15CellHostingViewGVS_15ModifiedContentVS_14_ViewList_ViewVS_26CollectionViewCellModifier__", + "subrole": "UIView", + "rect": { "x": 16, "y": 407.6666666666667, "width": 370, "height": 52 }, + "enabled": true, + "depth": 18, + "hittable": true + }, + { + "index": 36, + "parentIndex": 35, + "type": "Other", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "rect": { "x": 16, "y": 407.6666666666667, "width": 370, "height": 52 }, + "enabled": true, + "depth": 19, + "hittable": true + }, + { + "index": 37, + "parentIndex": 36, + "type": "Button", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "label": "About", + "identifier": "com.apple.settings.general.about", + "rect": { "x": 16, "y": 407.6666666666667, "width": 370, "height": 52 }, + "enabled": true, + "depth": 20, + "hittable": true + }, + { + "index": 38, + "parentIndex": 37, + "type": "StaticText", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "label": "About", + "rect": { "x": 30, "y": 419.6666564941406, "width": 88, "height": 28 }, + "enabled": true, + "depth": 21, + "hittable": true + }, + { + "index": 39, + "parentIndex": 37, + "type": "Image", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "rect": { "x": 30, "y": 419.6666666666667, "width": 28, "height": 28 }, + "enabled": true, + "depth": 21, + "hittable": true + }, + { + "index": 40, + "parentIndex": 37, + "type": "Image", + "role": "SwiftUI.AccessibilityNode", + "subrole": "NSObject", + "identifier": "chevron.forward", + "rect": { "x": 360.6666666666667, "y": 427.6666666666667, "width": 7, "height": 12 }, + "enabled": true, + "depth": 21, + "hittable": true + }, + { + "index": 41, + "parentIndex": 10, + "type": "Toolbar", + "role": "_UIFloatingBarContainerView", + "label": "Toolbar", + "identifier": "Toolbar", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 11, + "hittable": true + }, + { + "index": 42, + "parentIndex": 41, + "type": "Other", + "role": "_TtGC5UIKitP10$1c5774cd822FloatingBarHostingViewVS_20FloatingBarContainer_", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 12, + "hittable": true + }, + { + "index": 43, + "parentIndex": 8, + "type": "Toolbar", + "role": "_UIFloatingBarContainerView", + "label": "Toolbar", + "identifier": "Toolbar", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 9, + "hittable": true + }, + { + "index": 44, + "parentIndex": 43, + "type": "Other", + "role": "_TtGC5UIKitP10$1c5774cd822FloatingBarHostingViewVS_20FloatingBarContainer_", + "rect": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true, + "depth": 10, + "hittable": true + } + ] +} diff --git a/packages/capture-kit/src/snapshot-occlusion-ios27-system-toolbars.test.ts b/packages/capture-kit/src/snapshot-occlusion-ios27-system-toolbars.test.ts new file mode 100644 index 0000000000..b12043125f --- /dev/null +++ b/packages/capture-kit/src/snapshot-occlusion-ios27-system-toolbars.test.ts @@ -0,0 +1,173 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'vitest'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { + createIosSnapshotRequest, + deriveIosCaptureHint, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import { presentIosSnapshot } from './ios-snapshot-engine/engine.ts'; +import { annotateCoveredSnapshotNodes } from './snapshot-occlusion.ts'; + +// Captured on an iPhone 18 Pro simulator, iOS 27.0. `contactsNewContactInteractive` is the input the +// occlusion pass received for `snapshot -i` on Contacts > New Contact after typing a first name. +// `settingsGeneralRaw` is `snapshot --raw` on Settings > General, which the iOS engine projects, +// cut to the navigation bar, the first two rows, the toolbars, and their ancestors. +type Captures = { + contactsNewContactInteractive: RawSnapshotNode[]; + settingsGeneralRaw: RawSnapshotNode[]; +}; + +function captures(): Captures { + return JSON.parse( + readFileSync( + new URL('./snapshot-occlusion-ios27-system-toolbars.fixture.json', import.meta.url), + 'utf8', + ), + ) as Captures; +} + +function coveredLabels(nodes: RawSnapshotNode[]): string[] { + return annotateCoveredSnapshotNodes(nodes) + .filter((candidate) => candidate.interactionBlocked === 'covered') + .map((candidate) => candidate.label ?? candidate.type ?? ''); +} + +function nodeWithLabel(nodes: RawSnapshotNode[], label: string): RawSnapshotNode { + const node = nodes.find((candidate) => candidate.label === label); + assert.ok(node, label); + return node; +} + +test('the captured Contacts sheet keeps Done and its rows pressable under its own toolbars', () => { + // The sheet's floating bar container encloses the sheet's navigation bar and table. The bar-height + // passthrough toolbar listed before it encloses nothing, so it is not a presentation boundary, and + // it is drawn inside the navigation bar, so it backs Done instead of covering it. + assert.deepEqual(coveredLabels(captures().contactsNewContactInteractive), []); +}); + +test('a toolbar that extends past the navigation bar still covers Done', () => { + const nodes = captures().contactsNewContactInteractive; + const passthrough = nodes.find((candidate) => candidate.role === 'CNPassthroughToolbar'); + assert.ok(passthrough, 'CNPassthroughToolbar'); + passthrough.rect = { x: 0, y: 83, width: 402, height: 120 }; + + assert.ok(coveredLabels(nodes).includes('Done')); +}); + +test('a menu inside the navigation bar still covers Done', () => { + const nodes = captures().contactsNewContactInteractive; + nodes.push({ + index: nodes.length, + parentIndex: 0, + depth: 1, + type: 'Menu', + label: 'More', + rect: { x: 300, y: 80, width: 100, height: 50 }, + hittable: true, + }); + + assert.ok(coveredLabels(nodes).includes('Done')); +}); + +test('a bottom toolbar of the screen beneath an undimmed sheet keeps that screen covered', () => { + const screen = { x: 0, y: 0, width: 402, height: 874 }; + const nodes: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Application', label: 'App', rect: screen }, + { + index: 1, + parentIndex: 0, + depth: 1, + type: 'ScrollView', + rect: { x: 0, y: 0, width: 402, height: 791 }, + }, + { + index: 2, + parentIndex: 1, + depth: 2, + type: 'Button', + label: 'Lower row', + rect: { x: 16, y: 600, width: 370, height: 52 }, + }, + { + index: 3, + parentIndex: 0, + depth: 1, + type: 'Toolbar', + role: 'UIToolbar', + rect: { x: 0, y: 791, width: 402, height: 49 }, + }, + { + index: 4, + parentIndex: 3, + depth: 2, + type: 'Button', + label: 'Compose', + rect: { x: 340, y: 796, width: 44, height: 40 }, + }, + { + index: 5, + parentIndex: 0, + depth: 1, + type: 'NavigationBar', + label: 'Sheet', + rect: { x: 0, y: 78, width: 402, height: 54 }, + }, + { + index: 6, + parentIndex: 0, + depth: 1, + type: 'ScrollView', + rect: { x: 0, y: 132, width: 402, height: 400 }, + }, + { + index: 7, + parentIndex: 6, + depth: 2, + type: 'Button', + label: 'Sheet row', + rect: { x: 16, y: 140, width: 370, height: 52 }, + }, + { + index: 8, + parentIndex: 0, + depth: 1, + type: 'Toolbar', + role: '_UIFloatingBarContainerView', + label: 'Toolbar', + rect: { x: 0, y: 62, width: 402, height: 812 }, + }, + ]; + + const covered = coveredLabels(nodes); + assert.ok(covered.includes('Lower row')); + assert.ok(covered.includes('Compose')); + assert.ok(!covered.includes('Sheet row')); +}); + +test('the captured Settings screen keeps its rows pressable under its full-screen toolbars', () => { + // The app node shares the back button's label. The projection keeps it as the viewport root, so + // the full-screen toolbar containers under it are exempt and do not cover the rows. + const request = createIosSnapshotRequest({ interactiveOnly: true }); + const hint = deriveIosCaptureHint(request); + const presented = presentIosSnapshot( + { + stage: 'acquired', + acquisition: { + producer: 'simulator-ax-bridge', + intent: 'full', + hint: { ...hint, acquisitionIntent: 'full' }, + nodes: captures().settingsGeneralRaw, + truncated: false, + viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 402, height: 874 } }, + lineage: { targetId: 'simulator-1', generation: 'generation-1' }, + residue: [], + }, + }, + request, + ); + + nodeWithLabel(presented.nodes, 'About'); + assert.deepEqual(coveredLabels(presented.nodes), []); + assert.equal(presented.nodes[0]?.label, 'Settings'); +}); diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index 8ee5f86b00..822f352067 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -36,6 +36,7 @@ const FLOATING_BAR_CONTAINER_ROLE = '_UIFloatingBarContainerView'; const VIEWPORT_CHROME_TOP_INSET_PT = 4; const TOOLBAR_HOST_KIND_FRAGMENTS = ['toolbar']; const TAB_BAR_KIND_FRAGMENTS = ['tabbar']; +const NAVIGATION_BAR_KIND_FRAGMENTS = ['navigationbar']; const SCROLL_INDICATOR_KIND_FRAGMENTS = ['scrollbar', 'scrollindicator']; const SEMANTIC_TOUCH_KIND_FRAGMENTS = [ 'button', @@ -309,6 +310,8 @@ function visibleCoverRect( const candidateRect = positiveRect(candidate.rect); if (!candidateRect || areRectsApproximatelyEqual(targetRect, candidateRect)) return null; if (isInToolbarHostOwnPresentation(candidate, target, scan)) return null; + if (isToolbarInsideTargetNavigationBar(candidate, candidateRect, target, scan.byIndex)) + return null; if (findCoveringNode(scan, candidatePosition, candidate, options)) return null; return candidateRect; } @@ -404,12 +407,12 @@ function toolbarHostReach( /** * Toolbar hosts draw their descendants, not their enclosing frame. A host that encloses bars or - * scroll containers of its own presentation, the siblings after the previous toolbar and before - * this host, covers nothing of it. A host that encloses the presentation beneath it, which an - * undimmed sheet keeps in the capture, covers only that presentation, including enclosed tab bars - * listed first after the previous toolbar. A host that encloses nothing, or a presentation stacked - * over it, keeps covering. Like the sibling-sized exemption, this cannot tell an opaque toolbar - * from a host. + * scroll containers of its own presentation, the siblings after the previous toolbar host and + * before this host, covers nothing of it. A host that encloses the presentation beneath it, which + * an undimmed sheet keeps in the capture, covers only that presentation, including enclosed tab + * bars listed first after the previous toolbar host. A host that encloses nothing, or a + * presentation stacked over it, keeps covering. Like the sibling-sized exemption, this cannot tell + * an opaque toolbar from a host. */ function classifyToolbarHostReach( node: RawSnapshotNode, @@ -467,6 +470,31 @@ function isInToolbarHostOwnPresentation( return branch !== undefined && reach.ownBranchIndexes.has(branch.index); } +/** + * A toolbar drawn inside the navigation bar that holds the target backs that bar, not covers it. + * Accepted limit: a stacked presentation whose top bar is a toolbar inside the frame of the + * navigation bar beneath it does not cover that bar. + */ +function isToolbarInsideTargetNavigationBar( + candidate: RawSnapshotNode, + candidateRect: Rect, + target: RawSnapshotNode, + byIndex: Map, +): boolean { + if (!nodeKindIncludesAny(candidate, TOOLBAR_HOST_KIND_FRAGMENTS)) return false; + let current: RawSnapshotNode | undefined = target; + const visited = new Set(); + while (current && !visited.has(current.index)) { + if (nodeKindIncludesAny(current, NAVIGATION_BAR_KIND_FRAGMENTS)) { + return isNavigationBarAround(current, candidateRect); + } + visited.add(current.index); + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + return false; +} + /** The target itself or its ancestor that is a direct child of `parentIndex`. */ function branchUnderParent( node: RawSnapshotNode, @@ -484,13 +512,30 @@ function branchUnderParent( return undefined; } +/** After the previous toolbar host; a toolbar drawn inside a sibling navigation bar is not one. */ function ownPresentationStart(siblings: readonly RawSnapshotNode[], position: number): number { for (let previous = position - 1; previous >= 0; previous--) { - if (nodeKindIncludesAny(siblings[previous]!, TOOLBAR_HOST_KIND_FRAGMENTS)) return previous + 1; + if (isToolbarHostSibling(siblings, previous)) return previous + 1; } return 0; } +function isToolbarHostSibling(siblings: readonly RawSnapshotNode[], position: number): boolean { + const node = siblings[position]!; + if (!nodeKindIncludesAny(node, TOOLBAR_HOST_KIND_FRAGMENTS)) return false; + const rect = positiveRect(node.rect); + return !rect || !siblings.some((sibling) => isNavigationBarAround(sibling, rect)); +} + +function isNavigationBarAround(bar: RawSnapshotNode, rect: Rect): boolean { + const barRect = positiveRect(bar.rect); + return Boolean( + barRect && + nodeKindIncludesAny(bar, NAVIGATION_BAR_KIND_FRAGMENTS) && + rectContains(barRect, rect), + ); +} + /** A bar or a scroll container inside `rect`; a scroll indicator is not a container. */ function isEnclosedHostedRegion(rect: Rect, node: RawSnapshotNode): boolean { const isHostedRegion =