From 3ce716a0ee42efe873529d70a2d867e3cb363322 Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 14:59:31 +0900 Subject: [PATCH 1/5] fix(ios): establish the screen origin of a private AX root that fills the screen --- .../RunnerTests+AXSnapshotFallback.swift | 38 +++++++++++++++---- .../RunnerXCTestEventBridge.h | 7 ++++ .../RunnerXCTestEventBridge.m | 18 ++++++++- .../RunnerTests+AXSnapshotFallbackTests.swift | 28 ++++++++++++++ 4 files changed, 83 insertions(+), 8 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index e9f2fc569c..0a6058da2d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -273,17 +273,19 @@ extension RunnerTests { !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: bundleId) } - /// The app's reported viewport when XCTest can read it, else the bridge's own root frame declared - /// `.derived`, which cannot establish a screen origin or anchor a rotation. The wire retains this - /// uncertainty, so local coordinates cannot authorize input (#4399). + /// The app's reported viewport when XCTest can read it, else the bridge's own root frame. That root + /// is `.derived`, which cannot establish a screen origin or anchor a rotation, so its local + /// coordinates cannot authorize input (#4399), unless AXRuntime proves the app fills the screen. private func privateAXSnapshotViewport( app: XCUIApplication, bundleId: String?, rootFrame: CGRect ) -> SnapshotViewport { - let fallback = SnapshotViewport.derived(box: rootFrame) + let fallback = { + Self.privateAXRootViewport(rootFrame: rootFrame, fullScreenFrame: Self.fullScreenFrame(of: app)) + } guard shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId) else { - return fallback + return fallback() } do { let reported = try runMainThreadWork( @@ -294,13 +296,35 @@ extension RunnerTests { try self.safeSnapshotViewport(app: app, readingOrientation: true) } if case .missing = reported { - return fallback + return fallback() } return reported } catch { NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_VIEWPORT_FALLBACK=%@", String(describing: error)) - return fallback + return fallback() + } + } + + /// The screen's frame when AXRuntime reports `app` as the primary app filling it, else `.null`. + private static func fullScreenFrame(of app: XCUIApplication) -> CGRect { + #if os(iOS) + return RunnerFullScreenApplicationFrame(app) + #else + return .null + #endif + } + + /// The viewport a private AX root establishes without XCTest. A root that is exactly the screen + /// AXRuntime reports the app filling, at the origin, is `.reported`: two independent reads agree + /// that window space is screen space, as on every iPhone app. Any other root stays `.derived`, + /// including a placed iPad window's local root (#4399) and a root that is the screen rotated. + static func privateAXRootViewport(rootFrame: CGRect, fullScreenFrame: CGRect) -> SnapshotViewport { + // CONSERVATIVE: orientation is unknown here, so a landscape root (the portrait screen with width + // and height swapped) stays `.derived` rather than guess which way to rotate it. + guard !fullScreenFrame.isNull, rootFrame.origin == .zero, rootFrame == fullScreenFrame else { + return .derived(box: rootFrame) } + return .reported(box: rootFrame, interfaceOrientation: RunnerInterfaceOrientation.unknown) } func privateAXRect(_ value: Any?) -> CGRect { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index dadac43409..91dbdfba8f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -133,6 +133,13 @@ FOUNDATION_EXPORT RunnerFullScreenEvidence RunnerFullScreenEvidenceFromForegroun CGRect screenFrame ); +// The screen's frame when AXRuntime reports `application` as the primary app filling it, else +// `CGRectNull`. The same probe the sole-screen route admits a non-phone app with; it reads two frames +// and walks no tree, so it answers while XCTest's accessibility channel is penalized. +#if TARGET_OS_IOS +FOUNDATION_EXPORT CGRect RunnerFullScreenApplicationFrame(id application); +#endif + FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass( Class cls, NSString *className, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 1d72e33eaa..eb736cd942 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -436,7 +436,10 @@ CGSize RunnerSoleScreenSize(void) { // One contained probe: it checks each selector before sending it and reads the target pid and both // frames inside one exception scope, so any failed read answers Unknown. The target pid comes from // the existing runner owner for that read (RunnerAXSnapshotBridge). -static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application) { +static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreenFrame( + id application, + CGRect *_Nullable screenFrame +) { if (application == nil) { return RunnerFullScreenEvidenceUnknown; } @@ -470,9 +473,22 @@ static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application if (exception != nil) { return RunnerFullScreenEvidenceUnknown; } + if (screenFrame != NULL) *screenFrame = screen; return RunnerFullScreenEvidenceFromForeground(targetProcessID, primaryProcessID, window, screen); } +static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application) { + return RunnerProbeApplicationFillsScreenFrame(application, NULL); +} + +CGRect RunnerFullScreenApplicationFrame(id application) { + CGRect screen = CGRectNull; + if (RunnerProbeApplicationFillsScreenFrame(application, &screen) != RunnerFullScreenEvidenceFillsScreen) { + return CGRectNull; + } + return screen; +} + #endif NSString * _Nullable RunnerResolveSoleScreen( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index daa25fbb4e..4c03a667c7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -5,6 +5,34 @@ import AgentDeviceSnapshotPresentation // MARK: - In-bundle unit tests extension RunnerTests { + func testPrivateAXRootViewportReportsOnlyAFullScreenRootAtTheOrigin() { + let screen = CGRect(x: 0, y: 0, width: 440, height: 956) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: screen, fullScreenFrame: screen), + .reported(box: screen, interfaceOrientation: RunnerInterfaceOrientation.unknown) + ) + // No full-screen proof: a placed window's local root looks the same as a full-screen one (#4399). + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: screen, fullScreenFrame: .null), + .derived(box: screen) + ) + let placed = CGRect(x: 0, y: 0, width: 551, height: 813) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: placed, fullScreenFrame: CGRect(x: 0, y: 0, width: 1024, height: 1366)), + .derived(box: placed) + ) + let offset = CGRect(x: 0, y: 20, width: 440, height: 956) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: offset, fullScreenFrame: offset), + .derived(box: offset) + ) + let landscape = CGRect(x: 0, y: 0, width: 956, height: 440) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: landscape, fullScreenFrame: screen), + .derived(box: landscape) + ) + } + func testPrivateAXAttemptDepthsAppliesRememberedDepth() { XCTAssertEqual( Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), From b2cb988b9b95ccefcf3abafcf5d6901ecfd84ba3 Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 14:59:31 +0900 Subject: [PATCH 2/5] docs(ios): document the full-screen private AX root as a reported viewport --- .changeset/ios-private-ax-full-screen-origin.md | 5 +++++ apple/runner/README.md | 2 ++ docs/adr/0004-ios-snapshot-backend-strategy.md | 1 + 3 files changed, 8 insertions(+) create mode 100644 .changeset/ios-private-ax-full-screen-origin.md diff --git a/.changeset/ios-private-ax-full-screen-origin.md b/.changeset/ios-private-ax-full-screen-origin.md new file mode 100644 index 0000000000..be00ac5c85 --- /dev/null +++ b/.changeset/ios-private-ax-full-screen-origin.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +iOS simulator taps by ref work again on a full-screen app while the runner reads the screen through its private accessibility fallback, for example right after a large web view loads. Since 0.21.37 they failed with `window-origin-unknown`: under the XCTest channel penalty the fallback's viewport came from the tree's root frame and was never reported. A root that is exactly the screen at the origin, while AXRuntime reports the target as the primary app filling that screen, now establishes the screen origin. A placed iPad window, a root that is offset, and a root with width and height swapped still refuse coordinate-derived actions. diff --git a/apple/runner/README.md b/apple/runner/README.md index a2a3626332..dd1bcd51d3 100644 --- a/apple/runner/README.md +++ b/apple/runner/README.md @@ -133,6 +133,8 @@ Thus, if a viewport read takes too long, the plan skips the sweep instead of que Private-AX fallback keeps the viewport authority it acquired. When a penalty suppresses the XCTest viewport read, its root-derived box does not establish a screen origin. +Exception: AXRuntime can report the target as the primary app that fills the screen. If the root is that screen frame at the origin, the box is `reported`. +This probe reads two frames and walks no tree. The payload carries that uncertainty without a new probe or a changed channel budget. A failure that XCTest records ends `testCommand` when the main-thread block that recorded it returns, whatever `continueAfterFailure` says. diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index f85fe95ffd..2e6b36160c 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -56,6 +56,7 @@ A placed iPad window publishes screen space in points, because its viewport, its The acquisition's `reported`, `derived`, or `missing` viewport evidence survives the runner boundary and capture transformations. Only `reported` establishes the screen origin for capture-derived input. A root-derived validation box preserves readable fallback trees but cannot upgrade unknown origin evidence. +A private-AX root is `reported` only when it is exactly the screen frame at the origin and AXRuntime reports the target as the primary app that fills that screen. Element mutations, mutating find, Maestro selector geometry, and snapshot-derived gesture bounds refuse an unknown origin with `window-origin-unknown` and `dispatched: no` before input. Independent native queries and authored screen coordinates retain their own authorities. From d91a86b18bae1a94769fcb2446e0e71663fe96d9 Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 16:07:36 +0900 Subject: [PATCH 3/5] fix(ios): log the private AX root viewport verdict with both frames --- .../RunnerTests+AXSnapshotFallback.swift | 12 +++++++++++- .../RunnerTests+AXSnapshotFallbackTests.swift | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 0a6058da2d..82911223fa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -282,7 +282,17 @@ extension RunnerTests { rootFrame: CGRect ) -> SnapshotViewport { let fallback = { - Self.privateAXRootViewport(rootFrame: rootFrame, fullScreenFrame: Self.fullScreenFrame(of: app)) + let fullScreenFrame = Self.fullScreenFrame(of: app) + let viewport = Self.privateAXRootViewport(rootFrame: rootFrame, fullScreenFrame: fullScreenFrame) + let kind: String + if case .reported = viewport { kind = "reported" } else { kind = "derived" } + NSLog( + "AGENT_DEVICE_RUNNER_PRIVATE_AX_ROOT_VIEWPORT kind=%@ root=%@ fullScreen=%@", + kind, + String(describing: rootFrame), + fullScreenFrame.isNull ? "none" : String(describing: fullScreenFrame) + ) + return viewport } guard shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId) else { return fallback() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index 4c03a667c7..2125fc020a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -26,6 +26,8 @@ extension RunnerTests { Self.privateAXRootViewport(rootFrame: offset, fullScreenFrame: offset), .derived(box: offset) ) + // Measured on an iPhone 17 Pro Max simulator in landscape: the private AX root is in interface + // space, and AXRuntime reports the screen in native portrait space. let landscape = CGRect(x: 0, y: 0, width: 956, height: 440) XCTAssertEqual( Self.privateAXRootViewport(rootFrame: landscape, fullScreenFrame: screen), From d228f57d7c3f94c7cea9b366e188b41b9a0cd234 Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 16:07:36 +0900 Subject: [PATCH 4/5] docs(ios): say the full-screen probe adds no XCTest tree read --- apple/runner/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apple/runner/README.md b/apple/runner/README.md index dd1bcd51d3..12f791e53b 100644 --- a/apple/runner/README.md +++ b/apple/runner/README.md @@ -134,8 +134,9 @@ Thus, if a viewport read takes too long, the plan skips the sweep instead of que Private-AX fallback keeps the viewport authority it acquired. When a penalty suppresses the XCTest viewport read, its root-derived box does not establish a screen origin. Exception: AXRuntime can report the target as the primary app that fills the screen. If the root is that screen frame at the origin, the box is `reported`. -This probe reads two frames and walks no tree. -The payload carries that uncertainty without a new probe or a changed channel budget. +This probe reads two frames. It adds no XCTest tree read and does not change the channel budget. +The payload carries any remaining uncertainty. +In landscape, the root is in interface space and the screen frame is in native portrait space, so the root stays `derived`. A failure that XCTest records ends `testCommand` when the main-thread block that recorded it returns, whatever `continueAfterFailure` says. Mute XCTest's AX-server rejection in both fetch wordings, so XCTest does not record it. From 586d58a76a7ff25efc84e281c3aee54ab4fe198e Mon Sep 17 00:00:00 2001 From: gronxb Date: Mon, 16 Nov 2026 16:30:25 +0900 Subject: [PATCH 5/5] fix(ios): name a missing private AX root viewport in its log line --- .../RunnerTests+AXSnapshotFallback.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 82911223fa..61f5e26b38 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -285,7 +285,11 @@ extension RunnerTests { let fullScreenFrame = Self.fullScreenFrame(of: app) let viewport = Self.privateAXRootViewport(rootFrame: rootFrame, fullScreenFrame: fullScreenFrame) let kind: String - if case .reported = viewport { kind = "reported" } else { kind = "derived" } + switch viewport { + case .reported: kind = "reported" + case .derived: kind = "derived" + case .missing: kind = "missing" + } NSLog( "AGENT_DEVICE_RUNNER_PRIVATE_AX_ROOT_VIEWPORT kind=%@ root=%@ fullScreen=%@", kind,