diff --git a/.changeset/ios-private-ax-full-screen-origin.md b/.changeset/ios-private-ax-full-screen-origin.md new file mode 100644 index 0000000000..be00ac5c85 --- /dev/null +++ b/.changeset/ios-private-ax-full-screen-origin.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +iOS simulator taps by ref work again on a full-screen app while the runner reads the screen through its private accessibility fallback, for example right after a large web view loads. Since 0.21.37 they failed with `window-origin-unknown`: under the XCTest channel penalty the fallback's viewport came from the tree's root frame and was never reported. A root that is exactly the screen at the origin, while AXRuntime reports the target as the primary app filling that screen, now establishes the screen origin. A placed iPad window, a root that is offset, and a root with width and height swapped still refuse coordinate-derived actions. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index e9f2fc569c..61f5e26b38 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -273,17 +273,33 @@ extension RunnerTests { !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: bundleId) } - /// The app's reported viewport when XCTest can read it, else the bridge's own root frame declared - /// `.derived`, which cannot establish a screen origin or anchor a rotation. The wire retains this - /// uncertainty, so local coordinates cannot authorize input (#4399). + /// The app's reported viewport when XCTest can read it, else the bridge's own root frame. That root + /// is `.derived`, which cannot establish a screen origin or anchor a rotation, so its local + /// coordinates cannot authorize input (#4399), unless AXRuntime proves the app fills the screen. private func privateAXSnapshotViewport( app: XCUIApplication, bundleId: String?, rootFrame: CGRect ) -> SnapshotViewport { - let fallback = SnapshotViewport.derived(box: rootFrame) + let fallback = { + let fullScreenFrame = Self.fullScreenFrame(of: app) + let viewport = Self.privateAXRootViewport(rootFrame: rootFrame, fullScreenFrame: fullScreenFrame) + let kind: String + switch viewport { + case .reported: kind = "reported" + case .derived: kind = "derived" + case .missing: kind = "missing" + } + NSLog( + "AGENT_DEVICE_RUNNER_PRIVATE_AX_ROOT_VIEWPORT kind=%@ root=%@ fullScreen=%@", + kind, + String(describing: rootFrame), + fullScreenFrame.isNull ? "none" : String(describing: fullScreenFrame) + ) + return viewport + } guard shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId) else { - return fallback + return fallback() } do { let reported = try runMainThreadWork( @@ -294,13 +310,35 @@ extension RunnerTests { try self.safeSnapshotViewport(app: app, readingOrientation: true) } if case .missing = reported { - return fallback + return fallback() } return reported } catch { NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_VIEWPORT_FALLBACK=%@", String(describing: error)) - return fallback + return fallback() + } + } + + /// The screen's frame when AXRuntime reports `app` as the primary app filling it, else `.null`. + private static func fullScreenFrame(of app: XCUIApplication) -> CGRect { + #if os(iOS) + return RunnerFullScreenApplicationFrame(app) + #else + return .null + #endif + } + + /// The viewport a private AX root establishes without XCTest. A root that is exactly the screen + /// AXRuntime reports the app filling, at the origin, is `.reported`: two independent reads agree + /// that window space is screen space, as on every iPhone app. Any other root stays `.derived`, + /// including a placed iPad window's local root (#4399) and a root that is the screen rotated. + static func privateAXRootViewport(rootFrame: CGRect, fullScreenFrame: CGRect) -> SnapshotViewport { + // CONSERVATIVE: orientation is unknown here, so a landscape root (the portrait screen with width + // and height swapped) stays `.derived` rather than guess which way to rotate it. + guard !fullScreenFrame.isNull, rootFrame.origin == .zero, rootFrame == fullScreenFrame else { + return .derived(box: rootFrame) } + return .reported(box: rootFrame, interfaceOrientation: RunnerInterfaceOrientation.unknown) } func privateAXRect(_ value: Any?) -> CGRect { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index dadac43409..91dbdfba8f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -133,6 +133,13 @@ FOUNDATION_EXPORT RunnerFullScreenEvidence RunnerFullScreenEvidenceFromForegroun CGRect screenFrame ); +// The screen's frame when AXRuntime reports `application` as the primary app filling it, else +// `CGRectNull`. The same probe the sole-screen route admits a non-phone app with; it reads two frames +// and walks no tree, so it answers while XCTest's accessibility channel is penalized. +#if TARGET_OS_IOS +FOUNDATION_EXPORT CGRect RunnerFullScreenApplicationFrame(id application); +#endif + FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass( Class cls, NSString *className, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 1d72e33eaa..eb736cd942 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -436,7 +436,10 @@ CGSize RunnerSoleScreenSize(void) { // One contained probe: it checks each selector before sending it and reads the target pid and both // frames inside one exception scope, so any failed read answers Unknown. The target pid comes from // the existing runner owner for that read (RunnerAXSnapshotBridge). -static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application) { +static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreenFrame( + id application, + CGRect *_Nullable screenFrame +) { if (application == nil) { return RunnerFullScreenEvidenceUnknown; } @@ -470,9 +473,22 @@ static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application if (exception != nil) { return RunnerFullScreenEvidenceUnknown; } + if (screenFrame != NULL) *screenFrame = screen; return RunnerFullScreenEvidenceFromForeground(targetProcessID, primaryProcessID, window, screen); } +static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application) { + return RunnerProbeApplicationFillsScreenFrame(application, NULL); +} + +CGRect RunnerFullScreenApplicationFrame(id application) { + CGRect screen = CGRectNull; + if (RunnerProbeApplicationFillsScreenFrame(application, &screen) != RunnerFullScreenEvidenceFillsScreen) { + return CGRectNull; + } + return screen; +} + #endif NSString * _Nullable RunnerResolveSoleScreen( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index daa25fbb4e..2125fc020a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -5,6 +5,36 @@ import AgentDeviceSnapshotPresentation // MARK: - In-bundle unit tests extension RunnerTests { + func testPrivateAXRootViewportReportsOnlyAFullScreenRootAtTheOrigin() { + let screen = CGRect(x: 0, y: 0, width: 440, height: 956) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: screen, fullScreenFrame: screen), + .reported(box: screen, interfaceOrientation: RunnerInterfaceOrientation.unknown) + ) + // No full-screen proof: a placed window's local root looks the same as a full-screen one (#4399). + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: screen, fullScreenFrame: .null), + .derived(box: screen) + ) + let placed = CGRect(x: 0, y: 0, width: 551, height: 813) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: placed, fullScreenFrame: CGRect(x: 0, y: 0, width: 1024, height: 1366)), + .derived(box: placed) + ) + let offset = CGRect(x: 0, y: 20, width: 440, height: 956) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: offset, fullScreenFrame: offset), + .derived(box: offset) + ) + // Measured on an iPhone 17 Pro Max simulator in landscape: the private AX root is in interface + // space, and AXRuntime reports the screen in native portrait space. + let landscape = CGRect(x: 0, y: 0, width: 956, height: 440) + XCTAssertEqual( + Self.privateAXRootViewport(rootFrame: landscape, fullScreenFrame: screen), + .derived(box: landscape) + ) + } + func testPrivateAXAttemptDepthsAppliesRememberedDepth() { XCTAssertEqual( Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), diff --git a/apple/runner/README.md b/apple/runner/README.md index a2a3626332..12f791e53b 100644 --- a/apple/runner/README.md +++ b/apple/runner/README.md @@ -133,7 +133,10 @@ Thus, if a viewport read takes too long, the plan skips the sweep instead of que Private-AX fallback keeps the viewport authority it acquired. When a penalty suppresses the XCTest viewport read, its root-derived box does not establish a screen origin. -The payload carries that uncertainty without a new probe or a changed channel budget. +Exception: AXRuntime can report the target as the primary app that fills the screen. If the root is that screen frame at the origin, the box is `reported`. +This probe reads two frames. It adds no XCTest tree read and does not change the channel budget. +The payload carries any remaining uncertainty. +In landscape, the root is in interface space and the screen frame is in native portrait space, so the root stays `derived`. A failure that XCTest records ends `testCommand` when the main-thread block that recorded it returns, whatever `continueAfterFailure` says. Mute XCTest's AX-server rejection in both fetch wordings, so XCTest does not record it. diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index f85fe95ffd..2e6b36160c 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -56,6 +56,7 @@ A placed iPad window publishes screen space in points, because its viewport, its The acquisition's `reported`, `derived`, or `missing` viewport evidence survives the runner boundary and capture transformations. Only `reported` establishes the screen origin for capture-derived input. A root-derived validation box preserves readable fallback trees but cannot upgrade unknown origin evidence. +A private-AX root is `reported` only when it is exactly the screen frame at the origin and AXRuntime reports the target as the primary app that fills that screen. Element mutations, mutating find, Maestro selector geometry, and snapshot-derived gesture bounds refuse an unknown origin with `window-origin-unknown` and `dispatched: no` before input. Independent native queries and authored screen coordinates retain their own authorities.