From 722671b8b18a438596abf89ef65922a7a5822bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 15:29:52 +0100 Subject: [PATCH 01/14] fix(ios): press the navigation bar back button and speed up the back swipe iOS back now reads the current tree in the same runner call for the navigation bar's system back button (identifier BackButton inside a navigation bar). When exactly one is hittable, it taps it with a synthesized tap and the daemon's settle observes the pop. No other button counts, whatever its label; an unreadable or penalized tree, or two candidates, fall back to the swipe. The swipe is resolved before settling, samples 0.05 s apart, and measures the change from the last still sample, so the separate just-before capture is gone. The post-swipe delay drops from 0.25 s to 0.1 s, which also shortens the required still span. The settle budget is 2 s. The gap-coverage and baseline checks still decide every success. Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerAppFixtures.m | 71 +++++++- .../RunnerTests+Navigation.swift | 170 ++++++++++++------ .../RunnerTests+NavigationTests.swift | 56 ++++-- apple/runner/RUNNER_PROTOCOL.md | 37 ++-- .../src/flag-definitions-action.ts | 4 +- packages/command-surface/src/system/index.ts | 2 +- packages/platform-apple/src/interactor.ts | 3 +- .../platform-apple/src/navigation/runtime.ts | 4 +- website/docs/docs/commands.md | 8 +- 9 files changed, 262 insertions(+), 93 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 1df45efe69..af8f27b1b2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -39,6 +39,65 @@ @interface AgentDeviceRunnerFixtureViewController : AgentDeviceRunnerViewController @end +// A real navigation stack for `back`. `--agent-device-back-navigation-bar` pushes a detail screen, +// so the bar shows UIKit's system back button; the interactive pop is off, so only that button can +// pop it. `--custom-back-item` instead puts a bar button titled "Back" on the root, which is not the +// system back button and must never be pressed. The root's label reports what happened. +@interface AgentDeviceBackNavigationFixture : UINavigationController +@property(nonatomic, strong) UILabel *status; +@end + +@implementation AgentDeviceBackNavigationFixture +- (instancetype)initWithCustomBackItem:(BOOL)customBackItem { + UIViewController *root = [UIViewController new]; + root.title = @"Root"; + root.view.backgroundColor = UIColor.systemBackgroundColor; + self = [super initWithRootViewController:root]; + if (!self) return nil; + UILabel *status = [UILabel new]; + status.text = @"Navigation root"; + status.accessibilityIdentifier = @"agent-device-back-navigation-status"; + status.translatesAutoresizingMaskIntoConstraints = NO; + [root.view addSubview:status]; + [NSLayoutConstraint activateConstraints:@[ + [status.centerXAnchor constraintEqualToAnchor:root.view.centerXAnchor], + [status.centerYAnchor constraintEqualToAnchor:root.view.centerYAnchor], + ]]; + self.status = status; + self.modalPresentationStyle = UIModalPresentationFullScreen; + if (customBackItem) { + root.navigationItem.leftBarButtonItem = + [[UIBarButtonItem alloc] initWithTitle:@"Back" style:UIBarButtonItemStylePlain target:self + action:@selector(customBackPressed)]; + } else { + UIViewController *detail = [UIViewController new]; + detail.title = @"Detail"; + detail.view.backgroundColor = UIColor.systemTealColor; + [self pushViewController:detail animated:NO]; + } + return self; +} + +- (void)viewDidLoad { + [super viewDidLoad]; + self.interactivePopGestureRecognizer.enabled = NO; + self.delegate = self; +} + +- (void)customBackPressed { + self.status.text = @"Custom back pressed"; +} + +- (void)navigationController:(UINavigationController *)navigationController + didShowViewController:(UIViewController *)viewController + animated:(BOOL)animated { + // Only the pushed stack ever shows its root again, and only by popping the detail screen. + if (viewController == self.viewControllers.firstObject && viewController.navigationItem.leftBarButtonItem == nil) { + self.status.text = @"Back pressed"; + } +} +@end + @interface AgentDeviceRunnerFixtureViewController () // Held for the app's lifetime: a window nobody retained would be gone before the test queried it. @property(nonatomic, strong) UIWindow *secondWindow; @@ -368,6 +427,14 @@ - (void)presentAlertDeclineChoiceFixture { - (void)viewDidAppear:(BOOL)animated { [super viewDidAppear:animated]; + if (!self.alertFixtureStarted && + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-back-navigation-bar"]) { + self.alertFixtureStarted = YES; + BOOL customBackItem = [NSProcessInfo.processInfo.arguments containsObject:@"--custom-back-item"]; + [self presentViewController:[[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:customBackItem] + animated:NO + completion:nil]; + } if (!self.alertFixtureStarted && ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"] || [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-affirmative-only"])) { @@ -608,12 +675,12 @@ - (void)viewDidLoad { } }]; } - // `--slow-toggle` repaints the whole screen every 0.5 s, slower than one verification delay, as + // `--slow-toggle` repaints the whole screen every 0.3 s, slower than one verification delay, as // a carousel does: one quiet pair of samples must not pass it for settled. if ([NSProcessInfo.processInfo.arguments containsObject:@"--slow-toggle"]) { __weak UIView *view = self.view; [self.animationTimer invalidate]; - self.animationTimer = [NSTimer scheduledTimerWithTimeInterval:0.5 repeats:YES block:^(NSTimer *timer) { + self.animationTimer = [NSTimer scheduledTimerWithTimeInterval:0.3 repeats:YES block:^(NSTimer *timer) { view.backgroundColor = [view.backgroundColor isEqual:UIColor.blackColor] ? UIColor.whiteColor : UIColor.blackColor; }]; } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index b53e22fd9e..08c7a12f4f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -2,7 +2,13 @@ import AgentDeviceSnapshotPresentation import XCTest extension RunnerTests { - static let backSwipeVerificationDelay: TimeInterval = 0.25 + /// UIKit's identifier for a navigation bar's system back button, in UIKit and SwiftUI stacks alike. + static let navigationBarBackButtonIdentifier = "BackButton" + /// The wait from the end of the swipe's contact to the sample it is compared against. An + /// interactive pop follows the contact across most of the width, so the screen has already moved. + static let backSwipeVerificationDelay: TimeInterval = 0.1 + /// The pause between two settle samples; a capture itself takes about 0.1 s on a simulator. + static let backSwipeSettleSampleInterval: TimeInterval = 0.05 /// The time the contact itself is allowed in the gap from the sample taken just before the swipe to /// the one after it. Both are resolved before that sample: a simulator's edge touch runs a 100 ms /// contact, and a device's synthesized fling runs 200 ms. @@ -11,7 +17,8 @@ extension RunnerTests { #else static let backGestureAllowanceMs = 400.0 #endif - static let backSwipeSettleSampleBudget = 8 + /// How long settling may sample before back swipes anyway and reports the outcome unverified. + static let backSwipeSettleBudgetMs = 2000.0 /// How long the screen must agree with itself before the swipe: as long as the gap it will be /// credited across, the contact allowance, one verification delay, and the slowest capture seen @@ -66,8 +73,9 @@ extension RunnerTests { } } - /// iOS `back` is the left-edge back swipe, verified by a change of the app's screen. A visible - /// back control is the caller's to press by ref; the runner does not guess at buttons. + /// iOS `back` presses the navigation bar's system back button when exactly one is on screen, read + /// from the live tree in this call. Otherwise it is the left-edge back swipe, verified by a change + /// of the app's screen. No other button counts, whatever its label. @MainActor func performInAppBack(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) @@ -88,7 +96,15 @@ extension RunnerTests { // One line on every exit, a refusal before the swipe and a thrown error included. var trace = BackSwipeTrace() do { - let outcome = try verifiedBackSwipe(app: app, trace: &trace) + guard let context = try synthesizedCoordinateContext(app: app) else { + let outcome = InAppBackOutcome.refused(Self.backScreenRefusal(.unresolvedWindow)) + NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(outcome: outcome)) + return outcome + } + if try pressNavigationBarBackButton(app: app, context: context) { + return .performed + } + let outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(outcome: outcome)) return outcome } catch { @@ -144,42 +160,88 @@ extension RunnerTests { } } + /// Presses the navigation bar's system back button when exactly one hittable one is on screen. + /// It is read only where the tree answers queries: on a tree XCTest cannot serialize, the query + /// would grind through XCTest's retries, so back swipes there. Two candidates, such as a sheet over + /// a stack, are ambiguous and fall back to the swipe too. True once the tap was dispatched; false + /// when nothing was sent. @MainActor - private func verifiedBackSwipe(app: XCUIApplication, trace: inout BackSwipeTrace) throws -> InAppBackOutcome { - guard let context = try synthesizedCoordinateContext(app: app) else { - return .refused(Self.backScreenRefusal(.unresolvedWindow)) + private func pressNavigationBarBackButton(app: XCUIApplication, context: SynthesizedCoordinateContext) throws -> Bool { + let started = currentUptimeMs() + let frame = navigationBarBackButtonFrame(app: app, context: context) + let lookupMs = Int((currentUptimeMs() - started).rounded()) + guard let frame else { + NSLog("AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON found=no lookupMs=%ld", lookupMs) + return false } - let before: (sample: Data, stableSinceMs: Double, settled: Bool) - switch settledBackVisualState(app: app, context: context, trace: &trace) { - case .success(let settled): before = settled - case .failure(let refusal): return .refused(Self.backScreenRefusal(refusal)) + // The synthesized tap returns once the touch is delivered; an XCTest tap would also wait out the + // pop animation, which the daemon's settle already observes. + let pressed: Bool + switch try performSynthesizedTapGesture(app: app, x: frame.midX, y: frame.midY, context: context) { + case .performed: + pressed = true + case .xctestFallback: + let fallback = try performGesture(app) { try tapAt(app: app, x: frame.midX, y: frame.midY, context: context) } + if case .performed = fallback.outcome { pressed = true } else { pressed = false } + case .refused: + // Nothing was sent, so the swipe is still safe to try. + pressed = false } - // Everything but the contact is resolved before the sample the change is measured from. + NSLog( + "AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON found=yes pressed=%@ lookupMs=%ld", + pressed ? "yes" : "no", lookupMs + ) + return pressed + } + + @MainActor + private func navigationBarBackButtonFrame(app: XCUIApplication, context: SynthesizedCoordinateContext) -> CGRect? { + guard !context.treeUnreadable, context.accessibilityHealth != .unavailable, + !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) + else { return nil } + let rejectedBefore = rejectedAxSnapshotQueryCount() + // A root walk that fails finds no button; the swipe still runs. + let candidates = ((try? elementQueries(app: app, matching: .navigationBar)) ?? []) + .flatMap { $0.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex } + .filter { $0.isHittable } + guard rejectedAxSnapshotQueryCount() == rejectedBefore, candidates.count == 1 else { return nil } + let frame = candidates[0].frame + return SnapshotGeometry.isPositiveFinite(frame) ? frame : nil + } + + private func rejectedAxSnapshotQueryCount() -> Int { + suppressedIssueLock.lock() + defer { suppressedIssueLock.unlock() } + return suppressedAxSnapshotIssueCount + } + + @MainActor + private func verifiedBackSwipe( + app: XCUIApplication, + context: SynthesizedCoordinateContext, + trace: inout BackSwipeTrace + ) throws -> InAppBackOutcome { + // Everything but the contact is resolved before settling, so the last still sample is the one + // the change is measured from and the swipe follows it at once. let dispatchGesture: @MainActor () throws -> TouchDispatch switch try prepareBackGesture(app: app, context: context) { case .ready(let dispatch): dispatchGesture = dispatch case .notSent(let message, let hint): return .refused(ErrorPayload(code: RunnerWireErrorCode.backSwipeNotDispatched, message: message, hint: hint)) } - // One more sample right before the swipe: the change credited to the swipe is measured from it. - let justBefore: Data - switch timedBackCapture(app: app, context: context, trace: &trace) { - case .success(let sample): justBefore = sample + let before: SettledBackSample + switch settledBackVisualState(app: app, context: context, trace: &trace) { + case .success(let settled): before = settled case .failure(let refusal): return .refused(Self.backScreenRefusal(refusal)) } - let justBeforeMs = currentUptimeMs() + let justBeforeMs = before.sampledMs trace.stableMs = justBeforeMs - before.stableSinceMs - let stillSettled = Self.backSwipeBaselineHolds( - settled: before.settled, - baseline: before.sample, - justBefore: justBefore - ) let dispatch = try dispatchGesture() trace.gestureMs = currentUptimeMs() - justBeforeMs if let refusal = Self.backGestureRefusal(dispatch) { return refusal } - guard stillSettled else { + guard before.settled else { return .unconfirmed( Self.backSwipeUnverified( because: "the app's screen was already changing on its own", @@ -191,7 +253,7 @@ extension RunnerTests { let after = timedBackCapture(app: app, context: context, trace: &trace) let afterMs = currentUptimeMs() trace.justBeforeToAfterMs = afterMs - justBeforeMs - let observation = Self.navigationVisualObservation(before: .success(justBefore), after: after) + let observation = Self.navigationVisualObservation(before: .success(before.sample), after: after) trace.changed = observation guard Self.backSwipeGapIsCovered( stableSinceMs: before.stableSinceMs, @@ -248,37 +310,45 @@ extension RunnerTests { } #if os(iOS) - /// Samples a verification delay apart until every sample of a run has agreed with its first one - /// for `backSwipeRequiredStillMs` of the slowest capture so far. `stableSinceMs` is when that run - /// began; `settled` is false when the budget ran out first. + /// The last settle sample: the baseline the swipe's change is measured from. `stableSinceMs` is + /// when the run of agreeing samples it ends began, and `sampledMs` when it was captured. + struct SettledBackSample { + let sample: Data + let stableSinceMs: Double + let sampledMs: Double + let settled: Bool + } + + /// Samples `backSwipeSettleSampleInterval` apart until every sample of a run has agreed with its + /// first one for `backSwipeRequiredStillMs` of the slowest capture so far. `settled` is false when + /// `backSwipeSettleBudgetMs` ran out first. @MainActor private func settledBackVisualState( app: XCUIApplication, context: SynthesizedCoordinateContext, trace: inout BackSwipeTrace - ) -> Result<(sample: Data, stableSinceMs: Double, settled: Bool), RunnerAppScreenCaptureFailure> { + ) -> Result { + let started = currentUptimeMs() var runStart: (sample: Data, ms: Double)? - var last = Data() - for index in 0.. 0 { sleepFor(Self.backSwipeVerificationDelay) } - trace.settleSamples = index + 1 + while true { + if trace.settleSamples > 0 { sleepFor(Self.backSwipeSettleSampleInterval) } + trace.settleSamples += 1 + let sample: Data switch timedBackCapture(app: app, context: context, trace: &trace) { - case .failure(let refusal): - return .failure(refusal) - case .success(let sample): - let now = currentUptimeMs() - last = sample - guard let start = runStart, !Self.visualSamplesDiffer(start.sample, sample) else { - runStart = (sample, now) - continue - } - if now - start.ms >= Self.backSwipeRequiredStillMs(maxCaptureMs: trace.maxCaptureMs) { - trace.settled = true - return .success((sample, start.ms, true)) - } + case .failure(let refusal): return .failure(refusal) + case .success(let captured): sample = captured + } + let now = currentUptimeMs() + let start = runStart.flatMap { Self.visualSamplesDiffer($0.sample, sample) ? nil : $0 } ?? (sample: sample, ms: now) + runStart = start + if now - start.ms >= Self.backSwipeRequiredStillMs(maxCaptureMs: trace.maxCaptureMs) { + trace.settled = true + return .success(SettledBackSample(sample: sample, stableSinceMs: start.ms, sampledMs: now, settled: true)) + } + if now - started >= Self.backSwipeSettleBudgetMs { + return .success(SettledBackSample(sample: sample, stableSinceMs: start.ms, sampledMs: now, settled: false)) } } - return .success((last, runStart?.ms ?? currentUptimeMs(), false)) } #endif @@ -368,12 +438,6 @@ extension RunnerTests { afterMs - justBeforeMs <= justBeforeMs - stableSinceMs } - /// Whether the swipe's effect can be measured: the screen settled, and the sample taken right - /// before the swipe still matches the settled baseline. - static func backSwipeBaselineHolds(settled: Bool, baseline: Data, justBefore: Data) -> Bool { - settled && !visualSamplesDiffer(baseline, justBefore) - } - /// What an observation of the swipe's before/after samples concludes. Kept apart from the sleep /// and the capture so the three-way decision is testable without a running app. static func backSwipeOutcome(observation: NavigationVisualObservation) -> InAppBackOutcome { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 6b58f0820e..cf8551f0a7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -44,9 +44,10 @@ extension RunnerTests { XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) } - /// A screen that changes on its own every 0.5 s is never still for the required span (the contact - /// allowance, one 0.25 s delay, and the slowest capture, over 0.5 s on a simulator), so it never - /// settles and back reports an unknown outcome instead of crediting the swipe. + /// A screen that changes on its own every 0.3 s, three verification delays apart, is never still + /// for the required span (the contact allowance, one 0.1 s delay, and the slowest capture, about + /// 0.4 s on a simulator), so it never settles and back reports an unknown outcome instead of + /// crediting the swipe. @MainActor func testInAppBackOnAScreenChangingSlowerThanOneDelayIsNotPerformed() throws { _ = try launchBackFixture(["--agent-device-back-regression", "--slow-toggle"]) @@ -58,6 +59,34 @@ extension RunnerTests { XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) } + /// The navigation bar's system back button is pressed from the live tree: the fixture's + /// interactive pop is off, so only that button can pop the detail screen. + @MainActor + func testInAppBackPressesTheNavigationBarBackButton() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + + guard case .performed = try performInAppBack(app: app) else { + return XCTFail("a visible navigation bar back button must be pressed") + } + let popped = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "Back pressed"), object: status) + XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed) + } + + /// A bar button titled "Back" is not the system back button, so back never presses it: it swipes, + /// and the swipe that changes nothing reports so. + @MainActor + func testInAppBackNeverPressesABarButtonOnlyTitledBack() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--custom-back-item"]) + defer { endBackFixture() } + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a bar button titled Back must not count as the back button") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNoChange) + XCTAssertEqual(status.label, "Navigation root") + } + @MainActor func testInAppBackDoesNotSwipeOffTheLitPanel() throws { let status = try launchBackFixture(["--agent-device-back-regression"]) @@ -266,6 +295,17 @@ extension RunnerTests { XCTAssertTrue(status.waitForFixtureExistence(timeout: appExistenceTimeout)) return status } + @MainActor + private func launchBackNavigationFixture(_ arguments: [String]) throws -> XCUIElement { + app.launchArguments = arguments + app.launch() + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + clearSnapshotXCTestChannelPenalty(reason: "navigation-test") + XCTAssertTrue(app.navigationBars.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) + return app.staticTexts["agent-device-back-navigation-status"] + } + @MainActor private func endBackFixture() { invalidateCachedTarget(reason: "unit_test_cleanup") @@ -439,16 +479,6 @@ extension RunnerTests { XCTAssertGreaterThanOrEqual(slow, Self.backSwipeDurationMs + Self.backSwipeVerificationDelay * 1000 + 600) } - /// A screen that moved between settling and the swipe has no baseline to credit a change against. - func testBackSwipeBaselineHoldsOnlyWhenSettledAndUnmovedBeforeTheSwipe() { - let baseline = grid(100) - let moved = grid(255) - - XCTAssertTrue(Self.backSwipeBaselineHolds(settled: true, baseline: baseline, justBefore: baseline)) - XCTAssertFalse(Self.backSwipeBaselineHolds(settled: true, baseline: baseline, justBefore: moved)) - XCTAssertFalse(Self.backSwipeBaselineHolds(settled: false, baseline: baseline, justBefore: baseline)) - } - func testBackSwipeOutcomeNamesNoChangeAndAnUnsampledScreenApart() { switch Self.backSwipeOutcome(observation: .unchanged) { case .unconfirmed(let payload): diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 477c72dc3b..bd5774d288 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -134,8 +134,14 @@ than silently applied: { "command": "backInApp", "appBundleId": "com.example.app" } ``` -`backInApp` is the only back command. On iOS the runner answers it with the left-edge back swipe and -verifies a change of the app's screen. Its failure codes are: +`backInApp` is the only back command. On iOS the runner first reads the current tree, in the same +call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a +navigation bar. When exactly one such button is hittable, it taps that button with a synthesized tap +and answers `ok` once the tap is delivered; the daemon's settle observes the pop. No other button +counts, whatever its label. The read is skipped when the tree cannot be queried (a rejected windows +query or a penalized XCTest channel), and two candidates, such as a sheet over a stack, also fall back. +Otherwise the runner answers with the left-edge back swipe and verifies a change of the app's screen. +Its failure codes are: | Code | When | Sent | | --- | --- | --- | @@ -144,32 +150,33 @@ verifies a change of the app's screen. Its failure codes are: | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | | `BACK_SWIPE_UNVERIFIED` | The swipe may have run; the outcome is unknown. | Possibly the swipe | -The settle window is a run of samples taken 0.25 s apart that agree for as long as the gap the -swipe will be credited across: a contact allowance, one 0.25 s delay, and the slowest capture seen -while settling. A slow host therefore settles for longer. The swipe is then resolved up to its -contact, and one more sample, taken right before the swipe, must still match the run: +The swipe is first resolved up to its contact. Then the settle window is a run of samples taken +0.05 s apart that agree for as long as the gap the swipe will be credited across: a contact +allowance, one 0.1 s delay, and the slowest capture seen while settling. A slow host therefore +settles for longer. The last sample of that run is the one the change is measured from, and the +swipe follows it at once: -| Platform | Resolved before the last sample | Left in the gap | Contact allowance | +| Platform | Resolved before settling | Left in the gap | Contact allowance | | --- | --- | --- | --- | | Simulator | the edge-touch plan, its display, and the HID touchscreen sender | the 100 ms edge contact | 0.2 s | | Device | the synthesized fling plan (2 pt inside the leading edge to 0.9 of the width, no hold), with every tree and window read it needs | the 200 ms fling | 0.4 s | -On both, every tree and window read happens during that preparation, before the just-before sample. +On both, every tree and window read happens during that preparation, before the first sample. -Settling takes at most 8 samples, so the wait before the swipe is at most 1.75 s of delays plus -eight captures, then the preparation and the last sample: about 2.6 s with the 0.1 s captures of a -local simulator, and about 5 s with the 0.4 s captures measured on one physical iPhone (iPhone 17 Pro, -iOS 27.0.1; https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541). Usually a still screen -settles after 3 samples. +Settling samples for at most 2 s, plus the capture in flight. A still screen usually settles after +4 samples, about 0.45 s on a local simulator with 0.1 s captures; with the 0.4 s captures measured +on one physical iPhone (iPhone 17 Pro, iOS 27.0.1; +https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541) the still span is 0.9 s. -Only a change from the last sample to the sample after the swipe counts, and only when the time +Only a change from the last sample to the sample taken 0.1 s after the swipe counts, and only when the time between those two does not exceed the time the screen had been still; otherwise the result is `BACK_SWIPE_UNVERIFIED`. Then the swipe may or may not have navigated: run `snapshot` to see the current screen before acting again. A screen that animates on its own more slowly than the settle window can still be misread as a back navigation; any finite window has that limit, so also run `snapshot` after `back` on a screen with auto-advancing content. -Each iOS back logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its +Each iOS back whose screen resolved logs `AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON` with +`found`, `pressed`, and `lookupMs` (`0` when the read was skipped). Each swipe logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its `outcome` (`performed`, `refused`, `unconfirmed`, or `error` for a thrown failure), `code`, `changed`, `stableMs`, `justBeforeToAfterMs`, `gestureMs`, `maxCaptureMs`, `settleSamples`, and `settled`. diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index e03b62befb..7713eb37aa 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -264,7 +264,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ setValue: 'in-app', usageLabel: '--in-app', usageDescription: - 'Back: use app-provided back UI when available; on iOS, the verified left-edge back swipe', + 'Back: use app-provided back UI when available; on iOS, the navigation bar back button, else the verified left-edge back swipe', projectConfig: true, recorded: false, }, @@ -276,7 +276,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ setValue: 'system', usageLabel: '--system', usageDescription: - 'Back: use system back input or gesture when available; on iOS, the same left-edge back swipe as --in-app', + 'Back: use system back input or gesture when available; on iOS, the same back as --in-app', projectConfig: true, recorded: false, }, diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 39594afc8e..dab71d2318 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes swipe from the left edge and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; + 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the system back button in the navigation bar when the current screen shows exactly one; otherwise they swipe from the left edge and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index f12218945c..56c8eeb510 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -148,7 +148,8 @@ export function createAppleInteractor( dispatched: 'no', }); } - // iOS has one back: the verified left-edge swipe, whichever mode was asked. + // iOS has one back, whichever mode was asked: the navigation bar back button, else the verified + // left-edge swipe. await runAppleRunnerCommand( device, { command: 'backInApp', appBundleId: runnerContext.appBundleId }, diff --git a/packages/platform-apple/src/navigation/runtime.ts b/packages/platform-apple/src/navigation/runtime.ts index 9d30160747..c47f724c8a 100644 --- a/packages/platform-apple/src/navigation/runtime.ts +++ b/packages/platform-apple/src/navigation/runtime.ts @@ -35,8 +35,8 @@ const backVisionOsUnavailable = Object.freeze({ reason: 'unsupported-platform-leaf', hint: 'iOS back is a left-edge swipe; visionOS has none. Press the visible back control by its ref.', } as const); -/** tvOS presses the remote's Menu button, macOS taps the window back control, and iOS swipes from - * the left edge. watchOS and visionOS have no back. */ +/** tvOS presses the remote's Menu button, macOS taps the window back control, and iOS presses the + * navigation bar back button or swipes from the left edge. watchOS and visionOS have no back. */ function appleBackFact(device: DeviceInfo): RuntimeOperationFact { if (device.kind !== 'simulator' && device.kind !== 'device') return backKindUnavailable; const os = resolveDeviceAppleOs(device); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d68250fd82..e6609de330 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,14 +124,14 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. It succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.25 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing: up to 8 samples 0.25 s apart, which is 1.75 s plus eight screen captures: about 2.6 s on a simulator, and about 5 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button on the current screen, read fresh in the same call. When the screen shows exactly one, `back` presses it and the result is confirmed by `--settle` or your next `snapshot`, as for `press`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, or when the app's accessibility tree cannot be read, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, sampling it 0.05 s apart for at most 2 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. - `reason: back_swipe_unverified`: the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, so nothing reached the device. - If the app's screen cannot be captured before the swipe, `back` does not swipe: it refuses with `COMMAND_FAILED` and `details.runnerErrorCode` set to `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, or `APP_SCREEN_OFF_LIT_PANEL`. Nothing is sent. - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. -- An app with a left-edge drawer may open the drawer instead of going back. When a back or close control is visible, press it by its ref. -- `back --system` sends system back input: the Android back keyevent, the iOS back swipe, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. +- An app with a left-edge drawer may open the drawer instead of going back. When a custom back or close control is visible, press it by its ref. +- `back --system` sends system back input: the Android back keyevent, the same iOS back as the default, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. - `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. In both default and scoped simulator sets, an iPhone Duo classified with more than one integrated panel refuses `portrait-upside-down` and rotates through the fold helper. It succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. Unresolved guest reads are retried inside the settle loop; an unresolved final read reports `orientation-panel-table-unavailable`. The picker offsets were measured on iPhone Duo; a new multi-panel model needs its own mapping. A simulator classified with one panel, or whose first classification read is unresolved, uses the standard runner rotation. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`. - `action-button` presses the iPhone or iPad Action Button once. It takes no arguments, no hold duration, and no `--settle`, and it doesn't activate the session app or drive Camera Control. Other platforms refuse it, and a model without the button (an iPhone SE, most iPad simulators) fails with `UNSUPPORTED_OPERATION`. @@ -169,7 +169,7 @@ Remote daemons and tenants: ```bash agent-device open "https://example.com" --platform ios # open link in web browser agent-device open MyApp "myapp://screen/to" --platform ios # open deep link to MyApp -agent-device back --platform ios # left-edge back swipe, verified by a screen change +agent-device back --platform ios # navigation bar back button, else a verified left-edge swipe agent-device reinstall MyApp /path/to/app-debug.apk --platform android --serial emulator-5554 agent-device open com.example.myapp --platform android --serial emulator-5554 --session my-session --relaunch agent-device metro reload From 2575f1808f39952491c056aa8808f25f12b04523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 15:56:59 +0100 Subject: [PATCH 02/14] fix(ios): confirm the back button tap and scale the settle budget The navigation bar back button path now reads the bars again after the tap (one XCTest snapshot per bar, 0.05 s apart, up to 1 s) and answers BACK_SWIPE_UNVERIFIED when neither the bar title nor its back button changed; a delivered tap is never followed by the swipe. The button log line is emitted on every exit. The back swipe settle budget grows with the slowest capture so a still screen can settle on a slow host. Help, flag text, command docs, and the runner protocol describe when the button is pressed and what each fallback answers. Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerAppFixtures.m | 14 +- .../RunnerTests+Navigation.swift | 190 ++++++++++++++---- .../RunnerTests+NavigationTests.swift | 32 +++ apple/runner/RUNNER_PROTOCOL.md | 39 ++-- .../src/flag-definitions-action.ts | 2 +- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 4 +- 7 files changed, 225 insertions(+), 58 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index af8f27b1b2..a08d0a5a6b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -41,14 +41,15 @@ @interface AgentDeviceRunnerFixtureViewController : AgentDeviceRunnerViewControl // A real navigation stack for `back`. `--agent-device-back-navigation-bar` pushes a detail screen, // so the bar shows UIKit's system back button; the interactive pop is off, so only that button can -// pop it. `--custom-back-item` instead puts a bar button titled "Back" on the root, which is not the +// pop it. `--blocked-back` keeps that button but replaces its action, so pressing it navigates +// nowhere. `--custom-back-item` instead puts a bar button titled "Back" on the root, which is not the // system back button and must never be pressed. The root's label reports what happened. @interface AgentDeviceBackNavigationFixture : UINavigationController @property(nonatomic, strong) UILabel *status; @end @implementation AgentDeviceBackNavigationFixture -- (instancetype)initWithCustomBackItem:(BOOL)customBackItem { +- (instancetype)initWithCustomBackItem:(BOOL)customBackItem blockedBack:(BOOL)blockedBack { UIViewController *root = [UIViewController new]; root.title = @"Root"; root.view.backgroundColor = UIColor.systemBackgroundColor; @@ -73,6 +74,11 @@ - (instancetype)initWithCustomBackItem:(BOOL)customBackItem { UIViewController *detail = [UIViewController new]; detail.title = @"Detail"; detail.view.backgroundColor = UIColor.systemTealColor; + if (blockedBack) { + if (@available(iOS 16.0, *)) { + detail.navigationItem.backAction = [UIAction actionWithHandler:^(__kindof UIAction *action) {}]; + } + } [self pushViewController:detail animated:NO]; } return self; @@ -431,7 +437,9 @@ - (void)viewDidAppear:(BOOL)animated { [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-back-navigation-bar"]) { self.alertFixtureStarted = YES; BOOL customBackItem = [NSProcessInfo.processInfo.arguments containsObject:@"--custom-back-item"]; - [self presentViewController:[[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:customBackItem] + BOOL blockedBack = [NSProcessInfo.processInfo.arguments containsObject:@"--blocked-back"]; + [self presentViewController:[[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:customBackItem + blockedBack:blockedBack] animated:NO completion:nil]; } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 08c7a12f4f..8ba6b8bc55 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -17,8 +17,21 @@ extension RunnerTests { #else static let backGestureAllowanceMs = 400.0 #endif - /// How long settling may sample before back swipes anyway and reports the outcome unverified. - static let backSwipeSettleBudgetMs = 2000.0 + /// The least time settling may sample before back swipes anyway and reports the outcome + /// unverified; `backSwipeSettleBudgetMs` extends it on a host that captures slowly. + static let backSwipeSettleBaseBudgetMs = 2000.0 + + /// How long settling may sample: the base budget, or, when longer, the time a still screen needs + /// to settle at the slowest capture seen: the first capture, the required still span, and one more + /// pause and capture for the sample that completes the span. A slow host still settles a still + /// screen instead of always reporting it unverified. + static func backSwipeSettleBudgetMs(maxCaptureMs: Double) -> Double { + let sampleIntervalMs = backSwipeSettleSampleInterval * 1000 + maxCaptureMs + return max( + backSwipeSettleBaseBudgetMs, + maxCaptureMs + backSwipeRequiredStillMs(maxCaptureMs: maxCaptureMs) + sampleIntervalMs + ) + } /// How long the screen must agree with itself before the swipe: as long as the gap it will be /// credited across, the contact allowance, one verification delay, and the slowest capture seen @@ -74,8 +87,9 @@ extension RunnerTests { } /// iOS `back` presses the navigation bar's system back button when exactly one is on screen, read - /// from the live tree in this call. Otherwise it is the left-edge back swipe, verified by a change - /// of the app's screen. No other button counts, whatever its label. + /// from the live tree in this call, and confirms that the bar changed. Otherwise it is the + /// left-edge back swipe, verified by a change of the app's screen. No other button counts, + /// whatever its label. @MainActor func performInAppBack(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) @@ -101,8 +115,8 @@ extension RunnerTests { NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(outcome: outcome)) return outcome } - if try pressNavigationBarBackButton(app: app, context: context) { - return .performed + if let pressed = try pressNavigationBarBackButton(app: app, context: context) { + return pressed } let outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(outcome: outcome)) @@ -160,53 +174,151 @@ extension RunnerTests { } } - /// Presses the navigation bar's system back button when exactly one hittable one is on screen. - /// It is read only where the tree answers queries: on a tree XCTest cannot serialize, the query - /// would grind through XCTest's retries, so back swipes there. Two candidates, such as a sheet over - /// a stack, are ambiguous and fall back to the swipe too. True once the tap was dispatched; false - /// when nothing was sent. + /// Presses the navigation bar's system back button when exactly one hittable one is on screen, + /// and confirms that the bar changed. It is read only where the tree answers queries: on a tree + /// XCTest cannot serialize, the query would grind through XCTest's retries, so back swipes there. + /// Two candidates, such as a sheet over a stack, are ambiguous and fall back to the swipe too. Nil + /// when nothing was sent, so the swipe is still safe; a delivered tap is never followed by a swipe. @MainActor - private func pressNavigationBarBackButton(app: XCUIApplication, context: SynthesizedCoordinateContext) throws -> Bool { + private func pressNavigationBarBackButton( + app: XCUIApplication, + context: SynthesizedCoordinateContext + ) throws -> InAppBackOutcome? { + var trace = NavigationBarBackTrace() + // One line on every exit, a thrown tap included. + defer { NSLog("AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON %@", trace.logLine) } + guard navigationBarBackButtonIsReadable(context: context) else { return nil } let started = currentUptimeMs() - let frame = navigationBarBackButtonFrame(app: app, context: context) - let lookupMs = Int((currentUptimeMs() - started).rounded()) - guard let frame else { - NSLog("AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON found=no lookupMs=%ld", lookupMs) - return false + let frames = navigationBarBackButtonFrames(app: app) + let before = frames?.count == 1 ? readNavigationBarBackState(app: app) : nil + trace.lookupMs = currentUptimeMs() - started + guard let frames, frames.count == 1 else { + trace.found = frames == nil ? "unreadable" : frames!.isEmpty ? "no" : "ambiguous" + return nil + } + guard let before else { + trace.found = "unreadable" + return nil } + trace.found = "yes" + let frame = frames[0] // The synthesized tap returns once the touch is delivered; an XCTest tap would also wait out the - // pop animation, which the daemon's settle already observes. - let pressed: Bool + // pop animation, which the bar check below reads sooner. switch try performSynthesizedTapGesture(app: app, x: frame.midX, y: frame.midY, context: context) { case .performed: - pressed = true + break case .xctestFallback: let fallback = try performGesture(app) { try tapAt(app: app, x: frame.midX, y: frame.midY, context: context) } - if case .performed = fallback.outcome { pressed = true } else { pressed = false } + // A refused XCTest tap sent nothing either. + guard case .performed = fallback.outcome else { return nil } case .refused: - // Nothing was sent, so the swipe is still safe to try. - pressed = false + return nil } - NSLog( - "AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON found=yes pressed=%@ lookupMs=%ld", - pressed ? "yes" : "no", lookupMs - ) - return pressed + trace.pressed = true + let tapped = currentUptimeMs() + let confirmed = navigationBarBackStateChanged(app: app, from: before) + trace.confirmMs = currentUptimeMs() - tapped + trace.confirmed = confirmed + return confirmed ? .performed : .unconfirmed(Self.navigationBarBackUnverified) } + /// What the back button path saw, logged as one line on every exit. + struct NavigationBarBackTrace { + /// `skipped` when the tree was not read, `unreadable` when a read failed, `no`, `ambiguous`, or `yes`. + var found = "skipped" + var pressed = false + var confirmed: Bool? + var lookupMs = 0.0 + var confirmMs: Double? + + var logLine: String { + func ms(_ value: Double?) -> String { value.map { String(Int($0.rounded())) } ?? "-" } + return [ + "found=\(found)", + "pressed=\(pressed ? "yes" : "no")", + "confirmed=\(confirmed.map { $0 ? "yes" : "no" } ?? "-")", + "lookupMs=\(ms(lookupMs))", + "confirmMs=\(ms(confirmMs))", + ].joined(separator: " ") + } + } + + /// Each navigation bar's identifier (the top screen's title in UIKit and SwiftUI stacks) with the + /// label and frame of its system back buttons. A pop removes, moves, or relabels the pressed button + /// or retitles its bar, so a change of this state is the evidence that the tap navigated. + struct NavigationBarBackState: Equatable { + let entries: [String] + } + + /// The pause between two reads of the bar after the tap, and how long they may run before the + /// outcome is reported unconfirmed. Only a tap that did not navigate spends the whole budget. + static let navigationBarBackConfirmInterval: TimeInterval = 0.05 + static let navigationBarBackConfirmBudgetMs = 1000.0 + + static let navigationBarBackUnverified = ErrorPayload( + code: RunnerWireErrorCode.backSwipeUnverified, + message: "back pressed the navigation bar back button, but the navigation bar did not change within 1 s. This is an unknown outcome.", + hint: "Take a snapshot to see where the app is." + ) + + /// Whether a later read of the bar differs from `before` within the confirmation budget. A read + /// that fails proves nothing, so it never confirms. @MainActor - private func navigationBarBackButtonFrame(app: XCUIApplication, context: SynthesizedCoordinateContext) -> CGRect? { - guard !context.treeUnreadable, context.accessibilityHealth != .unavailable, - !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) - else { return nil } + private func navigationBarBackStateChanged(app: XCUIApplication, from before: NavigationBarBackState) -> Bool { + let started = currentUptimeMs() + repeat { + if let after = readNavigationBarBackState(app: app), after != before { return true } + if currentUptimeMs() - started >= Self.navigationBarBackConfirmBudgetMs { return false } + sleepFor(Self.navigationBarBackConfirmInterval) + } while true + } + + @MainActor + private func navigationBarBackButtonIsReadable(context: SynthesizedCoordinateContext) -> Bool { + guard !context.treeUnreadable, context.accessibilityHealth != .unavailable else { return false } + return !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) + } + + /// The frames of the hittable system back buttons; nil when the tree did not answer: a root walk + /// that failed, or a query XCTest rejected. + @MainActor + private func navigationBarBackButtonFrames(app: XCUIApplication) -> [CGRect]? { let rejectedBefore = rejectedAxSnapshotQueryCount() - // A root walk that fails finds no button; the swipe still runs. - let candidates = ((try? elementQueries(app: app, matching: .navigationBar)) ?? []) + guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } + let frames = bars .flatMap { $0.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex } .filter { $0.isHittable } - guard rejectedAxSnapshotQueryCount() == rejectedBefore, candidates.count == 1 else { return nil } - let frame = candidates[0].frame - return SnapshotGeometry.isPositiveFinite(frame) ? frame : nil + .map(\.frame) + guard rejectedAxSnapshotQueryCount() == rejectedBefore, frames.allSatisfy(SnapshotGeometry.isPositiveFinite) + else { return nil } + return frames + } + + /// One XCTest snapshot per bar, so a read after the tap costs a query per bar rather than one per + /// attribute. Nil when the tree did not answer. + @MainActor + private func readNavigationBarBackState(app: XCUIApplication) -> NavigationBarBackState? { + let rejectedBefore = rejectedAxSnapshotQueryCount() + guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } + var entries: [String] = [] + for bar in bars.flatMap({ $0.allElementsBoundByIndex }) { + guard let snapshot = try? bar.snapshot() else { return nil } + entries.append(Self.navigationBarBackEntry(snapshot)) + } + guard rejectedAxSnapshotQueryCount() == rejectedBefore else { return nil } + return NavigationBarBackState(entries: entries) + } + + static func navigationBarBackEntry(_ bar: XCUIElementSnapshot) -> String { + var buttons: [String] = [] + var pending = bar.children + while let element = pending.popLast() { + if element.identifier == navigationBarBackButtonIdentifier { + buttons.append("\(element.label)\u{1F}\(NSCoder.string(for: element.frame))") + } + pending.append(contentsOf: element.children) + } + return ([bar.identifier] + buttons).joined(separator: "\u{1E}") } private func rejectedAxSnapshotQueryCount() -> Int { @@ -321,7 +433,7 @@ extension RunnerTests { /// Samples `backSwipeSettleSampleInterval` apart until every sample of a run has agreed with its /// first one for `backSwipeRequiredStillMs` of the slowest capture so far. `settled` is false when - /// `backSwipeSettleBudgetMs` ran out first. + /// `backSwipeSettleBudgetMs` of the slowest capture so far ran out first. @MainActor private func settledBackVisualState( app: XCUIApplication, @@ -345,7 +457,7 @@ extension RunnerTests { trace.settled = true return .success(SettledBackSample(sample: sample, stableSinceMs: start.ms, sampledMs: now, settled: true)) } - if now - started >= Self.backSwipeSettleBudgetMs { + if now - started >= Self.backSwipeSettleBudgetMs(maxCaptureMs: trace.maxCaptureMs) { return .success(SettledBackSample(sample: sample, stableSinceMs: start.ms, sampledMs: now, settled: false)) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index cf8551f0a7..b8c8e23057 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -73,6 +73,21 @@ extension RunnerTests { XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed) } + /// A delivered tap on the system back button is not a navigation: when the bar does not change, + /// back reports an unknown outcome and never swipes after the tap. + @MainActor + func testInAppBackReportsAPressedBackButtonThatDoesNotNavigateAsUnverified() throws { + guard #available(iOS 16.0, *) else { throw XCTSkip("UINavigationItem.backAction needs iOS 16") } + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--blocked-back"]) + defer { endBackFixture() } + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a back button tap that leaves the bar unchanged must not read as performed") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertTrue(app.navigationBars["Detail"].exists) + } + /// A bar button titled "Back" is not the system back button, so back never presses it: it swipes, /// and the swipe that changes nothing reports so. @MainActor @@ -479,6 +494,23 @@ extension RunnerTests { XCTAssertGreaterThanOrEqual(slow, Self.backSwipeDurationMs + Self.backSwipeVerificationDelay * 1000 + 600) } + /// A still screen settles before the budget runs out at any capture time: the settle loop's + /// timeline, replayed with fixed captures, reaches the required still span first. On a simulator + /// the base budget holds; a 1 s capture extends it. + func testBackSwipeSettleBudgetLetsAStillScreenSettleOnASlowHost() { + XCTAssertEqual(Self.backSwipeSettleBudgetMs(maxCaptureMs: 100), Self.backSwipeSettleBaseBudgetMs) + for captureMs in stride(from: 50.0, through: 2000.0, by: 50.0) { + let required = Self.backSwipeRequiredStillMs(maxCaptureMs: captureMs) + let budget = Self.backSwipeSettleBudgetMs(maxCaptureMs: captureMs) + var now = captureMs + let runStart = now + while now - runStart < required { + XCTAssertLessThan(now, budget, "capture \(captureMs) ms ran out of budget before settling") + now += Self.backSwipeSettleSampleInterval * 1000 + captureMs + } + } + } + func testBackSwipeOutcomeNamesNoChangeAndAnUnsampledScreenApart() { switch Self.backSwipeOutcome(observation: .unchanged) { case .unconfirmed(let payload): diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index bd5774d288..75d82d3e07 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -136,22 +136,35 @@ than silently applied: `backInApp` is the only back command. On iOS the runner first reads the current tree, in the same call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a -navigation bar. When exactly one such button is hittable, it taps that button with a synthesized tap -and answers `ok` once the tap is delivered; the daemon's settle observes the pop. No other button -counts, whatever its label. The read is skipped when the tree cannot be queried (a rejected windows -query or a penalized XCTest channel), and two candidates, such as a sheet over a stack, also fall back. -Otherwise the runner answers with the left-edge back swipe and verifies a change of the app's screen. -Its failure codes are: +navigation bar. It taps that button with a synthesized tap only when exactly one live, hittable +candidate is found in a readable tree. No other button counts, whatever its label. After the tap it +reads every navigation bar's identifier and the label and frame of its `BackButton` children, one +XCTest snapshot per bar, with a 0.05 s pause between reads for up to 1 s. A change from the read +taken before the tap answers `ok`. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; +a delivered tap is never followed by the swipe. + +The runner swipes instead when no tap was sent: + +- The tree read is skipped: the window resolution marked the tree unreadable, accessibility is + unavailable, or the XCTest snapshot channel is penalized for the app. +- The read fails or XCTest rejects one of its queries. +- No candidate is found, or two or more are (such as a sheet over a stack). +- The synthesized tap and its XCTest fallback are both refused before anything is sent. + +A tap whose outcome is unknown after it may have been sent throws, and neither the bar check nor +the swipe follows. The swipe verifies a change of the app's screen. When XCTest cannot resolve the +app's window, it samples the sole screen only on an iPhone, or on an iPad while the app provably +fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure codes are: | Code | When | Sent | | --- | --- | --- | | `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, `APP_SCREEN_OFF_LIT_PANEL` | The screen could not be sampled before the swipe. | Nothing | | `BACK_SWIPE_NOT_DISPATCHED` | The swipe was refused before any touch went down, or the platform has no back gesture (visionOS; the daemon refuses it first). | Nothing | | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | -| `BACK_SWIPE_UNVERIFIED` | The swipe may have run; the outcome is unknown. | Possibly the swipe | +| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar did not change within 1 s; the outcome is unknown. | Possibly the swipe, or the tap | -The swipe is first resolved up to its contact. Then the settle window is a run of samples taken -0.05 s apart that agree for as long as the gap the swipe will be credited across: a contact +The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a +0.05 s pause between captures plus each capture's own time, that agree for as long as the gap the swipe will be credited across: a contact allowance, one 0.1 s delay, and the slowest capture seen while settling. A slow host therefore settles for longer. The last sample of that run is the one the change is measured from, and the swipe follows it at once: @@ -163,7 +176,7 @@ swipe follows it at once: On both, every tree and window read happens during that preparation, before the first sample. -Settling samples for at most 2 s, plus the capture in flight. A still screen usually settles after +Settling samples for at most 2 s, plus the capture in flight. On a slow host the budget grows to the first capture, the still span, and one more pause and capture, so a still screen settles instead of always answering `BACK_SWIPE_UNVERIFIED`. A still screen usually settles after 4 samples, about 0.45 s on a local simulator with 0.1 s captures; with the 0.4 s captures measured on one physical iPhone (iPhone 17 Pro, iOS 27.0.1; https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541) the still span is 0.9 s. @@ -175,8 +188,10 @@ current screen before acting again. A screen that animates on its own more slowl window can still be misread as a back navigation; any finite window has that limit, so also run `snapshot` after `back` on a screen with auto-advancing content. -Each iOS back whose screen resolved logs `AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON` with -`found`, `pressed`, and `lookupMs` (`0` when the read was skipped). Each swipe logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its +Each iOS back whose window context resolved logs one `AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON` +line on every exit of the button path, a thrown tap included, with `found` (`skipped`, +`unreadable`, `no`, `ambiguous`, or `yes`), `pressed`, `confirmed`, `lookupMs` (`0` when the read was +skipped), and `confirmMs`. Each swipe logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its `outcome` (`performed`, `refused`, `unconfirmed`, or `error` for a thrown failure), `code`, `changed`, `stableMs`, `justBeforeToAfterMs`, `gestureMs`, `maxCaptureMs`, `settleSamples`, and `settled`. diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 7713eb37aa..284edc3106 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -264,7 +264,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ setValue: 'in-app', usageLabel: '--in-app', usageDescription: - 'Back: use app-provided back UI when available; on iOS, the navigation bar back button, else the verified left-edge back swipe', + 'Back: use app-provided back UI when available; on iOS, the one hittable navigation bar back button, confirmed by a bar change, else the verified left-edge back swipe', projectConfig: true, recorded: false, }, diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index dab71d2318..deb271f0d0 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the system back button in the navigation bar when the current screen shows exactly one; otherwise they swipe from the left edge and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; + 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the navigation bar then changes; with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e6609de330..cd4af3938a 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,9 +124,9 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button on the current screen, read fresh in the same call. When the screen shows exactly one, `back` presses it and the result is confirmed by `--settle` or your next `snapshot`, as for `press`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, or when the app's accessibility tree cannot be read, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, sampling it 0.05 s apart for at most 2 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found and the app's accessibility tree is readable: the tree is not read when XCTest could not query it earlier in the session. After the tap, `back` reads the navigation bars again, about every 0.05 s for up to 1 s, and succeeds once the bar's title or its back button changes; if neither changes, it reports `back_swipe_unverified` and does not swipe. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on an iPhone always, and on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), for at most 2 s, or longer on a host whose captures are slow enough that a still screen needs more time to settle. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. -- `reason: back_swipe_unverified`: the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. +- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, so nothing reached the device. - If the app's screen cannot be captured before the swipe, `back` does not swipe: it refuses with `COMMAND_FAILED` and `details.runnerErrorCode` set to `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, or `APP_SCREEN_OFF_LIT_PANEL`. Nothing is sent. - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. From 32278daabb66d9f396abb018f9b4955a50f89d72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 16:04:33 +0100 Subject: [PATCH 03/14] docs(back): scope the iOS swipe fallback and state the settle budget plainly Co-Authored-By: Claude Opus 5.5 --- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index deb271f0d0..bc05cc7679 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the navigation bar then changes; with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; + 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the navigation bar then changes; with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index cd4af3938a..55f6ad2040 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,7 +124,7 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found and the app's accessibility tree is readable: the tree is not read when XCTest could not query it earlier in the session. After the tap, `back` reads the navigation bars again, about every 0.05 s for up to 1 s, and succeeds once the bar's title or its back button changes; if neither changes, it reports `back_swipe_unverified` and does not swipe. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on an iPhone always, and on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), for at most 2 s, or longer on a host whose captures are slow enough that a still screen needs more time to settle. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found and the app's accessibility tree is readable: the tree is not read when XCTest could not query it earlier in the session. After the tap, `back` reads the navigation bars again, about every 0.05 s for up to 1 s, and succeeds once the bar's title or its back button changes; if neither changes, it reports `back_swipe_unverified` and does not swipe. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, so nothing reached the device. From 0d17e3ecec254605d55dc8ce3fc06a03030589e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 17:11:30 +0100 Subject: [PATCH 04/14] fix(ios): hold XCTest issues in the back button reads Every navigation bar read in iOS back runs inside holdingRecordedIssues. A held issue before the tap means no button, so back swipes. A held issue after the tap proves nothing, so back answers back_swipe_unverified and never swipes. This replaces the suppressed-issue counter diff, which only saw kAXError issues. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Navigation.swift | 83 ++++++++++++------- .../RunnerTests.swift | 10 +++ .../RunnerTests+NavigationTests.swift | 41 +++++++++ 3 files changed, 105 insertions(+), 29 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 8ba6b8bc55..9441468ad6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -186,11 +186,16 @@ extension RunnerTests { ) throws -> InAppBackOutcome? { var trace = NavigationBarBackTrace() // One line on every exit, a thrown tap included. - defer { NSLog("AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON %@", trace.logLine) } + defer { + NSLog("AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON %@", trace.logLine) +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + lastNavigationBarBackTraceForTesting = trace +#endif + } guard navigationBarBackButtonIsReadable(context: context) else { return nil } let started = currentUptimeMs() let frames = navigationBarBackButtonFrames(app: app) - let before = frames?.count == 1 ? readNavigationBarBackState(app: app) : nil + let before = frames?.count == 1 ? readNavigationBarBackState(app: app, phase: .lookup) : nil trace.lookupMs = currentUptimeMs() - started guard let frames, frames.count == 1 else { trace.found = frames == nil ? "unreadable" : frames!.isEmpty ? "no" : "ambiguous" @@ -267,7 +272,7 @@ extension RunnerTests { private func navigationBarBackStateChanged(app: XCUIApplication, from before: NavigationBarBackState) -> Bool { let started = currentUptimeMs() repeat { - if let after = readNavigationBarBackState(app: app), after != before { return true } + if let after = readNavigationBarBackState(app: app, phase: .confirmation), after != before { return true } if currentUptimeMs() - started >= Self.navigationBarBackConfirmBudgetMs { return false } sleepFor(Self.navigationBarBackConfirmInterval) } while true @@ -279,34 +284,60 @@ extension RunnerTests { return !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) } - /// The frames of the hittable system back buttons; nil when the tree did not answer: a root walk - /// that failed, or a query XCTest rejected. + /// Which side of the tap a read of the bar is on: a held issue in the lookup means no button, so + /// back swipes; one in the confirmation proves nothing, so it never confirms. + enum NavigationBarBackReadPhase { + case lookup + case confirmation + } + + /// Runs one read of the bar with its XCTest issues held. A button gone mid-transition or a hung + /// query records an issue, and an unheld issue would turn the command into a recorded failure. + /// Nil when the read recorded an issue or raised an exception. + @MainActor + private func heldNavigationBarRead(_ phase: NavigationBarBackReadPhase, _ read: () -> T?) -> T? { + var value: T? + let held = holdingRecordedIssues { + value = read() +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + if navigationBarBackReadIssueForTesting == phase { + record(XCTIssue(type: .assertionFailure, compactDescription: "Back button vanished during the navigation bar read")) + } +#endif + } + guard held.issues.isEmpty, held.exceptionMessage == nil else { return nil } + return value + } + + /// The frames of the hittable system back buttons; nil when the tree did not answer. @MainActor private func navigationBarBackButtonFrames(app: XCUIApplication) -> [CGRect]? { - let rejectedBefore = rejectedAxSnapshotQueryCount() - guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } - let frames = bars - .flatMap { $0.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex } - .filter { $0.isHittable } - .map(\.frame) - guard rejectedAxSnapshotQueryCount() == rejectedBefore, frames.allSatisfy(SnapshotGeometry.isPositiveFinite) - else { return nil } - return frames + heldNavigationBarRead(.lookup) { + guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } + let frames = bars + .flatMap { $0.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex } + .filter { $0.isHittable } + .map(\.frame) + return frames.allSatisfy(SnapshotGeometry.isPositiveFinite) ? frames : nil + } } /// One XCTest snapshot per bar, so a read after the tap costs a query per bar rather than one per /// attribute. Nil when the tree did not answer. @MainActor - private func readNavigationBarBackState(app: XCUIApplication) -> NavigationBarBackState? { - let rejectedBefore = rejectedAxSnapshotQueryCount() - guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } - var entries: [String] = [] - for bar in bars.flatMap({ $0.allElementsBoundByIndex }) { - guard let snapshot = try? bar.snapshot() else { return nil } - entries.append(Self.navigationBarBackEntry(snapshot)) + private func readNavigationBarBackState( + app: XCUIApplication, + phase: NavigationBarBackReadPhase + ) -> NavigationBarBackState? { + heldNavigationBarRead(phase) { + guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } + var entries: [String] = [] + for bar in bars.flatMap({ $0.allElementsBoundByIndex }) { + guard let snapshot = try? bar.snapshot() else { return nil } + entries.append(Self.navigationBarBackEntry(snapshot)) + } + return NavigationBarBackState(entries: entries) } - guard rejectedAxSnapshotQueryCount() == rejectedBefore else { return nil } - return NavigationBarBackState(entries: entries) } static func navigationBarBackEntry(_ bar: XCUIElementSnapshot) -> String { @@ -321,12 +352,6 @@ extension RunnerTests { return ([bar.identifier] + buttons).joined(separator: "\u{1E}") } - private func rejectedAxSnapshotQueryCount() -> Int { - suppressedIssueLock.lock() - defer { suppressedIssueLock.unlock() } - return suppressedAxSnapshotIssueCount - } - @MainActor private func verifiedBackSwipe( app: XCUIApplication, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b0b3e8f569..552ea84fcc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -218,6 +218,16 @@ final class RunnerTests: XCTestCase { /// Records an XCTest issue during the lit-panel window walk, as a window that vanishes mid-walk does. var litPanelWindowWalkIssueForTesting = false +#if os(iOS) + /// Records an XCTest issue during every navigation bar read of this phase of iOS `back`, as a back + /// button that vanishes mid-transition does. + var navigationBarBackReadIssueForTesting: NavigationBarBackReadPhase? + + /// What the last iOS `back` saw of the navigation bar back button, so a test can tell the button + /// path from the swipe when both navigate. + var lastNavigationBarBackTraceForTesting: NavigationBarBackTrace? +#endif + static let injectedTapFailureFlagPathForTesting = "/tmp/agent-device-inject-tap-recorded-failure-for-testing" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index b8c8e23057..0d29c4ad08 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -69,6 +69,7 @@ extension RunnerTests { guard case .performed = try performInAppBack(app: app) else { return XCTFail("a visible navigation bar back button must be pressed") } + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true, "the button, not the swipe, popped") let popped = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "Back pressed"), object: status) XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed) } @@ -88,6 +89,46 @@ extension RunnerTests { XCTAssertTrue(app.navigationBars["Detail"].exists) } + /// An XCTest issue while the back button is looked up, such as a button gone mid-transition, means + /// no button: back answers from the swipe path. An issue that escaped the read would fail this + /// test as it fails the command with a recorded failure. + @MainActor + func testInAppBackWithAnIssueInTheButtonLookupAnswersFromTheSwipe() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + navigationBarBackReadIssueForTesting = .lookup + defer { navigationBarBackReadIssueForTesting = nil } + + let response = try execute(command: try runnerCommandFixture( + #"{"command":"backInApp","commandId":"back-lookup-issue","appBundleId":"com.callstack.agentdevice.runner"}"# + )) + + XCTAssertTrue(response.ok, "the swipe answered: \(String(describing: response.error))") + let trace = try XCTUnwrap(lastNavigationBarBackTraceForTesting) + XCTAssertEqual(trace.found, "unreadable") + XCTAssertFalse(trace.pressed, "back answered from the swipe path") + } + + /// An XCTest issue in every read after a delivered tap proves nothing: back answers unverified and + /// never swipes after the tap. An escaped issue would fail this test. + @MainActor + func testInAppBackWithAnIssueInTheConfirmationAnswersUnverified() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + navigationBarBackReadIssueForTesting = .confirmation + defer { navigationBarBackReadIssueForTesting = nil } + + let response = try execute(command: try runnerCommandFixture( + #"{"command":"backInApp","commandId":"back-confirmation-issue","appBundleId":"com.callstack.agentdevice.runner"}"# + )) + + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + navigationBarBackReadIssueForTesting = nil + let popped = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "Back pressed"), object: status) + XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed, "the tap itself was delivered") + } + /// A bar button titled "Back" is not the system back button, so back never presses it: it swipes, /// and the swipe that changes nothing reports so. @MainActor From dbf400b95e8cddcae40957ff681e612c67a1f80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 17:38:37 +0100 Subject: [PATCH 05/14] fix(ios): hold the back bar still before the tap and surface a hung query - Read the bar state twice around the button lookup and tap only when the two reads agree and contain the button's frame, so a push still animating is not taken for the state a pop changes. - Compare bar count, identifier, static-text titles, and back buttons. - Start bar reads only within the 1 s budget, checked before each bar's snapshot; document what bounds a read in flight. - Re-record a hung XCTest query as record(_:) requires, send nothing more: BACK_SWIPE_NOT_DISPATCHED before the tap, BACK_SWIPE_UNVERIFIED after it. - Drop the dead outcome guard on the XCTest tap fallback. - Correct the visionOS back hints; a thrown tap no longer logs a swipe verification error line. - Test the two-candidate rule, the skipped read, and hung queries; the lookup-issue test asserts the swipe path, not a swipe outcome. - Name the simulator still span in commands.md. Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerAppFixtures.m | 39 ++- .../RunnerTests+Navigation.swift | 273 +++++++++++++----- .../RunnerTests.swift | 14 +- .../RunnerTests+NavigationTests.swift | 101 ++++++- apple/runner/RUNNER_PROTOCOL.md | 28 +- .../platform-apple/src/navigation/runtime.ts | 2 +- packages/platform-apple/src/runtime.test.ts | 2 +- website/docs/docs/commands.md | 8 +- 8 files changed, 369 insertions(+), 98 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index a08d0a5a6b..7870118a59 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -104,6 +104,36 @@ - (void)navigationController:(UINavigationController *)navigationController } @end +// `--two-stacks` shows two pushed navigation stacks, one above the other, so two hittable system +// back buttons are on screen at once and neither is the one `back` should press. +@interface AgentDeviceTwoStacksBackFixture : UIViewController +@end + +@implementation AgentDeviceTwoStacksBackFixture +- (void)viewDidLoad { + [super viewDidLoad]; + self.view.backgroundColor = UIColor.systemBackgroundColor; + UIStackView *stack = [UIStackView new]; + stack.axis = UILayoutConstraintAxisVertical; + stack.distribution = UIStackViewDistributionFillEqually; + stack.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:stack]; + [NSLayoutConstraint activateConstraints:@[ + [stack.topAnchor constraintEqualToAnchor:self.view.safeAreaLayoutGuide.topAnchor], + [stack.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor], + [stack.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor], + [stack.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor], + ]]; + for (NSInteger index = 0; index < 2; index++) { + AgentDeviceBackNavigationFixture *child = [[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:NO + blockedBack:NO]; + [self addChildViewController:child]; + [stack addArrangedSubview:child.view]; + [child didMoveToParentViewController:self]; + } +} +@end + @interface AgentDeviceRunnerFixtureViewController () // Held for the app's lifetime: a window nobody retained would be gone before the test queried it. @property(nonatomic, strong) UIWindow *secondWindow; @@ -438,10 +468,11 @@ - (void)viewDidAppear:(BOOL)animated { self.alertFixtureStarted = YES; BOOL customBackItem = [NSProcessInfo.processInfo.arguments containsObject:@"--custom-back-item"]; BOOL blockedBack = [NSProcessInfo.processInfo.arguments containsObject:@"--blocked-back"]; - [self presentViewController:[[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:customBackItem - blockedBack:blockedBack] - animated:NO - completion:nil]; + UIViewController *fixture = [NSProcessInfo.processInfo.arguments containsObject:@"--two-stacks"] + ? [AgentDeviceTwoStacksBackFixture new] + : [[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:customBackItem blockedBack:blockedBack]; + fixture.modalPresentationStyle = UIModalPresentationFullScreen; + [self presentViewController:fixture animated:NO completion:nil]; } if (!self.alertFixtureStarted && ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"] || diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 9441468ad6..be0f196baa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -107,32 +107,41 @@ extension RunnerTests { _ = pressTvRemote(.menu) return .performed #elseif os(iOS) - // One line on every exit, a refusal before the swipe and a thrown error included. + // The swipe logs one line on every exit, a refusal before it and a thrown error included. The + // button path logs its own line on every exit, a thrown tap included. var trace = BackSwipeTrace() + func logSwipe(_ line: String) { NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", line) } + let resolved: SynthesizedCoordinateContext? + do { + resolved = try synthesizedCoordinateContext(app: app) + } catch { + logSwipe(trace.logLine(name: "error", code: String(describing: type(of: error)))) + throw error + } + guard let context = resolved else { + let outcome = InAppBackOutcome.refused(Self.backScreenRefusal(.unresolvedWindow)) + logSwipe(trace.logLine(outcome: outcome)) + return outcome + } + if let pressed = try pressNavigationBarBackButton(app: app, context: context) { + return pressed + } do { - guard let context = try synthesizedCoordinateContext(app: app) else { - let outcome = InAppBackOutcome.refused(Self.backScreenRefusal(.unresolvedWindow)) - NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(outcome: outcome)) - return outcome - } - if let pressed = try pressNavigationBarBackButton(app: app, context: context) { - return pressed - } let outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) - NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(outcome: outcome)) + logSwipe(trace.logLine(outcome: outcome)) return outcome } catch { - NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(name: "error", code: String(describing: type(of: error)))) + logSwipe(trace.logLine(name: "error", code: String(describing: type(of: error)))) throw error } #else - // visionOS has no edge swipe that pops a navigation stack. The daemon refuses it at the back - // capability fact, so this answers only a direct runner request, with nothing sent. + // visionOS has no back command. The daemon refuses it at the back capability fact, so this + // answers only a direct runner request, with nothing sent. return .refused( ErrorPayload( code: RunnerWireErrorCode.backSwipeNotDispatched, - message: "visionOS has no back gesture", - hint: "iOS back is a left-edge swipe; visionOS has none. Press the visible back control by its ref." + message: "visionOS has no back command", + hint: "Press the visible back control by its ref." ) ) #endif @@ -177,8 +186,9 @@ extension RunnerTests { /// Presses the navigation bar's system back button when exactly one hittable one is on screen, /// and confirms that the bar changed. It is read only where the tree answers queries: on a tree /// XCTest cannot serialize, the query would grind through XCTest's retries, so back swipes there. - /// Two candidates, such as a sheet over a stack, are ambiguous and fall back to the swipe too. Nil - /// when nothing was sent, so the swipe is still safe; a delivered tap is never followed by a swipe. + /// Two candidates, such as a sheet over a stack, are ambiguous and fall back to the swipe too, as + /// does a bar still moving after the read budget. Nil when nothing was sent, so the swipe is still + /// safe; a delivered tap is never followed by a swipe. A hung XCTest query answers without either. @MainActor private func pressNavigationBarBackButton( app: XCUIApplication, @@ -194,45 +204,58 @@ extension RunnerTests { } guard navigationBarBackButtonIsReadable(context: context) else { return nil } let started = currentUptimeMs() - let frames = navigationBarBackButtonFrames(app: app) - let before = frames?.count == 1 ? readNavigationBarBackState(app: app, phase: .lookup) : nil + let lookup = stillNavigationBarBackButton(app: app, deadlineMs: started + Self.navigationBarBackReadBudgetMs) trace.lookupMs = currentUptimeMs() - started - guard let frames, frames.count == 1 else { - trace.found = frames == nil ? "unreadable" : frames!.isEmpty ? "no" : "ambiguous" - return nil - } - guard let before else { - trace.found = "unreadable" + let before: NavigationBarBackState + let frame: CGRect + switch lookup { + case .still(let state, let button): + before = state + frame = button + case .notFound(let found): + trace.found = found return nil + case .hung: + trace.found = "hung" + return .refused(Self.navigationBarBackHungLookup) } trace.found = "yes" - let frame = frames[0] // The synthesized tap returns once the touch is delivered; an XCTest tap would also wait out the - // pop animation, which the bar check below reads sooner. + // pop animation, which the bar check below reads sooner. On iOS a readable tree's XCTest tap + // either runs or throws, so its fallback needs no outcome check. switch try performSynthesizedTapGesture(app: app, x: frame.midX, y: frame.midY, context: context) { case .performed: break case .xctestFallback: - let fallback = try performGesture(app) { try tapAt(app: app, x: frame.midX, y: frame.midY, context: context) } - // A refused XCTest tap sent nothing either. - guard case .performed = fallback.outcome else { return nil } + _ = try performGesture(app) { try tapAt(app: app, x: frame.midX, y: frame.midY, context: context) } case .refused: return nil } trace.pressed = true let tapped = currentUptimeMs() - let confirmed = navigationBarBackStateChanged(app: app, from: before) + let confirmation = navigationBarBackConfirmation( + app: app, + from: before, + deadlineMs: tapped + Self.navigationBarBackReadBudgetMs + ) trace.confirmMs = currentUptimeMs() - tapped - trace.confirmed = confirmed - return confirmed ? .performed : .unconfirmed(Self.navigationBarBackUnverified) + trace.confirmed = confirmation.logToken + switch confirmation { + case .changed: return .performed + case .unchanged: return .unconfirmed(Self.navigationBarBackUnverified) + case .hung: return .unconfirmed(Self.navigationBarBackHungConfirmation) + } } /// What the back button path saw, logged as one line on every exit. struct NavigationBarBackTrace { - /// `skipped` when the tree was not read, `unreadable` when a read failed, `no`, `ambiguous`, or `yes`. + /// `skipped` when the tree was not read, `unreadable` when a read failed, `hung` when an XCTest + /// query hung, `moving` when the bar did not hold still within the read budget, `no`, + /// `ambiguous`, or `yes`. var found = "skipped" var pressed = false - var confirmed: Bool? + /// `yes`, `no`, or `hung`; nil until a tap was delivered. + var confirmed: String? var lookupMs = 0.0 var confirmMs: Double? @@ -241,24 +264,29 @@ extension RunnerTests { return [ "found=\(found)", "pressed=\(pressed ? "yes" : "no")", - "confirmed=\(confirmed.map { $0 ? "yes" : "no" } ?? "-")", + "confirmed=\(confirmed ?? "-")", "lookupMs=\(ms(lookupMs))", "confirmMs=\(ms(confirmMs))", ].joined(separator: " ") } } - /// Each navigation bar's identifier (the top screen's title in UIKit and SwiftUI stacks) with the - /// label and frame of its system back buttons. A pop removes, moves, or relabels the pressed button - /// or retitles its bar, so a change of this state is the evidence that the tap navigated. + /// Each navigation bar's identifier, the labels of its static texts (its title), and the label and + /// frame of its system back buttons, in bar order, so the bar count counts too. A pop removes, + /// moves, or relabels the pressed button, or retitles its bar, so a change of this state is the + /// evidence that the tap navigated. A pop that changes none of these, such as one between two + /// screens with the same title in a bar with a fixed identifier and an unlabeled back button, + /// answers unverified. struct NavigationBarBackState: Equatable { let entries: [String] + let backButtonFrames: [CGRect] } - /// The pause between two reads of the bar after the tap, and how long they may run before the - /// outcome is reported unconfirmed. Only a tap that did not navigate spends the whole budget. - static let navigationBarBackConfirmInterval: TimeInterval = 0.05 - static let navigationBarBackConfirmBudgetMs = 1000.0 + /// The pause between two reads of the bar, and how long reads may start, both before the tap + /// while the bar settles and after it while the pop is confirmed. Only a bar that never holds + /// still, or a tap that did not navigate, spends the whole budget. + static let navigationBarBackReadInterval: TimeInterval = 0.05 + static let navigationBarBackReadBudgetMs = 1000.0 static let navigationBarBackUnverified = ErrorPayload( code: RunnerWireErrorCode.backSwipeUnverified, @@ -266,16 +294,96 @@ extension RunnerTests { hint: "Take a snapshot to see where the app is." ) - /// Whether a later read of the bar differs from `before` within the confirmation budget. A read - /// that fails proves nothing, so it never confirms. + static let navigationBarBackHungConfirmation = ErrorPayload( + code: RunnerWireErrorCode.backSwipeUnverified, + message: "back pressed the navigation bar back button, but an XCTest query on the navigation bar hung afterwards. This is an unknown outcome.", + hint: "Take a snapshot to see where the app is." + ) + + static let navigationBarBackHungLookup = ErrorPayload( + code: RunnerWireErrorCode.backSwipeNotDispatched, + message: "back sent nothing: an XCTest query on the navigation bar hung before the back button or the swipe.", + hint: "Take a snapshot to check that the app responds, then retry back." + ) + + /// What the read before the tap found: a still bar with its one hittable back button, no button + /// to press (with the trace's `found` value), or a hung query. + enum NavigationBarBackLookup { + case still(NavigationBarBackState, button: CGRect) + case notFound(String) + case hung + } + + /// Reads the hittable back buttons, then the bar state until two reads in a row agree and the + /// button's frame is in them, so a push still animating is not read as the state a pop changes. + /// Each round re-reads the buttons. Reads start only until `deadlineMs`. @MainActor - private func navigationBarBackStateChanged(app: XCUIApplication, from before: NavigationBarBackState) -> Bool { - let started = currentUptimeMs() - repeat { - if let after = readNavigationBarBackState(app: app, phase: .confirmation), after != before { return true } - if currentUptimeMs() - started >= Self.navigationBarBackConfirmBudgetMs { return false } - sleepFor(Self.navigationBarBackConfirmInterval) - } while true + private func stillNavigationBarBackButton(app: XCUIApplication, deadlineMs: Double) -> NavigationBarBackLookup { + var previous: NavigationBarBackState? + while true { + let button: CGRect + switch navigationBarBackButtonFrames(app: app, deadlineMs: deadlineMs) { + case .hung: return .hung + case .unreadable: return .notFound("unreadable") + case .read(let frames) where frames.isEmpty: return .notFound("no") + case .read(let frames) where frames.count > 1: return .notFound("ambiguous") + case .read(let frames): button = frames[0] + } + if previous == nil { + switch readNavigationBarBackState(app: app, phase: .lookup, deadlineMs: deadlineMs) { + case .hung: return .hung + case .unreadable: return .notFound("unreadable") + case .read(let state): previous = state + } + } + let current: NavigationBarBackState + switch readNavigationBarBackState(app: app, phase: .lookup, deadlineMs: deadlineMs) { + case .hung: return .hung + case .unreadable: return .notFound("unreadable") + case .read(let state): current = state + } + if current == previous, current.backButtonFrames.contains(button) { + return .still(current, button: button) + } + if currentUptimeMs() >= deadlineMs { return .notFound("moving") } + previous = current + sleepFor(Self.navigationBarBackReadInterval) + } + } + + enum NavigationBarBackConfirmation { + case changed + case unchanged + case hung + + var logToken: String { + switch self { + case .changed: return "yes" + case .unchanged: return "no" + case .hung: return "hung" + } + } + } + + /// Whether a later read of the bar differs from `before`. Reads start only until `deadlineMs`; a + /// read in flight then finishes, or ends at XCTest's own query timeout, which is a hung query. A + /// read that fails proves nothing, so it never confirms. + @MainActor + private func navigationBarBackConfirmation( + app: XCUIApplication, + from before: NavigationBarBackState, + deadlineMs: Double + ) -> NavigationBarBackConfirmation { + while true { + switch readNavigationBarBackState(app: app, phase: .confirmation, deadlineMs: deadlineMs) { + case .read(let after) where after != before: return .changed + case .hung: return .hung + case .read, .unreadable: break + } + let remainingMs = deadlineMs - currentUptimeMs() + if remainingMs <= 0 { return .unchanged } + sleepFor(min(Self.navigationBarBackReadInterval, remainingMs / 1000)) + } } @MainActor @@ -291,28 +399,40 @@ extension RunnerTests { case confirmation } - /// Runs one read of the bar with its XCTest issues held. A button gone mid-transition or a hung - /// query records an issue, and an unheld issue would turn the command into a recorded failure. - /// Nil when the read recorded an issue or raised an exception. + /// One read of the bar: its value, a read that proved nothing, or an XCTest query that hung. + enum NavigationBarBackRead { + case read(Value) + case unreadable + case hung + } + + /// Runs one read of the bar with its XCTest issues held. A button gone mid-transition records an + /// issue that the read explains: the read proves nothing. A hung query is recorded again, as + /// `record(_:)` requires, and no further query is sent into it. @MainActor - private func heldNavigationBarRead(_ phase: NavigationBarBackReadPhase, _ read: () -> T?) -> T? { + private func heldNavigationBarRead(_ phase: NavigationBarBackReadPhase, _ read: () -> T?) -> NavigationBarBackRead { var value: T? let held = holdingRecordedIssues { value = read() #if AGENT_DEVICE_RUNNER_UNIT_TESTS - if navigationBarBackReadIssueForTesting == phase { - record(XCTIssue(type: .assertionFailure, compactDescription: "Back button vanished during the navigation bar read")) + if let injected = navigationBarBackReadIssueForTesting, injected.phase == phase { + record(XCTIssue(type: .assertionFailure, compactDescription: injected.description)) } #endif } - guard held.issues.isEmpty, held.exceptionMessage == nil else { return nil } - return value + let hung = held.issues.filter { Self.isHungQueryIssueDescription($0.compactDescription) } + hung.forEach { record($0) } + if !hung.isEmpty { return .hung } + guard held.issues.isEmpty, held.exceptionMessage == nil, let value else { return .unreadable } + return .read(value) } - /// The frames of the hittable system back buttons; nil when the tree did not answer. + /// The frames of the hittable system back buttons; unreadable when the tree did not answer or the + /// read budget had run out. @MainActor - private func navigationBarBackButtonFrames(app: XCUIApplication) -> [CGRect]? { - heldNavigationBarRead(.lookup) { + private func navigationBarBackButtonFrames(app: XCUIApplication, deadlineMs: Double) -> NavigationBarBackRead<[CGRect]> { + guard currentUptimeMs() < deadlineMs else { return .unreadable } + return heldNavigationBarRead(.lookup) { guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } let frames = bars .flatMap { $0.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex } @@ -322,34 +442,43 @@ extension RunnerTests { } } - /// One XCTest snapshot per bar, so a read after the tap costs a query per bar rather than one per - /// attribute. Nil when the tree did not answer. + /// One XCTest snapshot per bar, so a read costs a query per bar rather than one per attribute. + /// Unreadable when the tree did not answer or the read budget ran out before a bar's snapshot. @MainActor private func readNavigationBarBackState( app: XCUIApplication, - phase: NavigationBarBackReadPhase - ) -> NavigationBarBackState? { + phase: NavigationBarBackReadPhase, + deadlineMs: Double + ) -> NavigationBarBackRead { heldNavigationBarRead(phase) { guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } var entries: [String] = [] + var backButtonFrames: [CGRect] = [] for bar in bars.flatMap({ $0.allElementsBoundByIndex }) { - guard let snapshot = try? bar.snapshot() else { return nil } - entries.append(Self.navigationBarBackEntry(snapshot)) + guard currentUptimeMs() < deadlineMs, let snapshot = try? bar.snapshot() else { return nil } + let entry = Self.navigationBarBackEntry(snapshot) + entries.append(entry.entry) + backButtonFrames.append(contentsOf: entry.backButtonFrames) } - return NavigationBarBackState(entries: entries) + return NavigationBarBackState(entries: entries, backButtonFrames: backButtonFrames) } } - static func navigationBarBackEntry(_ bar: XCUIElementSnapshot) -> String { + static func navigationBarBackEntry(_ bar: XCUIElementSnapshot) -> (entry: String, backButtonFrames: [CGRect]) { + var texts: [String] = [] var buttons: [String] = [] + var frames: [CGRect] = [] var pending = bar.children while let element = pending.popLast() { if element.identifier == navigationBarBackButtonIdentifier { buttons.append("\(element.label)\u{1F}\(NSCoder.string(for: element.frame))") + frames.append(element.frame) + } else if element.elementType == .staticText { + texts.append(element.label) } pending.append(contentsOf: element.children) } - return ([bar.identifier] + buttons).joined(separator: "\u{1E}") + return (([bar.identifier] + texts + buttons).joined(separator: "\u{1E}"), frames) } @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 552ea84fcc..141b35cd3d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -219,9 +219,9 @@ final class RunnerTests: XCTestCase { var litPanelWindowWalkIssueForTesting = false #if os(iOS) - /// Records an XCTest issue during every navigation bar read of this phase of iOS `back`, as a back - /// button that vanishes mid-transition does. - var navigationBarBackReadIssueForTesting: NavigationBarBackReadPhase? + /// Records an XCTest issue with this description during every navigation bar read of this phase + /// of iOS `back`, as a back button that vanishes mid-transition, or a hung query, does. + var navigationBarBackReadIssueForTesting: (phase: NavigationBarBackReadPhase, description: String)? /// What the last iOS `back` saw of the navigation bar back button, so a test can tell the button /// path from the swipe when both navigate. @@ -325,6 +325,14 @@ final class RunnerTests: XCTestCase { "Failed to resolve query" ] + /// XCTest's wording for a query that hung. It must keep reaching `record(_:)`: a read that holds + /// its issues records this one again. + static let hungQueryIssueWording = "Timed out while evaluating UI query" + + static func isHungQueryIssueDescription(_ description: String) -> Bool { + description.contains(hungQueryIssueWording) + } + /// True for the one recorded-issue class the runner deliberately mutes: an AX-server error /// (`kAXError*`, deliberately covering kAXErrorIllegalArgument and its sibling codes) inside one /// of the fetch wordings above. Variants without that token MUST keep recording: the timeout diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 0d29c4ad08..d300cde64d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -91,31 +91,103 @@ extension RunnerTests { /// An XCTest issue while the back button is looked up, such as a button gone mid-transition, means /// no button: back answers from the swipe path. An issue that escaped the read would fail this - /// test as it fails the command with a recorded failure. + /// test as it fails the command with a recorded failure. The swipe's own outcome is not asserted: + /// iOS 27 pops this fixture on the edge swipe even with its interactive pop turned off. @MainActor func testInAppBackWithAnIssueInTheButtonLookupAnswersFromTheSwipe() throws { _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - navigationBarBackReadIssueForTesting = .lookup + navigationBarBackReadIssueForTesting = (.lookup, Self.vanishedBackButtonIssue) defer { navigationBarBackReadIssueForTesting = nil } let response = try execute(command: try runnerCommandFixture( #"{"command":"backInApp","commandId":"back-lookup-issue","appBundleId":"com.callstack.agentdevice.runner"}"# )) - XCTAssertTrue(response.ok, "the swipe answered: \(String(describing: response.error))") + XCTAssertNotEqual(response.error?.code, "XCTEST_RECORDED_FAILURE") let trace = try XCTUnwrap(lastNavigationBarBackTraceForTesting) XCTAssertEqual(trace.found, "unreadable") XCTAssertFalse(trace.pressed, "back answered from the swipe path") } + /// A hung query before the tap stays visible to XCTest, as `record(_:)` requires, and back sends + /// nothing: neither the button tap nor the swipe follows a hung query. + @MainActor + func testInAppBackWithAHungQueryInTheButtonLookupSendsNothing() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + defer { navigationBarBackReadIssueForTesting = nil } + + let outcome = try expectingHungQueryRecorded { + navigationBarBackReadIssueForTesting = (.lookup, Self.hungQueryIssue) + return try performInAppBack(app: app) + } + + guard case .refused(let payload) = outcome else { return XCTFail("a hung lookup must send nothing") } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNotDispatched) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.found, "hung") + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, false) + navigationBarBackReadIssueForTesting = nil + XCTAssertTrue(app.navigationBars["Detail"].exists, "nothing popped the detail screen") + } + + /// A hung query after a delivered tap stays visible to XCTest, and back answers unverified at once + /// instead of querying the hung tree again. + @MainActor + func testInAppBackWithAHungQueryInTheConfirmationAnswersUnverified() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + defer { navigationBarBackReadIssueForTesting = nil } + + let outcome = try expectingHungQueryRecorded { + navigationBarBackReadIssueForTesting = (.confirmation, Self.hungQueryIssue) + return try performInAppBack(app: app) + } + + guard case .unconfirmed(let payload) = outcome else { return XCTFail("a hung confirmation must not read as performed") } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.confirmed, "hung") + } + + /// Two hittable system back buttons, one per stack, are ambiguous: back presses neither and + /// answers from the swipe path. + @MainActor + func testInAppBackPressesNoneOfTwoNavigationBarBackButtons() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--two-stacks"]) + defer { endBackFixture() } + XCTAssertEqual(app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).count, 2) + + _ = try performInAppBack(app: app) + + let trace = try XCTUnwrap(lastNavigationBarBackTraceForTesting) + XCTAssertEqual(trace.found, "ambiguous") + XCTAssertFalse(trace.pressed, "back answered from the swipe path") + } + + /// A penalized XCTest snapshot channel is not read: back skips the button lookup and swipes. + @MainActor + func testInAppBackSkipsTheButtonLookupOnAPenalizedChannel() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + penalizeSnapshotXCTestChannel(bundleId: mainOwned.bundleId, reason: "navigation-test") + defer { clearSnapshotXCTestChannelPenalty(reason: "navigation-test") } + + _ = try performInAppBack(app: app) + + let trace = try XCTUnwrap(lastNavigationBarBackTraceForTesting) + XCTAssertEqual(trace.found, "skipped") + XCTAssertFalse(trace.pressed, "back answered from the swipe path") + XCTAssertEqual(trace.lookupMs, 0) + } + /// An XCTest issue in every read after a delivered tap proves nothing: back answers unverified and /// never swipes after the tap. An escaped issue would fail this test. @MainActor func testInAppBackWithAnIssueInTheConfirmationAnswersUnverified() throws { let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - navigationBarBackReadIssueForTesting = .confirmation + navigationBarBackReadIssueForTesting = (.confirmation, Self.vanishedBackButtonIssue) defer { navigationBarBackReadIssueForTesting = nil } let response = try execute(command: try runnerCommandFixture( @@ -351,6 +423,27 @@ extension RunnerTests { XCTAssertTrue(status.waitForFixtureExistence(timeout: appExistenceTimeout)) return status } + static let vanishedBackButtonIssue = "Back button vanished during the navigation bar read" + static let hungQueryIssue = "Timed out while evaluating UI query." + + /// Runs `body` and requires the hung query it records to reach XCTest exactly once. + @MainActor + private func expectingHungQueryRecorded(_ body: () throws -> T) throws -> T { + let options = XCTExpectedFailure.Options() + var observed = 0 + options.issueMatcher = { issue in + guard issue.compactDescription == Self.hungQueryIssue else { return false } + observed += 1 + return true + } + var result: Result? + XCTExpectFailure("The hung query is the condition under test", options: options) { + result = Result { try body() } + } + XCTAssertEqual(observed, 1, "a hung query reaches XCTest once, and no query follows it") + return try XCTUnwrap(result).get() + } + @MainActor private func launchBackNavigationFixture(_ arguments: [String]) throws -> XCUIElement { app.launchArguments = arguments diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 75d82d3e07..9346a6cc9d 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -137,11 +137,19 @@ than silently applied: `backInApp` is the only back command. On iOS the runner first reads the current tree, in the same call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a navigation bar. It taps that button with a synthesized tap only when exactly one live, hittable -candidate is found in a readable tree. No other button counts, whatever its label. After the tap it -reads every navigation bar's identifier and the label and frame of its `BackButton` children, one -XCTest snapshot per bar, with a 0.05 s pause between reads for up to 1 s. A change from the read -taken before the tap answers `ok`. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; -a delivered tap is never followed by the swipe. +candidate is found in a readable tree, and two reads in a row of the bar state agree and contain +that button's frame; each round re-reads the buttons. No other button counts, whatever its label. +The bar state is every navigation bar's identifier, the labels of its static texts, and the label +and frame of its `BackButton` children, in bar order, one XCTest snapshot per bar. After the tap it +reads that state again with a 0.05 s pause between reads. A change from the state read before the +tap answers `ok`. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is +never followed by the swipe. Before and after the tap, reads start for up to 1 s each, checked +before each bar's snapshot; a query in flight finishes or ends at XCTest's own query timeout. + +Every bar read holds its XCTest issues. A held issue means the read proved nothing. A hung query +("Timed out while evaluating UI query") is recorded again, and no further query follows it: before +the tap the runner answers `BACK_SWIPE_NOT_DISPATCHED` with nothing sent, and after it +`BACK_SWIPE_UNVERIFIED`. The runner swipes instead when no tap was sent: @@ -149,7 +157,8 @@ The runner swipes instead when no tap was sent: unavailable, or the XCTest snapshot channel is penalized for the app. - The read fails or XCTest rejects one of its queries. - No candidate is found, or two or more are (such as a sheet over a stack). -- The synthesized tap and its XCTest fallback are both refused before anything is sent. +- The bar state does not hold still within 1 s. +- The synthesized tap is refused before anything is sent. A tap whose outcome is unknown after it may have been sent throws, and neither the bar check nor the swipe follows. The swipe verifies a change of the app's screen. When XCTest cannot resolve the @@ -159,9 +168,9 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure | Code | When | Sent | | --- | --- | --- | | `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, `APP_SCREEN_OFF_LIT_PANEL` | The screen could not be sampled before the swipe. | Nothing | -| `BACK_SWIPE_NOT_DISPATCHED` | The swipe was refused before any touch went down, or the platform has no back gesture (visionOS; the daemon refuses it first). | Nothing | +| `BACK_SWIPE_NOT_DISPATCHED` | The swipe was refused before any touch went down, an XCTest query on the navigation bar hung before the tap, or the platform has no back command (visionOS; the daemon refuses it first). | Nothing | | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | -| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar did not change within 1 s; the outcome is unknown. | Possibly the swipe, or the tap | +| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar did not change within 1 s or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a 0.05 s pause between captures plus each capture's own time, that agree for as long as the gap the swipe will be credited across: a contact @@ -190,7 +199,8 @@ window can still be misread as a back navigation; any finite window has that lim Each iOS back whose window context resolved logs one `AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON` line on every exit of the button path, a thrown tap included, with `found` (`skipped`, -`unreadable`, `no`, `ambiguous`, or `yes`), `pressed`, `confirmed`, `lookupMs` (`0` when the read was +`unreadable`, `hung`, `moving`, `no`, `ambiguous`, or `yes`), `pressed`, `confirmed` (`yes`, `no`, +`hung`, or `-` before a tap), `lookupMs` (`0` when the read was skipped), and `confirmMs`. Each swipe logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its `outcome` (`performed`, `refused`, `unconfirmed`, or `error` for a thrown failure), `code`, `changed`, `stableMs`, `justBeforeToAfterMs`, `gestureMs`, `maxCaptureMs`, `settleSamples`, and `settled`. diff --git a/packages/platform-apple/src/navigation/runtime.ts b/packages/platform-apple/src/navigation/runtime.ts index c47f724c8a..08740751e7 100644 --- a/packages/platform-apple/src/navigation/runtime.ts +++ b/packages/platform-apple/src/navigation/runtime.ts @@ -33,7 +33,7 @@ const backOsUnavailable = Object.freeze({ const backVisionOsUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', - hint: 'iOS back is a left-edge swipe; visionOS has none. Press the visible back control by its ref.', + hint: 'visionOS has no back command. Press the visible back control by its ref.', } as const); /** tvOS presses the remote's Menu button, macOS taps the window back control, and iOS presses the * navigation bar back button or swipes from the left edge. watchOS and visionOS have no back. */ diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 49f720b1d1..146ab7a220 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -528,7 +528,7 @@ function expectNavigationAndKeyboardFacts( if (device.appleOs === 'visionos') { expect(binding.facts.operations.back).toMatchObject({ reason: 'unsupported-platform-leaf', - hint: expect.stringContaining('visionOS has none'), + hint: expect.stringContaining('visionOS has no back command'), }); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 55f6ad2040..16765973c4 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,12 +124,12 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found and the app's accessibility tree is readable: the tree is not read when XCTest could not query it earlier in the session. After the tap, `back` reads the navigation bars again, about every 0.05 s for up to 1 s, and succeeds once the bar's title or its back button changes; if neither changes, it reports `back_swipe_unverified` and does not swipe. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads in a row of the navigation bars agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once the number of bars, a bar's identifier or title text, or its back button changes; if none of these changes, it reports `back_swipe_unverified` and does not swipe. A pop that changes none of them, for example between two screens with the same title in a bar with a fixed accessibility identifier and a back button without a title, also reports `back_swipe_unverified`. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. -- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. -- `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, so nothing reached the device. +- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. +- `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. - If the app's screen cannot be captured before the swipe, `back` does not swipe: it refuses with `COMMAND_FAILED` and `details.runnerErrorCode` set to `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, or `APP_SCREEN_OFF_LIT_PANEL`. Nothing is sent. -- A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. +- A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back command and refuses before dispatch with `UNSUPPORTED_OPERATION`. - An app with a left-edge drawer may open the drawer instead of going back. When a custom back or close control is visible, press it by its ref. - `back --system` sends system back input: the Android back keyevent, the same iOS back as the default, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. - `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. In both default and scoped simulator sets, an iPhone Duo classified with more than one integrated panel refuses `portrait-upside-down` and rotates through the fold helper. It succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. Unresolved guest reads are retried inside the settle loop; an unresolved final read reports `orientation-panel-table-unavailable`. The picker offsets were measured on iPhone Duo; a new multi-panel model needs its own mapping. A simulator classified with one panel, or whose first classification read is unresolved, uses the standard runner rotation. From e946cb3b11f339f0302ee965be38d2b2a9fa1975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 18:25:15 +0100 Subject: [PATCH 06/14] fix(ios): match the back button in screen space on a placed iPad window A live XCUIElement frame is in screen points, but a snapshot frame of a placed iPad window starts at the window's own top-left (#3899). The pre-tap still check compared the two directly, so on a placed window the button was never found still: back spent the 1 s read budget and swiped. The check now places the snapshot frames at the window's screen origin through the synthesized coordinate context's SurfaceGeometry. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Navigation.swift | 27 ++++++++++++++++--- .../RunnerTests+NavigationTests.swift | 25 +++++++++++++++++ apple/runner/RUNNER_PROTOCOL.md | 4 ++- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index be0f196baa..3087d22656 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -204,7 +204,11 @@ extension RunnerTests { } guard navigationBarBackButtonIsReadable(context: context) else { return nil } let started = currentUptimeMs() - let lookup = stillNavigationBarBackButton(app: app, deadlineMs: started + Self.navigationBarBackReadBudgetMs) + let lookup = stillNavigationBarBackButton( + app: app, + geometry: context.geometry, + deadlineMs: started + Self.navigationBarBackReadBudgetMs + ) trace.lookupMs = currentUptimeMs() - started let before: NavigationBarBackState let frame: CGRect @@ -318,7 +322,11 @@ extension RunnerTests { /// button's frame is in them, so a push still animating is not read as the state a pop changes. /// Each round re-reads the buttons. Reads start only until `deadlineMs`. @MainActor - private func stillNavigationBarBackButton(app: XCUIApplication, deadlineMs: Double) -> NavigationBarBackLookup { + private func stillNavigationBarBackButton( + app: XCUIApplication, + geometry: SurfaceGeometry, + deadlineMs: Double + ) -> NavigationBarBackLookup { var previous: NavigationBarBackState? while true { let button: CGRect @@ -342,7 +350,7 @@ extension RunnerTests { case .unreadable: return .notFound("unreadable") case .read(let state): current = state } - if current == previous, current.backButtonFrames.contains(button) { + if current == previous, Self.navigationBarBackState(current, holdsButton: button, geometry: geometry) { return .still(current, button: button) } if currentUptimeMs() >= deadlineMs { return .notFound("moving") } @@ -351,6 +359,19 @@ extension RunnerTests { } } + /// Whether `state` holds the live button. A live frame is in screen points, and a snapshot frame + /// is in the window's own space, which a placed iPad window starts off the screen origin (#3899). + static func navigationBarBackState( + _ state: NavigationBarBackState, + holdsButton button: CGRect, + geometry: SurfaceGeometry + ) -> Bool { + state.backButtonFrames.contains { frame in + let origin = geometry.screenPoint(from: SurfacePoint(frame.origin)).point + return CGRect(origin: origin, size: frame.size) == button + } + } + enum NavigationBarBackConfirmation { case changed case unchanged diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index d300cde64d..cd6e98467a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -1,3 +1,4 @@ +import AgentDeviceSnapshotPresentation import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) import ObjectiveC.runtime @@ -628,6 +629,30 @@ extension RunnerTests { XCTAssertGreaterThanOrEqual(slow, Self.backSwipeDurationMs + Self.backSwipeVerificationDelay * 1000 + 600) } + /// #3899: a placed iPad window's snapshot frames start at the window's own origin, while the live + /// button frame is in screen points. The still bar holds the live button only through the window's + /// screen origin. + func testNavigationBarBackStateHoldsTheLiveButtonOfAPlacedWindow() { + let snapshotButton = CGRect(x: 8, y: 54, width: 44, height: 44) + let state = NavigationBarBackState(entries: ["Detail"], backButtonFrames: [snapshotButton]) + let placed = SurfaceGeometry( + windowOnScreen: SurfaceRect(CGRect(x: 224, y: 128, width: 386, height: 954)), + panelNativeSize: nil, + orientation: RunnerInterfaceOrientation.portrait + ) + let liveButton = CGRect(x: 232, y: 182, width: 44, height: 44) + + XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: liveButton, geometry: placed)) + XCTAssertFalse(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: placed)) + + let fullScreen = SurfaceGeometry( + windowOnScreen: SurfaceRect(CGRect(x: 0, y: 0, width: 402, height: 874)), + panelNativeSize: nil, + orientation: RunnerInterfaceOrientation.portrait + ) + XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: fullScreen)) + } + /// A still screen settles before the budget runs out at any capture time: the settle loop's /// timeline, replayed with fixed captures, reaches the required still span first. On a simulator /// the base budget holds; a 1 s capture extends it. diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 9346a6cc9d..8bd6913e77 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -138,7 +138,9 @@ than silently applied: call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a navigation bar. It taps that button with a synthesized tap only when exactly one live, hittable candidate is found in a readable tree, and two reads in a row of the bar state agree and contain -that button's frame; each round re-reads the buttons. No other button counts, whatever its label. +that button's frame; each round re-reads the buttons. The match places the snapshot frames at the +window's screen origin, because a placed iPad window reports them from its own top-left. No other +button counts, whatever its label. The bar state is every navigation bar's identifier, the labels of its static texts, and the label and frame of its `BackButton` children, in bar order, one XCTest snapshot per bar. After the tap it reads that state again with a 0.05 s pause between reads. A change from the state read before the From 3728f47e7d585b1f27ec5dc59254b19c45d86176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 18:30:52 +0100 Subject: [PATCH 07/14] test(ios): keep the placed-window back-button test iOS-only The macOS runner test build has no navigationBarBackState. Co-Authored-By: Claude Opus 5.5 --- .../UnitTests/RunnerTests+NavigationTests.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index cd6e98467a..5f75943c95 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -629,6 +629,7 @@ extension RunnerTests { XCTAssertGreaterThanOrEqual(slow, Self.backSwipeDurationMs + Self.backSwipeVerificationDelay * 1000 + 600) } +#if os(iOS) /// #3899: a placed iPad window's snapshot frames start at the window's own origin, while the live /// button frame is in screen points. The still bar holds the live button only through the window's /// screen origin. @@ -652,6 +653,7 @@ extension RunnerTests { ) XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: fullScreen)) } +#endif /// A still screen settles before the budget runs out at any capture time: the settle loop's /// timeline, replayed with fixed captures, reaches the required still span first. On a simulator From d1ffd66aecd5f2ba76187c3415c9622233de1aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 19:13:53 +0100 Subject: [PATCH 08/14] fix(ios): confirm back only from the back button and space the still reads The still check before the tap now needs two agreeing bar reads started at least one read interval apart, so two reads drawn in one animation frame no longer pass for a still bar. The bar state is now each bar's back button label and frame, so a bar title that updates itself, such as a countdown, can no longer confirm a tap that navigated nowhere. The placed-window button match allows 0.5 pt per edge for fractional frame values. Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerAppFixtures.m | 15 ++++- .../RunnerTests+Navigation.swift | 67 ++++++++++++------- .../RunnerTests+NavigationTests.swift | 51 ++++++++++++++ apple/runner/RUNNER_PROTOCOL.md | 20 +++--- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 119 insertions(+), 38 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 7870118a59..a8f8ccfe11 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -42,10 +42,13 @@ @interface AgentDeviceRunnerFixtureViewController : AgentDeviceRunnerViewControl // A real navigation stack for `back`. `--agent-device-back-navigation-bar` pushes a detail screen, // so the bar shows UIKit's system back button; the interactive pop is off, so only that button can // pop it. `--blocked-back` keeps that button but replaces its action, so pressing it navigates -// nowhere. `--custom-back-item` instead puts a bar button titled "Back" on the root, which is not the -// system back button and must never be pressed. The root's label reports what happened. +// nowhere. `--ticking-title` counts the detail screen's title down every 0.2 s, as a timer does, +// so its bar changes without any navigation. `--custom-back-item` instead puts a bar button titled +// "Back" on the root, which is not the system back button and must never be pressed. The root's +// label reports what happened. @interface AgentDeviceBackNavigationFixture : UINavigationController @property(nonatomic, strong) UILabel *status; +@property(nonatomic, strong) NSTimer *titleTimer; @end @implementation AgentDeviceBackNavigationFixture @@ -79,6 +82,14 @@ - (instancetype)initWithCustomBackItem:(BOOL)customBackItem blockedBack:(BOOL)bl detail.navigationItem.backAction = [UIAction actionWithHandler:^(__kindof UIAction *action) {}]; } } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--ticking-title"]) { + __block NSInteger remaining = 59; + __weak UIViewController *weakDetail = detail; + self.titleTimer = [NSTimer scheduledTimerWithTimeInterval:0.2 repeats:YES block:^(NSTimer *timer) { + remaining = remaining > 0 ? remaining - 1 : 59; + weakDetail.title = [NSString stringWithFormat:@"0:%02ld", (long)remaining]; + }]; + } [self pushViewController:detail animated:NO]; } return self; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 3087d22656..eb71e69df9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -275,17 +275,23 @@ extension RunnerTests { } } - /// Each navigation bar's identifier, the labels of its static texts (its title), and the label and - /// frame of its system back buttons, in bar order, so the bar count counts too. A pop removes, - /// moves, or relabels the pressed button, or retitles its bar, so a change of this state is the - /// evidence that the tap navigated. A pop that changes none of these, such as one between two - /// screens with the same title in a bar with a fixed identifier and an unlabeled back button, - /// answers unverified. + /// The label and frame of each navigation bar's system back buttons, one entry per bar in bar + /// order, so the bar count counts too. A pop removes, moves, or relabels the pressed button, or + /// removes its bar, so a change of this state is the evidence that the tap navigated. Bar titles + /// and identifiers are left out: a bar can retitle itself, such as a countdown, without any + /// navigation. A pop that leaves the back button as it was, such as one between two screens whose + /// back buttons have the same title, answers unverified. struct NavigationBarBackState: Equatable { let entries: [String] let backButtonFrames: [CGRect] } + /// One read of the bar state, with the time the read started. + struct TimedNavigationBarBackState { + let state: NavigationBarBackState + let startedMs: Double + } + /// The pause between two reads of the bar, and how long reads may start, both before the tap /// while the bar settles and after it while the pop is confirmed. Only a bar that never holds /// still, or a tap that did not navigate, spends the whole budget. @@ -318,16 +324,16 @@ extension RunnerTests { case hung } - /// Reads the hittable back buttons, then the bar state until two reads in a row agree and the - /// button's frame is in them, so a push still animating is not read as the state a pop changes. - /// Each round re-reads the buttons. Reads start only until `deadlineMs`. + /// Reads the hittable back buttons, then the bar state until two reads in a row, at least one read + /// interval apart, agree and the button's frame is in them, so a push still animating is not read + /// as the state a pop changes. Each round re-reads the buttons. Reads start only until `deadlineMs`. @MainActor private func stillNavigationBarBackButton( app: XCUIApplication, geometry: SurfaceGeometry, deadlineMs: Double ) -> NavigationBarBackLookup { - var previous: NavigationBarBackState? + var previous: TimedNavigationBarBackState? while true { let button: CGRect switch navigationBarBackButtonFrames(app: app, deadlineMs: deadlineMs) { @@ -337,21 +343,17 @@ extension RunnerTests { case .read(let frames) where frames.count > 1: return .notFound("ambiguous") case .read(let frames): button = frames[0] } - if previous == nil { - switch readNavigationBarBackState(app: app, phase: .lookup, deadlineMs: deadlineMs) { - case .hung: return .hung - case .unreadable: return .notFound("unreadable") - case .read(let state): previous = state - } - } - let current: NavigationBarBackState + let startedMs = currentUptimeMs() + let current: TimedNavigationBarBackState switch readNavigationBarBackState(app: app, phase: .lookup, deadlineMs: deadlineMs) { case .hung: return .hung case .unreadable: return .notFound("unreadable") - case .read(let state): current = state + case .read(let state): current = TimedNavigationBarBackState(state: state, startedMs: startedMs) } - if current == previous, Self.navigationBarBackState(current, holdsButton: button, geometry: geometry) { - return .still(current, button: button) + if let previous, + Self.navigationBarBackStateHeldStill(from: previous, to: current), + Self.navigationBarBackState(current.state, holdsButton: button, geometry: geometry) { + return .still(current.state, button: button) } if currentUptimeMs() >= deadlineMs { return .notFound("moving") } previous = current @@ -359,6 +361,19 @@ extension RunnerTests { } } + /// Whether two reads show a still bar: they agree, and they started at least one read interval + /// apart, so two reads drawn in one animation frame never pass for a still bar. + static func navigationBarBackStateHeldStill( + from earlier: TimedNavigationBarBackState, + to later: TimedNavigationBarBackState + ) -> Bool { + later.state == earlier.state && later.startedMs - earlier.startedMs >= navigationBarBackReadInterval * 1000 + } + + /// How far, in points, each edge of a snapshot frame placed in screen space may sit from the live + /// frame, so fractional frame values that XCTest rounds differently still match. + static let navigationBarBackFrameTolerance: CGFloat = 0.5 + /// Whether `state` holds the live button. A live frame is in screen points, and a snapshot frame /// is in the window's own space, which a placed iPad window starts off the screen origin (#3899). static func navigationBarBackState( @@ -368,7 +383,10 @@ extension RunnerTests { ) -> Bool { state.backButtonFrames.contains { frame in let origin = geometry.screenPoint(from: SurfacePoint(frame.origin)).point - return CGRect(origin: origin, size: frame.size) == button + let placed = CGRect(origin: origin, size: frame.size) + return [ + (placed.minX, button.minX), (placed.minY, button.minY), (placed.maxX, button.maxX), (placed.maxY, button.maxY), + ].allSatisfy { abs($0.0 - $0.1) <= navigationBarBackFrameTolerance } } } @@ -486,7 +504,6 @@ extension RunnerTests { } static func navigationBarBackEntry(_ bar: XCUIElementSnapshot) -> (entry: String, backButtonFrames: [CGRect]) { - var texts: [String] = [] var buttons: [String] = [] var frames: [CGRect] = [] var pending = bar.children @@ -494,12 +511,10 @@ extension RunnerTests { if element.identifier == navigationBarBackButtonIdentifier { buttons.append("\(element.label)\u{1F}\(NSCoder.string(for: element.frame))") frames.append(element.frame) - } else if element.elementType == .staticText { - texts.append(element.label) } pending.append(contentsOf: element.children) } - return (([bar.identifier] + texts + buttons).joined(separator: "\u{1E}"), frames) + return (buttons.joined(separator: "\u{1E}"), frames) } @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 5f75943c95..307ad5ae3b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -90,6 +90,22 @@ extension RunnerTests { XCTAssertTrue(app.navigationBars["Detail"].exists) } + /// A bar title that counts down on its own is not a navigation: a tap that navigates nowhere on + /// such a bar reports an unknown outcome, though the title changes within the confirmation window. + @MainActor + func testInAppBackDoesNotCreditATickingBarTitleToAPressedBackButton() throws { + guard #available(iOS 16.0, *) else { throw XCTSkip("UINavigationItem.backAction needs iOS 16") } + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--blocked-back", "--ticking-title"]) + defer { endBackFixture() } + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a ticking bar title must not confirm a back button tap that navigated nowhere") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + XCTAssertEqual(app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).count, 1) + } + /// An XCTest issue while the back button is looked up, such as a button gone mid-transition, means /// no button: back answers from the swipe path. An issue that escaped the read would fail this /// test as it fails the command with a recorded failure. The swipe's own outcome is not asserted: @@ -653,6 +669,41 @@ extension RunnerTests { ) XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: fullScreen)) } + + /// A snapshot frame placed in screen space matches a live frame up to half a point per edge, so + /// fractional frame values do not defeat the match; a button a point away is another button. + func testNavigationBarBackStateHoldsALiveButtonWithinHalfAPointPerEdge() { + let state = NavigationBarBackState( + entries: ["Root"], + backButtonFrames: [CGRect(x: 8.33, y: 54.67, width: 44.33, height: 44)] + ) + let placed = SurfaceGeometry( + windowOnScreen: SurfaceRect(CGRect(x: 224.5, y: 128, width: 386, height: 954)), + panelNativeSize: nil, + orientation: RunnerInterfaceOrientation.portrait + ) + + XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: CGRect(x: 233, y: 182.5, width: 44, height: 44), geometry: placed)) + XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: CGRect(x: 232.5, y: 183, width: 44.5, height: 43.5), geometry: placed)) + XCTAssertFalse(Self.navigationBarBackState(state, holdsButton: CGRect(x: 234, y: 182.5, width: 44, height: 44), geometry: placed)) + XCTAssertFalse(Self.navigationBarBackState(state, holdsButton: CGRect(x: 233, y: 182.5, width: 45, height: 44), geometry: placed)) + } + + /// Two agreeing reads of the bar show a still bar only when they started at least one read + /// interval apart: two reads drawn in one animation frame of a push agree, but prove nothing. + func testNavigationBarBackStateHeldStillOnlyAcrossOneReadInterval() { + let bar = NavigationBarBackState(entries: ["Root"], backButtonFrames: [CGRect(x: 8, y: 54, width: 44, height: 44)]) + let moved = NavigationBarBackState(entries: ["Root"], backButtonFrames: [CGRect(x: 40, y: 54, width: 44, height: 44)]) + let intervalMs = Self.navigationBarBackReadInterval * 1000 + let first = TimedNavigationBarBackState(state: bar, startedMs: 1000) + + let sameFrame = TimedNavigationBarBackState(state: bar, startedMs: 1000 + intervalMs / 2) + XCTAssertFalse(Self.navigationBarBackStateHeldStill(from: first, to: sameFrame)) + let oneIntervalLater = TimedNavigationBarBackState(state: bar, startedMs: 1000 + intervalMs) + XCTAssertTrue(Self.navigationBarBackStateHeldStill(from: first, to: oneIntervalLater)) + let movedLater = TimedNavigationBarBackState(state: moved, startedMs: 1000 + intervalMs) + XCTAssertFalse(Self.navigationBarBackStateHeldStill(from: first, to: movedLater)) + } #endif /// A still screen settles before the budget runs out at any capture time: the settle loop's diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 8bd6913e77..ca3ef05285 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -137,14 +137,18 @@ than silently applied: `backInApp` is the only back command. On iOS the runner first reads the current tree, in the same call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a navigation bar. It taps that button with a synthesized tap only when exactly one live, hittable -candidate is found in a readable tree, and two reads in a row of the bar state agree and contain -that button's frame; each round re-reads the buttons. The match places the snapshot frames at the -window's screen origin, because a placed iPad window reports them from its own top-left. No other -button counts, whatever its label. -The bar state is every navigation bar's identifier, the labels of its static texts, and the label -and frame of its `BackButton` children, in bar order, one XCTest snapshot per bar. After the tap it -reads that state again with a 0.05 s pause between reads. A change from the state read before the -tap answers `ok`. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is +candidate is found in a readable tree, and two reads in a row of the bar state, started at least +0.05 s apart, agree and contain that button's frame; each round re-reads the buttons. The match +places the snapshot frames at the window's screen origin, because a placed iPad window reports them +from its own top-left, and allows 0.5 pt per edge for fractional frame values. No other button +counts, whatever its label. +The bar state is the label and frame of each navigation bar's `BackButton` children, one entry per +bar in bar order, so the bar count counts too; one XCTest snapshot per bar. Bar titles and +identifiers are left out, because a bar can retitle itself (a countdown) without navigating. After +the tap it reads that state again with a 0.05 s pause between reads. A change from the state read +before the tap answers `ok`. A pop that leaves the back button as it was, such as one between two +screens whose back buttons have the same title, answers `BACK_SWIPE_UNVERIFIED`; so can a pop on a +back button whose title updates itself, misread as a navigation. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is never followed by the swipe. Before and after the tap, reads start for up to 1 s each, checked before each bar's snapshot; a query in flight finishes or ends at XCTest's own query timeout. diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index bc05cc7679..b8b09789fb 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the navigation bar then changes; with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; + 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the bar count or a back button then changes (a bar title alone never counts); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 16765973c4..72449c8a81 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,7 +124,7 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads in a row of the navigation bars agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once the number of bars, a bar's identifier or title text, or its back button changes; if none of these changes, it reports `back_swipe_unverified` and does not swipe. A pop that changes none of them, for example between two screens with the same title in a bar with a fixed accessibility identifier and a back button without a title, also reports `back_swipe_unverified`. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once the number of bars or a back button (its title or position) changes; if neither changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. From cee99413c39aaf63d48e4694287021a09fb540d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 19:41:09 +0100 Subject: [PATCH 09/14] fix(ios): confirm a back button pop only from bar and back button labels and counts A tap that navigates nowhere can still move the back button, such as by a large-title collapse, so its frame no longer confirms a pop. Frames stay in the still check before the tap and in matching the button to press. Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerAppFixtures.m | 9 +++- .../RunnerTests+Navigation.swift | 41 ++++++++++-------- .../RunnerTests+NavigationTests.swift | 43 +++++++++++++++++-- apple/runner/RUNNER_PROTOCOL.md | 16 ++++--- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 80 insertions(+), 33 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index a8f8ccfe11..87a32bb113 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -42,7 +42,8 @@ @interface AgentDeviceRunnerFixtureViewController : AgentDeviceRunnerViewControl // A real navigation stack for `back`. `--agent-device-back-navigation-bar` pushes a detail screen, // so the bar shows UIKit's system back button; the interactive pop is off, so only that button can // pop it. `--blocked-back` keeps that button but replaces its action, so pressing it navigates -// nowhere. `--ticking-title` counts the detail screen's title down every 0.2 s, as a timer does, +// nowhere. `--shifting-back` makes that press move the bar down instead, so the back button moves +// without any navigation. `--ticking-title` counts the detail screen's title down every 0.2 s, as a timer does, // so its bar changes without any navigation. `--custom-back-item` instead puts a bar button titled // "Back" on the root, which is not the system back button and must never be pressed. The root's // label reports what happened. @@ -79,7 +80,11 @@ - (instancetype)initWithCustomBackItem:(BOOL)customBackItem blockedBack:(BOOL)bl detail.view.backgroundColor = UIColor.systemTealColor; if (blockedBack) { if (@available(iOS 16.0, *)) { - detail.navigationItem.backAction = [UIAction actionWithHandler:^(__kindof UIAction *action) {}]; + BOOL shifting = [NSProcessInfo.processInfo.arguments containsObject:@"--shifting-back"]; + __weak UINavigationController *weakSelf = self; + detail.navigationItem.backAction = [UIAction actionWithHandler:^(__kindof UIAction *action) { + if (shifting) weakSelf.additionalSafeAreaInsets = UIEdgeInsetsMake(30, 0, 0, 0); + }]; } } if ([NSProcessInfo.processInfo.arguments containsObject:@"--ticking-title"]) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index eb71e69df9..38b058711e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -275,17 +275,22 @@ extension RunnerTests { } } - /// The label and frame of each navigation bar's system back buttons, one entry per bar in bar - /// order, so the bar count counts too. A pop removes, moves, or relabels the pressed button, or - /// removes its bar, so a change of this state is the evidence that the tap navigated. Bar titles - /// and identifiers are left out: a bar can retitle itself, such as a countdown, without any - /// navigation. A pop that leaves the back button as it was, such as one between two screens whose - /// back buttons have the same title, answers unverified. + /// The labels of each navigation bar's system back buttons, one list per bar in bar order, and + /// the frames of those buttons. Both must agree for a still bar before the tap. Bar titles and + /// identifiers are left out: a bar can retitle itself, such as a countdown, without any navigation. struct NavigationBarBackState: Equatable { - let entries: [String] + let backButtonLabels: [[String]] let backButtonFrames: [CGRect] } + /// Whether the bar after the tap shows a pop: the bar count, the back button count, or a back + /// button label changed. Frames are left out: a tap that navigates nowhere can still move the + /// button, such as by a large-title collapse or a bar button that appears. A pop that keeps the + /// same back button labels and bar count answers unverified. + static func navigationBarBackPopped(from before: NavigationBarBackState, to after: NavigationBarBackState) -> Bool { + after.backButtonLabels != before.backButtonLabels + } + /// One read of the bar state, with the time the read started. struct TimedNavigationBarBackState { let state: NavigationBarBackState @@ -415,7 +420,7 @@ extension RunnerTests { ) -> NavigationBarBackConfirmation { while true { switch readNavigationBarBackState(app: app, phase: .confirmation, deadlineMs: deadlineMs) { - case .read(let after) where after != before: return .changed + case .read(let after) where Self.navigationBarBackPopped(from: before, to: after): return .changed case .hung: return .hung case .read, .unreadable: break } @@ -491,30 +496,28 @@ extension RunnerTests { ) -> NavigationBarBackRead { heldNavigationBarRead(phase) { guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } - var entries: [String] = [] + var backButtonLabels: [[String]] = [] var backButtonFrames: [CGRect] = [] for bar in bars.flatMap({ $0.allElementsBoundByIndex }) { guard currentUptimeMs() < deadlineMs, let snapshot = try? bar.snapshot() else { return nil } - let entry = Self.navigationBarBackEntry(snapshot) - entries.append(entry.entry) - backButtonFrames.append(contentsOf: entry.backButtonFrames) + let buttons = Self.navigationBarBackButtons(snapshot) + backButtonLabels.append(buttons.map(\.label)) + backButtonFrames.append(contentsOf: buttons.map(\.frame)) } - return NavigationBarBackState(entries: entries, backButtonFrames: backButtonFrames) + return NavigationBarBackState(backButtonLabels: backButtonLabels, backButtonFrames: backButtonFrames) } } - static func navigationBarBackEntry(_ bar: XCUIElementSnapshot) -> (entry: String, backButtonFrames: [CGRect]) { - var buttons: [String] = [] - var frames: [CGRect] = [] + static func navigationBarBackButtons(_ bar: XCUIElementSnapshot) -> [(label: String, frame: CGRect)] { + var buttons: [(label: String, frame: CGRect)] = [] var pending = bar.children while let element = pending.popLast() { if element.identifier == navigationBarBackButtonIdentifier { - buttons.append("\(element.label)\u{1F}\(NSCoder.string(for: element.frame))") - frames.append(element.frame) + buttons.append((element.label, element.frame)) } pending.append(contentsOf: element.children) } - return (buttons.joined(separator: "\u{1E}"), frames) + return buttons } @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 307ad5ae3b..1e3b8b8441 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -106,6 +106,25 @@ extension RunnerTests { XCTAssertEqual(app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).count, 1) } + /// A back button that only moves after the tap is not a navigation: a tap that navigates nowhere + /// but shifts the bar, as a large-title collapse does, reports an unknown outcome. + @MainActor + func testInAppBackDoesNotCreditAMovedBackButtonToAPressedBackButton() throws { + guard #available(iOS 16.0, *) else { throw XCTSkip("UINavigationItem.backAction needs iOS 16") } + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--blocked-back", "--shifting-back"]) + defer { endBackFixture() } + let backButton = app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).firstMatch + let frameBefore = backButton.frame + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a back button that only moved must not confirm a tap that navigated nowhere") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + XCTAssertTrue(app.navigationBars["Detail"].exists) + XCTAssertNotEqual(backButton.frame.minY, frameBefore.minY, "the fixture must move the back button") + } + /// An XCTest issue while the back button is looked up, such as a button gone mid-transition, means /// no button: back answers from the swipe path. An issue that escaped the read would fail this /// test as it fails the command with a recorded failure. The swipe's own outcome is not asserted: @@ -651,7 +670,7 @@ extension RunnerTests { /// screen origin. func testNavigationBarBackStateHoldsTheLiveButtonOfAPlacedWindow() { let snapshotButton = CGRect(x: 8, y: 54, width: 44, height: 44) - let state = NavigationBarBackState(entries: ["Detail"], backButtonFrames: [snapshotButton]) + let state = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [snapshotButton]) let placed = SurfaceGeometry( windowOnScreen: SurfaceRect(CGRect(x: 224, y: 128, width: 386, height: 954)), panelNativeSize: nil, @@ -674,7 +693,7 @@ extension RunnerTests { /// fractional frame values do not defeat the match; a button a point away is another button. func testNavigationBarBackStateHoldsALiveButtonWithinHalfAPointPerEdge() { let state = NavigationBarBackState( - entries: ["Root"], + backButtonLabels: [["Root"]], backButtonFrames: [CGRect(x: 8.33, y: 54.67, width: 44.33, height: 44)] ) let placed = SurfaceGeometry( @@ -692,8 +711,8 @@ extension RunnerTests { /// Two agreeing reads of the bar show a still bar only when they started at least one read /// interval apart: two reads drawn in one animation frame of a push agree, but prove nothing. func testNavigationBarBackStateHeldStillOnlyAcrossOneReadInterval() { - let bar = NavigationBarBackState(entries: ["Root"], backButtonFrames: [CGRect(x: 8, y: 54, width: 44, height: 44)]) - let moved = NavigationBarBackState(entries: ["Root"], backButtonFrames: [CGRect(x: 40, y: 54, width: 44, height: 44)]) + let bar = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [CGRect(x: 8, y: 54, width: 44, height: 44)]) + let moved = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [CGRect(x: 40, y: 54, width: 44, height: 44)]) let intervalMs = Self.navigationBarBackReadInterval * 1000 let first = TimedNavigationBarBackState(state: bar, startedMs: 1000) @@ -704,6 +723,22 @@ extension RunnerTests { let movedLater = TimedNavigationBarBackState(state: moved, startedMs: 1000 + intervalMs) XCTAssertFalse(Self.navigationBarBackStateHeldStill(from: first, to: movedLater)) } + + /// After the tap, only a change of the bar count, the back button count, or a back button label + /// shows a pop. A back button that only moved, such as by a large-title collapse, shows none. + func testNavigationBarBackPoppedOnlyOnABarOrBackButtonChange() { + let button = CGRect(x: 8, y: 54, width: 44, height: 44) + let detail = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button]) + + let moved = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button.offsetBy(dx: 0, dy: 30)]) + XCTAssertFalse(Self.navigationBarBackPopped(from: detail, to: moved)) + let relabeled = NavigationBarBackState(backButtonLabels: [["Settings"]], backButtonFrames: [button]) + XCTAssertTrue(Self.navigationBarBackPopped(from: detail, to: relabeled)) + let atRoot = NavigationBarBackState(backButtonLabels: [[]], backButtonFrames: []) + XCTAssertTrue(Self.navigationBarBackPopped(from: detail, to: atRoot)) + let barGone = NavigationBarBackState(backButtonLabels: [], backButtonFrames: []) + XCTAssertTrue(Self.navigationBarBackPopped(from: atRoot, to: barGone)) + } #endif /// A still screen settles before the budget runs out at any capture time: the settle loop's diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index ca3ef05285..0f59bc6988 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -142,12 +142,16 @@ candidate is found in a readable tree, and two reads in a row of the bar state, places the snapshot frames at the window's screen origin, because a placed iPad window reports them from its own top-left, and allows 0.5 pt per edge for fractional frame values. No other button counts, whatever its label. -The bar state is the label and frame of each navigation bar's `BackButton` children, one entry per -bar in bar order, so the bar count counts too; one XCTest snapshot per bar. Bar titles and -identifiers are left out, because a bar can retitle itself (a countdown) without navigating. After -the tap it reads that state again with a 0.05 s pause between reads. A change from the state read -before the tap answers `ok`. A pop that leaves the back button as it was, such as one between two -screens whose back buttons have the same title, answers `BACK_SWIPE_UNVERIFIED`; so can a pop on a +The bar state is the labels of each navigation bar's `BackButton` children, one list per bar in +bar order, and the frames of those buttons; one XCTest snapshot per bar. The still check before the +tap compares labels and frames. Bar titles and identifiers are left out, because a bar can retitle +itself (a countdown) without navigating. After the tap it reads that state again with a 0.05 s +pause between reads. Only these changes from the state read before the tap answer `ok`: the number +of bars, the number of `BackButton` children (such as none at the root), or a `BackButton` label. +Frames are left out after the tap, because a tap that navigates nowhere can still move the button +(a large-title collapse, a bar button that appears). A pop that keeps the same `BackButton` labels +and bar count, such as one between two screens whose back buttons have the same title, answers +`BACK_SWIPE_UNVERIFIED`; so can a pop on a back button whose title updates itself, misread as a navigation. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is never followed by the swipe. Before and after the tap, reads start for up to 1 s each, checked before each bar's snapshot; a query in flight finishes or ends at XCTest's own query timeout. diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index b8b09789fb..c08fba53c4 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the bar count or a back button then changes (a bar title alone never counts); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; + 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 72449c8a81..ddc7a5b8a1 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,7 +124,7 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once the number of bars or a back button (its title or position) changes; if neither changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. From f443d78d2b569da8ebb2b33bb9b1caef607116e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 20:00:15 +0100 Subject: [PATCH 10/14] refactor(ios): drop the timed read and frame tolerance from the back button check Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Navigation.swift | 43 ++++--------------- .../RunnerTests+NavigationTests.swift | 39 ++--------------- apple/runner/RUNNER_PROTOCOL.md | 8 ++-- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 2 +- 5 files changed, 18 insertions(+), 76 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 38b058711e..433d665ccf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -291,12 +291,6 @@ extension RunnerTests { after.backButtonLabels != before.backButtonLabels } - /// One read of the bar state, with the time the read started. - struct TimedNavigationBarBackState { - let state: NavigationBarBackState - let startedMs: Double - } - /// The pause between two reads of the bar, and how long reads may start, both before the tap /// while the bar settles and after it while the pop is confirmed. Only a bar that never holds /// still, or a tap that did not navigate, spends the whole budget. @@ -305,7 +299,7 @@ extension RunnerTests { static let navigationBarBackUnverified = ErrorPayload( code: RunnerWireErrorCode.backSwipeUnverified, - message: "back pressed the navigation bar back button, but the navigation bar did not change within 1 s. This is an unknown outcome.", + message: "back pressed the navigation bar back button, but the navigation bar's back buttons did not change within 1 s. This is an unknown outcome.", hint: "Take a snapshot to see where the app is." ) @@ -329,8 +323,8 @@ extension RunnerTests { case hung } - /// Reads the hittable back buttons, then the bar state until two reads in a row, at least one read - /// interval apart, agree and the button's frame is in them, so a push still animating is not read + /// Reads the hittable back buttons, then the bar state until two reads in a row, one read interval + /// apart, agree and the button's frame is in them, so a push still animating is not read /// as the state a pop changes. Each round re-reads the buttons. Reads start only until `deadlineMs`. @MainActor private func stillNavigationBarBackButton( @@ -338,7 +332,7 @@ extension RunnerTests { geometry: SurfaceGeometry, deadlineMs: Double ) -> NavigationBarBackLookup { - var previous: TimedNavigationBarBackState? + var previous: NavigationBarBackState? while true { let button: CGRect switch navigationBarBackButtonFrames(app: app, deadlineMs: deadlineMs) { @@ -348,17 +342,14 @@ extension RunnerTests { case .read(let frames) where frames.count > 1: return .notFound("ambiguous") case .read(let frames): button = frames[0] } - let startedMs = currentUptimeMs() - let current: TimedNavigationBarBackState + let current: NavigationBarBackState switch readNavigationBarBackState(app: app, phase: .lookup, deadlineMs: deadlineMs) { case .hung: return .hung case .unreadable: return .notFound("unreadable") - case .read(let state): current = TimedNavigationBarBackState(state: state, startedMs: startedMs) + case .read(let state): current = state } - if let previous, - Self.navigationBarBackStateHeldStill(from: previous, to: current), - Self.navigationBarBackState(current.state, holdsButton: button, geometry: geometry) { - return .still(current.state, button: button) + if current == previous, Self.navigationBarBackState(current, holdsButton: button, geometry: geometry) { + return .still(current, button: button) } if currentUptimeMs() >= deadlineMs { return .notFound("moving") } previous = current @@ -366,19 +357,6 @@ extension RunnerTests { } } - /// Whether two reads show a still bar: they agree, and they started at least one read interval - /// apart, so two reads drawn in one animation frame never pass for a still bar. - static func navigationBarBackStateHeldStill( - from earlier: TimedNavigationBarBackState, - to later: TimedNavigationBarBackState - ) -> Bool { - later.state == earlier.state && later.startedMs - earlier.startedMs >= navigationBarBackReadInterval * 1000 - } - - /// How far, in points, each edge of a snapshot frame placed in screen space may sit from the live - /// frame, so fractional frame values that XCTest rounds differently still match. - static let navigationBarBackFrameTolerance: CGFloat = 0.5 - /// Whether `state` holds the live button. A live frame is in screen points, and a snapshot frame /// is in the window's own space, which a placed iPad window starts off the screen origin (#3899). static func navigationBarBackState( @@ -388,10 +366,7 @@ extension RunnerTests { ) -> Bool { state.backButtonFrames.contains { frame in let origin = geometry.screenPoint(from: SurfacePoint(frame.origin)).point - let placed = CGRect(origin: origin, size: frame.size) - return [ - (placed.minX, button.minX), (placed.minY, button.minY), (placed.maxX, button.maxX), (placed.maxY, button.maxY), - ].allSatisfy { abs($0.0 - $0.1) <= navigationBarBackFrameTolerance } + return CGRect(origin: origin, size: frame.size) == button } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 1e3b8b8441..3f2b353208 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -689,49 +689,16 @@ extension RunnerTests { XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: fullScreen)) } - /// A snapshot frame placed in screen space matches a live frame up to half a point per edge, so - /// fractional frame values do not defeat the match; a button a point away is another button. - func testNavigationBarBackStateHoldsALiveButtonWithinHalfAPointPerEdge() { - let state = NavigationBarBackState( - backButtonLabels: [["Root"]], - backButtonFrames: [CGRect(x: 8.33, y: 54.67, width: 44.33, height: 44)] - ) - let placed = SurfaceGeometry( - windowOnScreen: SurfaceRect(CGRect(x: 224.5, y: 128, width: 386, height: 954)), - panelNativeSize: nil, - orientation: RunnerInterfaceOrientation.portrait - ) - - XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: CGRect(x: 233, y: 182.5, width: 44, height: 44), geometry: placed)) - XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: CGRect(x: 232.5, y: 183, width: 44.5, height: 43.5), geometry: placed)) - XCTAssertFalse(Self.navigationBarBackState(state, holdsButton: CGRect(x: 234, y: 182.5, width: 44, height: 44), geometry: placed)) - XCTAssertFalse(Self.navigationBarBackState(state, holdsButton: CGRect(x: 233, y: 182.5, width: 45, height: 44), geometry: placed)) - } - - /// Two agreeing reads of the bar show a still bar only when they started at least one read - /// interval apart: two reads drawn in one animation frame of a push agree, but prove nothing. - func testNavigationBarBackStateHeldStillOnlyAcrossOneReadInterval() { - let bar = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [CGRect(x: 8, y: 54, width: 44, height: 44)]) - let moved = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [CGRect(x: 40, y: 54, width: 44, height: 44)]) - let intervalMs = Self.navigationBarBackReadInterval * 1000 - let first = TimedNavigationBarBackState(state: bar, startedMs: 1000) - - let sameFrame = TimedNavigationBarBackState(state: bar, startedMs: 1000 + intervalMs / 2) - XCTAssertFalse(Self.navigationBarBackStateHeldStill(from: first, to: sameFrame)) - let oneIntervalLater = TimedNavigationBarBackState(state: bar, startedMs: 1000 + intervalMs) - XCTAssertTrue(Self.navigationBarBackStateHeldStill(from: first, to: oneIntervalLater)) - let movedLater = TimedNavigationBarBackState(state: moved, startedMs: 1000 + intervalMs) - XCTAssertFalse(Self.navigationBarBackStateHeldStill(from: first, to: movedLater)) - } - /// After the tap, only a change of the bar count, the back button count, or a back button label - /// shows a pop. A back button that only moved, such as by a large-title collapse, shows none. + /// shows a pop. A back button that only moved, such as by a large-title collapse, shows none, but + /// it is never a still bar before the tap. func testNavigationBarBackPoppedOnlyOnABarOrBackButtonChange() { let button = CGRect(x: 8, y: 54, width: 44, height: 44) let detail = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button]) let moved = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button.offsetBy(dx: 0, dy: 30)]) XCTAssertFalse(Self.navigationBarBackPopped(from: detail, to: moved)) + XCTAssertNotEqual(moved, detail) let relabeled = NavigationBarBackState(backButtonLabels: [["Settings"]], backButtonFrames: [button]) XCTAssertTrue(Self.navigationBarBackPopped(from: detail, to: relabeled)) let atRoot = NavigationBarBackState(backButtonLabels: [[]], backButtonFrames: []) diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 0f59bc6988..c00c1e5ebb 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -137,10 +137,10 @@ than silently applied: `backInApp` is the only back command. On iOS the runner first reads the current tree, in the same call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a navigation bar. It taps that button with a synthesized tap only when exactly one live, hittable -candidate is found in a readable tree, and two reads in a row of the bar state, started at least -0.05 s apart, agree and contain that button's frame; each round re-reads the buttons. The match +candidate is found in a readable tree, and two reads in a row of the bar state, with a 0.05 s pause +between them, agree and contain that button's frame; each round re-reads the buttons. The match places the snapshot frames at the window's screen origin, because a placed iPad window reports them -from its own top-left, and allows 0.5 pt per edge for fractional frame values. No other button +from its own top-left. No other button counts, whatever its label. The bar state is the labels of each navigation bar's `BackButton` children, one list per bar in bar order, and the frames of those buttons; one XCTest snapshot per bar. The still check before the @@ -180,7 +180,7 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure | `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, `APP_SCREEN_OFF_LIT_PANEL` | The screen could not be sampled before the swipe. | Nothing | | `BACK_SWIPE_NOT_DISPATCHED` | The swipe was refused before any touch went down, an XCTest query on the navigation bar hung before the tap, or the platform has no back command (visionOS; the daemon refuses it first). | Nothing | | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | -| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar did not change within 1 s or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | +| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a 0.05 s pause between captures plus each capture's own time, that agree for as long as the gap the swipe will be credited across: a contact diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index c08fba53c4..a029af5093 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ddc7a5b8a1..c82a55839b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -126,7 +126,7 @@ Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. - On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. -- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. +- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. - If the app's screen cannot be captured before the swipe, `back` does not swipe: it refuses with `COMMAND_FAILED` and `details.runnerErrorCode` set to `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, or `APP_SCREEN_OFF_LIT_PANEL`. Nothing is sent. - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back command and refuses before dispatch with `UNSUPPORTED_OPERATION`. From 1154091411cfb78dfbdae0ab2731e518a5575acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 20:17:15 +0100 Subject: [PATCH 11/14] fix(ios): read the bar once more after the back button confirmation budget A snapshot of a navigation bar that vanished mid-pop spends about 2 s in XCTest's element-resolution retries before it raises a held issue. On a slow CI simulator that read used up the 1 s confirmation budget, so no read saw the settled screen and back answered back_swipe_unverified after a real pop. One read now always follows the budget, with no deadline, so the screen after the pop is read. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Navigation.swift | 21 +++++---- .../RunnerTests.swift | 7 +-- .../RunnerTests+NavigationTests.swift | 44 ++++++++++++++----- apple/runner/RUNNER_PROTOCOL.md | 2 + website/docs/docs/commands.md | 2 +- 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 433d665ccf..51d0571358 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -384,9 +384,11 @@ extension RunnerTests { } } - /// Whether a later read of the bar differs from `before`. Reads start only until `deadlineMs`; a - /// read in flight then finishes, or ends at XCTest's own query timeout, which is a hung query. A - /// read that fails proves nothing, so it never confirms. + /// Whether a later read of the bar differs from `before`. Reads start until `deadlineMs`, and one + /// final read starts after it and runs to the end, so the screen after the pop is always read: a + /// read of a bar that vanishes mid-pop can spend the whole budget in XCTest's retries. A read in + /// flight ends at XCTest's own query timeout, which is a hung query. A read that fails proves + /// nothing, so it never confirms. @MainActor private func navigationBarBackConfirmation( app: XCUIApplication, @@ -394,14 +396,15 @@ extension RunnerTests { deadlineMs: Double ) -> NavigationBarBackConfirmation { while true { - switch readNavigationBarBackState(app: app, phase: .confirmation, deadlineMs: deadlineMs) { + let isFinalRead = currentUptimeMs() >= deadlineMs + let readDeadlineMs = isFinalRead ? Double.infinity : deadlineMs + switch readNavigationBarBackState(app: app, phase: .confirmation, deadlineMs: readDeadlineMs) { case .read(let after) where Self.navigationBarBackPopped(from: before, to: after): return .changed case .hung: return .hung case .read, .unreadable: break } - let remainingMs = deadlineMs - currentUptimeMs() - if remainingMs <= 0 { return .unchanged } - sleepFor(min(Self.navigationBarBackReadInterval, remainingMs / 1000)) + if isFinalRead { return .unchanged } + sleepFor(min(Self.navigationBarBackReadInterval, max(0, deadlineMs - currentUptimeMs()) / 1000)) } } @@ -434,8 +437,8 @@ extension RunnerTests { let held = holdingRecordedIssues { value = read() #if AGENT_DEVICE_RUNNER_UNIT_TESTS - if let injected = navigationBarBackReadIssueForTesting, injected.phase == phase { - record(XCTIssue(type: .assertionFailure, compactDescription: injected.description)) + if let description = navigationBarBackReadForTesting?(phase) { + record(XCTIssue(type: .assertionFailure, compactDescription: description)) } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 141b35cd3d..09bfe4e010 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -219,9 +219,10 @@ final class RunnerTests: XCTestCase { var litPanelWindowWalkIssueForTesting = false #if os(iOS) - /// Records an XCTest issue with this description during every navigation bar read of this phase - /// of iOS `back`, as a back button that vanishes mid-transition, or a hung query, does. - var navigationBarBackReadIssueForTesting: (phase: NavigationBarBackReadPhase, description: String)? + /// Runs at the end of every navigation bar read of iOS `back`, and records an XCTest issue with + /// the description it returns, as a bar that vanishes mid-transition, or a hung query, does. It + /// can also sleep, as a read on a slow host does. + var navigationBarBackReadForTesting: ((NavigationBarBackReadPhase) -> String?)? /// What the last iOS `back` saw of the navigation bar back button, so a test can tell the button /// path from the swipe when both navigate. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 3f2b353208..82f47f2dc6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -133,8 +133,8 @@ extension RunnerTests { func testInAppBackWithAnIssueInTheButtonLookupAnswersFromTheSwipe() throws { _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - navigationBarBackReadIssueForTesting = (.lookup, Self.vanishedBackButtonIssue) - defer { navigationBarBackReadIssueForTesting = nil } + navigationBarBackReadForTesting = { $0 == .lookup ? Self.vanishedBackButtonIssue : nil } + defer { navigationBarBackReadForTesting = nil } let response = try execute(command: try runnerCommandFixture( #"{"command":"backInApp","commandId":"back-lookup-issue","appBundleId":"com.callstack.agentdevice.runner"}"# @@ -152,10 +152,10 @@ extension RunnerTests { func testInAppBackWithAHungQueryInTheButtonLookupSendsNothing() throws { _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - defer { navigationBarBackReadIssueForTesting = nil } + defer { navigationBarBackReadForTesting = nil } let outcome = try expectingHungQueryRecorded { - navigationBarBackReadIssueForTesting = (.lookup, Self.hungQueryIssue) + navigationBarBackReadForTesting = { $0 == .lookup ? Self.hungQueryIssue : nil } return try performInAppBack(app: app) } @@ -163,7 +163,7 @@ extension RunnerTests { XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNotDispatched) XCTAssertEqual(lastNavigationBarBackTraceForTesting?.found, "hung") XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, false) - navigationBarBackReadIssueForTesting = nil + navigationBarBackReadForTesting = nil XCTAssertTrue(app.navigationBars["Detail"].exists, "nothing popped the detail screen") } @@ -173,10 +173,10 @@ extension RunnerTests { func testInAppBackWithAHungQueryInTheConfirmationAnswersUnverified() throws { _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - defer { navigationBarBackReadIssueForTesting = nil } + defer { navigationBarBackReadForTesting = nil } let outcome = try expectingHungQueryRecorded { - navigationBarBackReadIssueForTesting = (.confirmation, Self.hungQueryIssue) + navigationBarBackReadForTesting = { $0 == .confirmation ? Self.hungQueryIssue : nil } return try performInAppBack(app: app) } @@ -223,8 +223,8 @@ extension RunnerTests { func testInAppBackWithAnIssueInTheConfirmationAnswersUnverified() throws { let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - navigationBarBackReadIssueForTesting = (.confirmation, Self.vanishedBackButtonIssue) - defer { navigationBarBackReadIssueForTesting = nil } + navigationBarBackReadForTesting = { $0 == .confirmation ? Self.vanishedBackButtonIssue : nil } + defer { navigationBarBackReadForTesting = nil } let response = try execute(command: try runnerCommandFixture( #"{"command":"backInApp","commandId":"back-confirmation-issue","appBundleId":"com.callstack.agentdevice.runner"}"# @@ -232,11 +232,35 @@ extension RunnerTests { XCTAssertEqual(response.error?.code, RunnerWireErrorCode.backSwipeUnverified) XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) - navigationBarBackReadIssueForTesting = nil + navigationBarBackReadForTesting = nil let popped = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "Back pressed"), object: status) XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed, "the tap itself was delivered") } + /// A slow host can spend the whole confirmation budget on reads of the bar that is vanishing mid-pop, + /// each one an issue that proves nothing. The read after the budget sees the settled screen, so + /// the pop is confirmed, not left unverified. + @MainActor + func testInAppBackConfirmsAPopWhoseReadsInTheBudgetAllFailOnASlowHost() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + var confirmationReads = 0 + navigationBarBackReadForTesting = { phase in + guard phase == .confirmation else { return nil } + confirmationReads += 1 + guard confirmationReads <= 2 else { return nil } + Thread.sleep(forTimeInterval: 0.6) + return Self.vanishedBackButtonIssue + } + defer { navigationBarBackReadForTesting = nil } + + guard case .performed = try performInAppBack(app: app) else { + return XCTFail("a pop read after the budget must read as performed") + } + XCTAssertEqual(confirmationReads, 3, "both reads in the budget failed, and one read followed it") + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.confirmed, "yes") + } + /// A bar button titled "Back" is not the system back button, so back never presses it: it swipes, /// and the swipe that changes nothing reports so. @MainActor diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index c00c1e5ebb..b043a2eda2 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -155,6 +155,8 @@ and bar count, such as one between two screens whose back buttons have the same back button whose title updates itself, misread as a navigation. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is never followed by the swipe. Before and after the tap, reads start for up to 1 s each, checked before each bar's snapshot; a query in flight finishes or ends at XCTest's own query timeout. +After the tap, one more read follows the 1 s and runs to its end, so the screen after the pop is +always read: a snapshot of a bar that vanished mid-pop can spend the 1 s in XCTest's retries. Every bar read holds its XCTest issues. A held issue means the read proved nothing. A hung query ("Timed out while evaluating UI query") is recorded again, and no further query follows it: before diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index c82a55839b..7f3c8a8e3e 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,7 +124,7 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. After the tap, one more read follows that 1 s, so the screen after the pop is always read, even on a slow host. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. From 5d375298f0da84b49055a5ed9d98802cce8275b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 20:32:25 +0100 Subject: [PATCH 12/14] test(ios): fail every in-budget back confirmation read, not a fixed count The slow-host confirmation test failed two reads that each slept 0.6 s, so whether the final read came third depended on real read speed. The confirmation loop now tells the test hook which read starts after the budget, so the test fails every read in the budget and succeeds only on that final read. The protocol and command docs now say the final read can also fail or hang, and that a read can run well past the 1 s. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Navigation.swift | 13 +++++---- .../RunnerTests+NavigationTests.swift | 28 +++++++++++-------- apple/runner/RUNNER_PROTOCOL.md | 8 ++++-- website/docs/docs/commands.md | 2 +- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 51d0571358..39a8a5bf50 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -384,9 +384,9 @@ extension RunnerTests { } } - /// Whether a later read of the bar differs from `before`. Reads start until `deadlineMs`, and one - /// final read starts after it and runs to the end, so the screen after the pop is always read: a - /// read of a bar that vanishes mid-pop can spend the whole budget in XCTest's retries. A read in + /// Whether a later read of the bar differs from `before`. Reads start until `deadlineMs`, and + /// unless one of them confirmed or hung, one final read starts after it with no deadline: a read + /// of a bar that vanishes mid-pop can spend the whole budget in XCTest's retries. A read in /// flight ends at XCTest's own query timeout, which is a hung query. A read that fails proves /// nothing, so it never confirms. @MainActor @@ -398,7 +398,8 @@ extension RunnerTests { while true { let isFinalRead = currentUptimeMs() >= deadlineMs let readDeadlineMs = isFinalRead ? Double.infinity : deadlineMs - switch readNavigationBarBackState(app: app, phase: .confirmation, deadlineMs: readDeadlineMs) { + let phase: NavigationBarBackReadPhase = isFinalRead ? .finalConfirmation : .confirmation + switch readNavigationBarBackState(app: app, phase: phase, deadlineMs: readDeadlineMs) { case .read(let after) where Self.navigationBarBackPopped(from: before, to: after): return .changed case .hung: return .hung case .read, .unreadable: break @@ -415,10 +416,12 @@ extension RunnerTests { } /// Which side of the tap a read of the bar is on: a held issue in the lookup means no button, so - /// back swipes; one in the confirmation proves nothing, so it never confirms. + /// back swipes; one in the confirmation proves nothing, so it never confirms. The final + /// confirmation read is the one read that starts after the confirmation budget. enum NavigationBarBackReadPhase { case lookup case confirmation + case finalConfirmation } /// One read of the bar: its value, a read that proved nothing, or an XCTest query that hung. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 82f47f2dc6..4c8029843f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -223,7 +223,7 @@ extension RunnerTests { func testInAppBackWithAnIssueInTheConfirmationAnswersUnverified() throws { let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - navigationBarBackReadForTesting = { $0 == .confirmation ? Self.vanishedBackButtonIssue : nil } + navigationBarBackReadForTesting = { $0 == .lookup ? nil : Self.vanishedBackButtonIssue } defer { navigationBarBackReadForTesting = nil } let response = try execute(command: try runnerCommandFixture( @@ -238,26 +238,32 @@ extension RunnerTests { } /// A slow host can spend the whole confirmation budget on reads of the bar that is vanishing mid-pop, - /// each one an issue that proves nothing. The read after the budget sees the settled screen, so - /// the pop is confirmed, not left unverified. + /// each one an issue that proves nothing. Every read that starts in the budget fails here, so only + /// the read after the budget, which sees the settled screen, can confirm the pop. @MainActor - func testInAppBackConfirmsAPopWhoseReadsInTheBudgetAllFailOnASlowHost() throws { + func testInAppBackConfirmsAPopWhoseReadsInTheBudgetAllFail() throws { _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) defer { endBackFixture() } - var confirmationReads = 0 + var budgetReads = 0 + var finalReads = 0 navigationBarBackReadForTesting = { phase in - guard phase == .confirmation else { return nil } - confirmationReads += 1 - guard confirmationReads <= 2 else { return nil } - Thread.sleep(forTimeInterval: 0.6) - return Self.vanishedBackButtonIssue + switch phase { + case .lookup: return nil + case .confirmation: + budgetReads += 1 + return Self.vanishedBackButtonIssue + case .finalConfirmation: + finalReads += 1 + return nil + } } defer { navigationBarBackReadForTesting = nil } guard case .performed = try performInAppBack(app: app) else { return XCTFail("a pop read after the budget must read as performed") } - XCTAssertEqual(confirmationReads, 3, "both reads in the budget failed, and one read followed it") + XCTAssertGreaterThan(budgetReads, 0, "the reads in the budget ran and failed") + XCTAssertEqual(finalReads, 1, "the one read after the budget confirmed the pop") XCTAssertEqual(lastNavigationBarBackTraceForTesting?.confirmed, "yes") } diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index b043a2eda2..0ecb2e604f 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -155,8 +155,12 @@ and bar count, such as one between two screens whose back buttons have the same back button whose title updates itself, misread as a navigation. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is never followed by the swipe. Before and after the tap, reads start for up to 1 s each, checked before each bar's snapshot; a query in flight finishes or ends at XCTest's own query timeout. -After the tap, one more read follows the 1 s and runs to its end, so the screen after the pop is -always read: a snapshot of a bar that vanished mid-pop can spend the 1 s in XCTest's retries. +After the tap, when no read in the 1 s saw a change or hung, the runner starts one more read after +the 1 s, with no deadline of its own: a snapshot of a bar that vanished mid-pop can spend the 1 s in +XCTest's retries, and this read can still see the screen after the pop. That read can also fail, +which answers `BACK_SWIPE_UNVERIFIED`, or hang, which ends at XCTest's own query timeout and also +answers `BACK_SWIPE_UNVERIFIED`. A read in flight can run well past the 1 s: one bar snapshot took +about 2 s on a CI simulator. Every bar read holds its XCTest issues. A held issue means the read proved nothing. A hung query ("Timed out while evaluating UI query") is recorded again, and no further query follows it: before diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 7f3c8a8e3e..2272ab1f1d 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,7 +124,7 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. After the tap, one more read follows that 1 s, so the screen after the pop is always read, even on a slow host. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. From f1799d6f0d255be75d97527401ca1d82698ea23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 20:36:42 +0100 Subject: [PATCH 13/14] docs(back): name the final confirmation read in back_swipe_unverified Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerUITests/RunnerTests+Navigation.swift | 2 +- apple/runner/RUNNER_PROTOCOL.md | 2 +- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 39a8a5bf50..1b7dd7fb55 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -299,7 +299,7 @@ extension RunnerTests { static let navigationBarBackUnverified = ErrorPayload( code: RunnerWireErrorCode.backSwipeUnverified, - message: "back pressed the navigation bar back button, but the navigation bar's back buttons did not change within 1 s. This is an unknown outcome.", + message: "back pressed the navigation bar back button, but the navigation bar's back buttons did not change within 1 s or on one more read after it. This is an unknown outcome.", hint: "Take a snapshot to see where the app is." ) diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 0ecb2e604f..d88201deee 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -186,7 +186,7 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure | `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, `APP_SCREEN_OFF_LIT_PANEL` | The screen could not be sampled before the swipe. | Nothing | | `BACK_SWIPE_NOT_DISPATCHED` | The swipe was refused before any touch went down, an XCTest query on the navigation bar hung before the tap, or the platform has no back command (visionOS; the daemon refuses it first). | Nothing | | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | -| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | +| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or on one more read after it or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a 0.05 s pause between captures plus each capture's own time, that agree for as long as the gap the swipe will be credited across: a contact diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index a029af5093..293b2fe6de 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 2272ab1f1d..a93e539340 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -126,7 +126,7 @@ Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. - On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. -- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. +- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. - If the app's screen cannot be captured before the swipe, `back` does not swipe: it refuses with `COMMAND_FAILED` and `details.runnerErrorCode` set to `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, or `APP_SCREEN_OFF_LIT_PANEL`. Nothing is sent. - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back command and refuses before dispatch with `UNSUPPORTED_OPERATION`. From 40c2f690e719ffe27732b94f6bfd4e1ff7a8f93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 20:48:52 +0100 Subject: [PATCH 14/14] docs(back): describe NavigationBarBackReadPhase as the read hook's phase Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerUITests/RunnerTests+Navigation.swift | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 1b7dd7fb55..3919ecdcc5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -415,9 +415,8 @@ extension RunnerTests { return !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) } - /// Which side of the tap a read of the bar is on: a held issue in the lookup means no button, so - /// back swipes; one in the confirmation proves nothing, so it never confirms. The final - /// confirmation read is the one read that starts after the confirmation budget. + /// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, or the + /// final confirmation read. The read passes it to the test read hook. enum NavigationBarBackReadPhase { case lookup case confirmation