diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 1df45efe69..87a32bb113 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -39,6 +39,117 @@ @interface AgentDeviceRunnerFixtureViewController : AgentDeviceRunnerViewController @end +// A real navigation stack for `back`. `--agent-device-back-navigation-bar` pushes a detail screen, +// so the bar shows UIKit's system back button; the interactive pop is off, so only that button can +// pop it. `--blocked-back` keeps that button but replaces its action, so pressing it navigates +// nowhere. `--shifting-back` makes that press move the bar down instead, so the back button moves +// without any navigation. `--ticking-title` counts the detail screen's title down every 0.2 s, as a timer does, +// so its bar changes without any navigation. `--custom-back-item` instead puts a bar button titled +// "Back" on the root, which is not the system back button and must never be pressed. The root's +// label reports what happened. +@interface AgentDeviceBackNavigationFixture : UINavigationController +@property(nonatomic, strong) UILabel *status; +@property(nonatomic, strong) NSTimer *titleTimer; +@end + +@implementation AgentDeviceBackNavigationFixture +- (instancetype)initWithCustomBackItem:(BOOL)customBackItem blockedBack:(BOOL)blockedBack { + UIViewController *root = [UIViewController new]; + root.title = @"Root"; + root.view.backgroundColor = UIColor.systemBackgroundColor; + self = [super initWithRootViewController:root]; + if (!self) return nil; + UILabel *status = [UILabel new]; + status.text = @"Navigation root"; + status.accessibilityIdentifier = @"agent-device-back-navigation-status"; + status.translatesAutoresizingMaskIntoConstraints = NO; + [root.view addSubview:status]; + [NSLayoutConstraint activateConstraints:@[ + [status.centerXAnchor constraintEqualToAnchor:root.view.centerXAnchor], + [status.centerYAnchor constraintEqualToAnchor:root.view.centerYAnchor], + ]]; + self.status = status; + self.modalPresentationStyle = UIModalPresentationFullScreen; + if (customBackItem) { + root.navigationItem.leftBarButtonItem = + [[UIBarButtonItem alloc] initWithTitle:@"Back" style:UIBarButtonItemStylePlain target:self + action:@selector(customBackPressed)]; + } else { + UIViewController *detail = [UIViewController new]; + detail.title = @"Detail"; + detail.view.backgroundColor = UIColor.systemTealColor; + if (blockedBack) { + if (@available(iOS 16.0, *)) { + BOOL shifting = [NSProcessInfo.processInfo.arguments containsObject:@"--shifting-back"]; + __weak UINavigationController *weakSelf = self; + detail.navigationItem.backAction = [UIAction actionWithHandler:^(__kindof UIAction *action) { + if (shifting) weakSelf.additionalSafeAreaInsets = UIEdgeInsetsMake(30, 0, 0, 0); + }]; + } + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--ticking-title"]) { + __block NSInteger remaining = 59; + __weak UIViewController *weakDetail = detail; + self.titleTimer = [NSTimer scheduledTimerWithTimeInterval:0.2 repeats:YES block:^(NSTimer *timer) { + remaining = remaining > 0 ? remaining - 1 : 59; + weakDetail.title = [NSString stringWithFormat:@"0:%02ld", (long)remaining]; + }]; + } + [self pushViewController:detail animated:NO]; + } + return self; +} + +- (void)viewDidLoad { + [super viewDidLoad]; + self.interactivePopGestureRecognizer.enabled = NO; + self.delegate = self; +} + +- (void)customBackPressed { + self.status.text = @"Custom back pressed"; +} + +- (void)navigationController:(UINavigationController *)navigationController + didShowViewController:(UIViewController *)viewController + animated:(BOOL)animated { + // Only the pushed stack ever shows its root again, and only by popping the detail screen. + if (viewController == self.viewControllers.firstObject && viewController.navigationItem.leftBarButtonItem == nil) { + self.status.text = @"Back pressed"; + } +} +@end + +// `--two-stacks` shows two pushed navigation stacks, one above the other, so two hittable system +// back buttons are on screen at once and neither is the one `back` should press. +@interface AgentDeviceTwoStacksBackFixture : UIViewController +@end + +@implementation AgentDeviceTwoStacksBackFixture +- (void)viewDidLoad { + [super viewDidLoad]; + self.view.backgroundColor = UIColor.systemBackgroundColor; + UIStackView *stack = [UIStackView new]; + stack.axis = UILayoutConstraintAxisVertical; + stack.distribution = UIStackViewDistributionFillEqually; + stack.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:stack]; + [NSLayoutConstraint activateConstraints:@[ + [stack.topAnchor constraintEqualToAnchor:self.view.safeAreaLayoutGuide.topAnchor], + [stack.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor], + [stack.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor], + [stack.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor], + ]]; + for (NSInteger index = 0; index < 2; index++) { + AgentDeviceBackNavigationFixture *child = [[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:NO + blockedBack:NO]; + [self addChildViewController:child]; + [stack addArrangedSubview:child.view]; + [child didMoveToParentViewController:self]; + } +} +@end + @interface AgentDeviceRunnerFixtureViewController () // Held for the app's lifetime: a window nobody retained would be gone before the test queried it. @property(nonatomic, strong) UIWindow *secondWindow; @@ -368,6 +479,17 @@ - (void)presentAlertDeclineChoiceFixture { - (void)viewDidAppear:(BOOL)animated { [super viewDidAppear:animated]; + if (!self.alertFixtureStarted && + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-back-navigation-bar"]) { + self.alertFixtureStarted = YES; + BOOL customBackItem = [NSProcessInfo.processInfo.arguments containsObject:@"--custom-back-item"]; + BOOL blockedBack = [NSProcessInfo.processInfo.arguments containsObject:@"--blocked-back"]; + UIViewController *fixture = [NSProcessInfo.processInfo.arguments containsObject:@"--two-stacks"] + ? [AgentDeviceTwoStacksBackFixture new] + : [[AgentDeviceBackNavigationFixture alloc] initWithCustomBackItem:customBackItem blockedBack:blockedBack]; + fixture.modalPresentationStyle = UIModalPresentationFullScreen; + [self presentViewController:fixture animated:NO completion:nil]; + } if (!self.alertFixtureStarted && ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"] || [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-affirmative-only"])) { @@ -608,12 +730,12 @@ - (void)viewDidLoad { } }]; } - // `--slow-toggle` repaints the whole screen every 0.5 s, slower than one verification delay, as + // `--slow-toggle` repaints the whole screen every 0.3 s, slower than one verification delay, as // a carousel does: one quiet pair of samples must not pass it for settled. if ([NSProcessInfo.processInfo.arguments containsObject:@"--slow-toggle"]) { __weak UIView *view = self.view; [self.animationTimer invalidate]; - self.animationTimer = [NSTimer scheduledTimerWithTimeInterval:0.5 repeats:YES block:^(NSTimer *timer) { + self.animationTimer = [NSTimer scheduledTimerWithTimeInterval:0.3 repeats:YES block:^(NSTimer *timer) { view.backgroundColor = [view.backgroundColor isEqual:UIColor.blackColor] ? UIColor.whiteColor : UIColor.blackColor; }]; } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index b53e22fd9e..3919ecdcc5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -2,7 +2,13 @@ import AgentDeviceSnapshotPresentation import XCTest extension RunnerTests { - static let backSwipeVerificationDelay: TimeInterval = 0.25 + /// UIKit's identifier for a navigation bar's system back button, in UIKit and SwiftUI stacks alike. + static let navigationBarBackButtonIdentifier = "BackButton" + /// The wait from the end of the swipe's contact to the sample it is compared against. An + /// interactive pop follows the contact across most of the width, so the screen has already moved. + static let backSwipeVerificationDelay: TimeInterval = 0.1 + /// The pause between two settle samples; a capture itself takes about 0.1 s on a simulator. + static let backSwipeSettleSampleInterval: TimeInterval = 0.05 /// The time the contact itself is allowed in the gap from the sample taken just before the swipe to /// the one after it. Both are resolved before that sample: a simulator's edge touch runs a 100 ms /// contact, and a device's synthesized fling runs 200 ms. @@ -11,7 +17,21 @@ extension RunnerTests { #else static let backGestureAllowanceMs = 400.0 #endif - static let backSwipeSettleSampleBudget = 8 + /// The least time settling may sample before back swipes anyway and reports the outcome + /// unverified; `backSwipeSettleBudgetMs` extends it on a host that captures slowly. + static let backSwipeSettleBaseBudgetMs = 2000.0 + + /// How long settling may sample: the base budget, or, when longer, the time a still screen needs + /// to settle at the slowest capture seen: the first capture, the required still span, and one more + /// pause and capture for the sample that completes the span. A slow host still settles a still + /// screen instead of always reporting it unverified. + static func backSwipeSettleBudgetMs(maxCaptureMs: Double) -> Double { + let sampleIntervalMs = backSwipeSettleSampleInterval * 1000 + maxCaptureMs + return max( + backSwipeSettleBaseBudgetMs, + maxCaptureMs + backSwipeRequiredStillMs(maxCaptureMs: maxCaptureMs) + sampleIntervalMs + ) + } /// How long the screen must agree with itself before the swipe: as long as the gap it will be /// credited across, the contact allowance, one verification delay, and the slowest capture seen @@ -66,8 +86,10 @@ extension RunnerTests { } } - /// iOS `back` is the left-edge back swipe, verified by a change of the app's screen. A visible - /// back control is the caller's to press by ref; the runner does not guess at buttons. + /// iOS `back` presses the navigation bar's system back button when exactly one is on screen, read + /// from the live tree in this call, and confirms that the bar changed. Otherwise it is the + /// left-edge back swipe, verified by a change of the app's screen. No other button counts, + /// whatever its label. @MainActor func performInAppBack(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) @@ -85,24 +107,41 @@ extension RunnerTests { _ = pressTvRemote(.menu) return .performed #elseif os(iOS) - // One line on every exit, a refusal before the swipe and a thrown error included. + // The swipe logs one line on every exit, a refusal before it and a thrown error included. The + // button path logs its own line on every exit, a thrown tap included. var trace = BackSwipeTrace() + func logSwipe(_ line: String) { NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", line) } + let resolved: SynthesizedCoordinateContext? + do { + resolved = try synthesizedCoordinateContext(app: app) + } catch { + logSwipe(trace.logLine(name: "error", code: String(describing: type(of: error)))) + throw error + } + guard let context = resolved else { + let outcome = InAppBackOutcome.refused(Self.backScreenRefusal(.unresolvedWindow)) + logSwipe(trace.logLine(outcome: outcome)) + return outcome + } + if let pressed = try pressNavigationBarBackButton(app: app, context: context) { + return pressed + } do { - let outcome = try verifiedBackSwipe(app: app, trace: &trace) - NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(outcome: outcome)) + let outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) + logSwipe(trace.logLine(outcome: outcome)) return outcome } catch { - NSLog("AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION %@", trace.logLine(name: "error", code: String(describing: type(of: error)))) + logSwipe(trace.logLine(name: "error", code: String(describing: type(of: error)))) throw error } #else - // visionOS has no edge swipe that pops a navigation stack. The daemon refuses it at the back - // capability fact, so this answers only a direct runner request, with nothing sent. + // visionOS has no back command. The daemon refuses it at the back capability fact, so this + // answers only a direct runner request, with nothing sent. return .refused( ErrorPayload( code: RunnerWireErrorCode.backSwipeNotDispatched, - message: "visionOS has no back gesture", - hint: "iOS back is a left-edge swipe; visionOS has none. Press the visible back control by its ref." + message: "visionOS has no back command", + hint: "Press the visible back control by its ref." ) ) #endif @@ -144,42 +183,350 @@ extension RunnerTests { } } + /// Presses the navigation bar's system back button when exactly one hittable one is on screen, + /// and confirms that the bar changed. It is read only where the tree answers queries: on a tree + /// XCTest cannot serialize, the query would grind through XCTest's retries, so back swipes there. + /// Two candidates, such as a sheet over a stack, are ambiguous and fall back to the swipe too, as + /// does a bar still moving after the read budget. Nil when nothing was sent, so the swipe is still + /// safe; a delivered tap is never followed by a swipe. A hung XCTest query answers without either. @MainActor - private func verifiedBackSwipe(app: XCUIApplication, trace: inout BackSwipeTrace) throws -> InAppBackOutcome { - guard let context = try synthesizedCoordinateContext(app: app) else { - return .refused(Self.backScreenRefusal(.unresolvedWindow)) + private func pressNavigationBarBackButton( + app: XCUIApplication, + context: SynthesizedCoordinateContext + ) throws -> InAppBackOutcome? { + var trace = NavigationBarBackTrace() + // One line on every exit, a thrown tap included. + defer { + NSLog("AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON %@", trace.logLine) +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + lastNavigationBarBackTraceForTesting = trace +#endif } - let before: (sample: Data, stableSinceMs: Double, settled: Bool) - switch settledBackVisualState(app: app, context: context, trace: &trace) { - case .success(let settled): before = settled - case .failure(let refusal): return .refused(Self.backScreenRefusal(refusal)) + guard navigationBarBackButtonIsReadable(context: context) else { return nil } + let started = currentUptimeMs() + let lookup = stillNavigationBarBackButton( + app: app, + geometry: context.geometry, + deadlineMs: started + Self.navigationBarBackReadBudgetMs + ) + trace.lookupMs = currentUptimeMs() - started + let before: NavigationBarBackState + let frame: CGRect + switch lookup { + case .still(let state, let button): + before = state + frame = button + case .notFound(let found): + trace.found = found + return nil + case .hung: + trace.found = "hung" + return .refused(Self.navigationBarBackHungLookup) + } + trace.found = "yes" + // The synthesized tap returns once the touch is delivered; an XCTest tap would also wait out the + // pop animation, which the bar check below reads sooner. On iOS a readable tree's XCTest tap + // either runs or throws, so its fallback needs no outcome check. + switch try performSynthesizedTapGesture(app: app, x: frame.midX, y: frame.midY, context: context) { + case .performed: + break + case .xctestFallback: + _ = try performGesture(app) { try tapAt(app: app, x: frame.midX, y: frame.midY, context: context) } + case .refused: + return nil + } + trace.pressed = true + let tapped = currentUptimeMs() + let confirmation = navigationBarBackConfirmation( + app: app, + from: before, + deadlineMs: tapped + Self.navigationBarBackReadBudgetMs + ) + trace.confirmMs = currentUptimeMs() - tapped + trace.confirmed = confirmation.logToken + switch confirmation { + case .changed: return .performed + case .unchanged: return .unconfirmed(Self.navigationBarBackUnverified) + case .hung: return .unconfirmed(Self.navigationBarBackHungConfirmation) + } + } + + /// What the back button path saw, logged as one line on every exit. + struct NavigationBarBackTrace { + /// `skipped` when the tree was not read, `unreadable` when a read failed, `hung` when an XCTest + /// query hung, `moving` when the bar did not hold still within the read budget, `no`, + /// `ambiguous`, or `yes`. + var found = "skipped" + var pressed = false + /// `yes`, `no`, or `hung`; nil until a tap was delivered. + var confirmed: String? + var lookupMs = 0.0 + var confirmMs: Double? + + var logLine: String { + func ms(_ value: Double?) -> String { value.map { String(Int($0.rounded())) } ?? "-" } + return [ + "found=\(found)", + "pressed=\(pressed ? "yes" : "no")", + "confirmed=\(confirmed ?? "-")", + "lookupMs=\(ms(lookupMs))", + "confirmMs=\(ms(confirmMs))", + ].joined(separator: " ") + } + } + + /// The labels of each navigation bar's system back buttons, one list per bar in bar order, and + /// the frames of those buttons. Both must agree for a still bar before the tap. Bar titles and + /// identifiers are left out: a bar can retitle itself, such as a countdown, without any navigation. + struct NavigationBarBackState: Equatable { + let backButtonLabels: [[String]] + let backButtonFrames: [CGRect] + } + + /// Whether the bar after the tap shows a pop: the bar count, the back button count, or a back + /// button label changed. Frames are left out: a tap that navigates nowhere can still move the + /// button, such as by a large-title collapse or a bar button that appears. A pop that keeps the + /// same back button labels and bar count answers unverified. + static func navigationBarBackPopped(from before: NavigationBarBackState, to after: NavigationBarBackState) -> Bool { + after.backButtonLabels != before.backButtonLabels + } + + /// The pause between two reads of the bar, and how long reads may start, both before the tap + /// while the bar settles and after it while the pop is confirmed. Only a bar that never holds + /// still, or a tap that did not navigate, spends the whole budget. + static let navigationBarBackReadInterval: TimeInterval = 0.05 + static let navigationBarBackReadBudgetMs = 1000.0 + + static let navigationBarBackUnverified = ErrorPayload( + code: RunnerWireErrorCode.backSwipeUnverified, + message: "back pressed the navigation bar back button, but the navigation bar's back buttons did not change within 1 s or on one more read after it. This is an unknown outcome.", + hint: "Take a snapshot to see where the app is." + ) + + static let navigationBarBackHungConfirmation = ErrorPayload( + code: RunnerWireErrorCode.backSwipeUnverified, + message: "back pressed the navigation bar back button, but an XCTest query on the navigation bar hung afterwards. This is an unknown outcome.", + hint: "Take a snapshot to see where the app is." + ) + + static let navigationBarBackHungLookup = ErrorPayload( + code: RunnerWireErrorCode.backSwipeNotDispatched, + message: "back sent nothing: an XCTest query on the navigation bar hung before the back button or the swipe.", + hint: "Take a snapshot to check that the app responds, then retry back." + ) + + /// What the read before the tap found: a still bar with its one hittable back button, no button + /// to press (with the trace's `found` value), or a hung query. + enum NavigationBarBackLookup { + case still(NavigationBarBackState, button: CGRect) + case notFound(String) + case hung + } + + /// Reads the hittable back buttons, then the bar state until two reads in a row, one read interval + /// apart, agree and the button's frame is in them, so a push still animating is not read + /// as the state a pop changes. Each round re-reads the buttons. Reads start only until `deadlineMs`. + @MainActor + private func stillNavigationBarBackButton( + app: XCUIApplication, + geometry: SurfaceGeometry, + deadlineMs: Double + ) -> NavigationBarBackLookup { + var previous: NavigationBarBackState? + while true { + let button: CGRect + switch navigationBarBackButtonFrames(app: app, deadlineMs: deadlineMs) { + case .hung: return .hung + case .unreadable: return .notFound("unreadable") + case .read(let frames) where frames.isEmpty: return .notFound("no") + case .read(let frames) where frames.count > 1: return .notFound("ambiguous") + case .read(let frames): button = frames[0] + } + let current: NavigationBarBackState + switch readNavigationBarBackState(app: app, phase: .lookup, deadlineMs: deadlineMs) { + case .hung: return .hung + case .unreadable: return .notFound("unreadable") + case .read(let state): current = state + } + if current == previous, Self.navigationBarBackState(current, holdsButton: button, geometry: geometry) { + return .still(current, button: button) + } + if currentUptimeMs() >= deadlineMs { return .notFound("moving") } + previous = current + sleepFor(Self.navigationBarBackReadInterval) + } + } + + /// Whether `state` holds the live button. A live frame is in screen points, and a snapshot frame + /// is in the window's own space, which a placed iPad window starts off the screen origin (#3899). + static func navigationBarBackState( + _ state: NavigationBarBackState, + holdsButton button: CGRect, + geometry: SurfaceGeometry + ) -> Bool { + state.backButtonFrames.contains { frame in + let origin = geometry.screenPoint(from: SurfacePoint(frame.origin)).point + return CGRect(origin: origin, size: frame.size) == button } - // Everything but the contact is resolved before the sample the change is measured from. + } + + enum NavigationBarBackConfirmation { + case changed + case unchanged + case hung + + var logToken: String { + switch self { + case .changed: return "yes" + case .unchanged: return "no" + case .hung: return "hung" + } + } + } + + /// Whether a later read of the bar differs from `before`. Reads start until `deadlineMs`, and + /// unless one of them confirmed or hung, one final read starts after it with no deadline: a read + /// of a bar that vanishes mid-pop can spend the whole budget in XCTest's retries. A read in + /// flight ends at XCTest's own query timeout, which is a hung query. A read that fails proves + /// nothing, so it never confirms. + @MainActor + private func navigationBarBackConfirmation( + app: XCUIApplication, + from before: NavigationBarBackState, + deadlineMs: Double + ) -> NavigationBarBackConfirmation { + while true { + let isFinalRead = currentUptimeMs() >= deadlineMs + let readDeadlineMs = isFinalRead ? Double.infinity : deadlineMs + let phase: NavigationBarBackReadPhase = isFinalRead ? .finalConfirmation : .confirmation + switch readNavigationBarBackState(app: app, phase: phase, deadlineMs: readDeadlineMs) { + case .read(let after) where Self.navigationBarBackPopped(from: before, to: after): return .changed + case .hung: return .hung + case .read, .unreadable: break + } + if isFinalRead { return .unchanged } + sleepFor(min(Self.navigationBarBackReadInterval, max(0, deadlineMs - currentUptimeMs()) / 1000)) + } + } + + @MainActor + private func navigationBarBackButtonIsReadable(context: SynthesizedCoordinateContext) -> Bool { + guard !context.treeUnreadable, context.accessibilityHealth != .unavailable else { return false } + return !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) + } + + /// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, or the + /// final confirmation read. The read passes it to the test read hook. + enum NavigationBarBackReadPhase { + case lookup + case confirmation + case finalConfirmation + } + + /// One read of the bar: its value, a read that proved nothing, or an XCTest query that hung. + enum NavigationBarBackRead { + case read(Value) + case unreadable + case hung + } + + /// Runs one read of the bar with its XCTest issues held. A button gone mid-transition records an + /// issue that the read explains: the read proves nothing. A hung query is recorded again, as + /// `record(_:)` requires, and no further query is sent into it. + @MainActor + private func heldNavigationBarRead(_ phase: NavigationBarBackReadPhase, _ read: () -> T?) -> NavigationBarBackRead { + var value: T? + let held = holdingRecordedIssues { + value = read() +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + if let description = navigationBarBackReadForTesting?(phase) { + record(XCTIssue(type: .assertionFailure, compactDescription: description)) + } +#endif + } + let hung = held.issues.filter { Self.isHungQueryIssueDescription($0.compactDescription) } + hung.forEach { record($0) } + if !hung.isEmpty { return .hung } + guard held.issues.isEmpty, held.exceptionMessage == nil, let value else { return .unreadable } + return .read(value) + } + + /// The frames of the hittable system back buttons; unreadable when the tree did not answer or the + /// read budget had run out. + @MainActor + private func navigationBarBackButtonFrames(app: XCUIApplication, deadlineMs: Double) -> NavigationBarBackRead<[CGRect]> { + guard currentUptimeMs() < deadlineMs else { return .unreadable } + return heldNavigationBarRead(.lookup) { + guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } + let frames = bars + .flatMap { $0.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex } + .filter { $0.isHittable } + .map(\.frame) + return frames.allSatisfy(SnapshotGeometry.isPositiveFinite) ? frames : nil + } + } + + /// One XCTest snapshot per bar, so a read costs a query per bar rather than one per attribute. + /// Unreadable when the tree did not answer or the read budget ran out before a bar's snapshot. + @MainActor + private func readNavigationBarBackState( + app: XCUIApplication, + phase: NavigationBarBackReadPhase, + deadlineMs: Double + ) -> NavigationBarBackRead { + heldNavigationBarRead(phase) { + guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } + var backButtonLabels: [[String]] = [] + var backButtonFrames: [CGRect] = [] + for bar in bars.flatMap({ $0.allElementsBoundByIndex }) { + guard currentUptimeMs() < deadlineMs, let snapshot = try? bar.snapshot() else { return nil } + let buttons = Self.navigationBarBackButtons(snapshot) + backButtonLabels.append(buttons.map(\.label)) + backButtonFrames.append(contentsOf: buttons.map(\.frame)) + } + return NavigationBarBackState(backButtonLabels: backButtonLabels, backButtonFrames: backButtonFrames) + } + } + + static func navigationBarBackButtons(_ bar: XCUIElementSnapshot) -> [(label: String, frame: CGRect)] { + var buttons: [(label: String, frame: CGRect)] = [] + var pending = bar.children + while let element = pending.popLast() { + if element.identifier == navigationBarBackButtonIdentifier { + buttons.append((element.label, element.frame)) + } + pending.append(contentsOf: element.children) + } + return buttons + } + + @MainActor + private func verifiedBackSwipe( + app: XCUIApplication, + context: SynthesizedCoordinateContext, + trace: inout BackSwipeTrace + ) throws -> InAppBackOutcome { + // Everything but the contact is resolved before settling, so the last still sample is the one + // the change is measured from and the swipe follows it at once. let dispatchGesture: @MainActor () throws -> TouchDispatch switch try prepareBackGesture(app: app, context: context) { case .ready(let dispatch): dispatchGesture = dispatch case .notSent(let message, let hint): return .refused(ErrorPayload(code: RunnerWireErrorCode.backSwipeNotDispatched, message: message, hint: hint)) } - // One more sample right before the swipe: the change credited to the swipe is measured from it. - let justBefore: Data - switch timedBackCapture(app: app, context: context, trace: &trace) { - case .success(let sample): justBefore = sample + let before: SettledBackSample + switch settledBackVisualState(app: app, context: context, trace: &trace) { + case .success(let settled): before = settled case .failure(let refusal): return .refused(Self.backScreenRefusal(refusal)) } - let justBeforeMs = currentUptimeMs() + let justBeforeMs = before.sampledMs trace.stableMs = justBeforeMs - before.stableSinceMs - let stillSettled = Self.backSwipeBaselineHolds( - settled: before.settled, - baseline: before.sample, - justBefore: justBefore - ) let dispatch = try dispatchGesture() trace.gestureMs = currentUptimeMs() - justBeforeMs if let refusal = Self.backGestureRefusal(dispatch) { return refusal } - guard stillSettled else { + guard before.settled else { return .unconfirmed( Self.backSwipeUnverified( because: "the app's screen was already changing on its own", @@ -191,7 +538,7 @@ extension RunnerTests { let after = timedBackCapture(app: app, context: context, trace: &trace) let afterMs = currentUptimeMs() trace.justBeforeToAfterMs = afterMs - justBeforeMs - let observation = Self.navigationVisualObservation(before: .success(justBefore), after: after) + let observation = Self.navigationVisualObservation(before: .success(before.sample), after: after) trace.changed = observation guard Self.backSwipeGapIsCovered( stableSinceMs: before.stableSinceMs, @@ -248,37 +595,45 @@ extension RunnerTests { } #if os(iOS) - /// Samples a verification delay apart until every sample of a run has agreed with its first one - /// for `backSwipeRequiredStillMs` of the slowest capture so far. `stableSinceMs` is when that run - /// began; `settled` is false when the budget ran out first. + /// The last settle sample: the baseline the swipe's change is measured from. `stableSinceMs` is + /// when the run of agreeing samples it ends began, and `sampledMs` when it was captured. + struct SettledBackSample { + let sample: Data + let stableSinceMs: Double + let sampledMs: Double + let settled: Bool + } + + /// Samples `backSwipeSettleSampleInterval` apart until every sample of a run has agreed with its + /// first one for `backSwipeRequiredStillMs` of the slowest capture so far. `settled` is false when + /// `backSwipeSettleBudgetMs` of the slowest capture so far ran out first. @MainActor private func settledBackVisualState( app: XCUIApplication, context: SynthesizedCoordinateContext, trace: inout BackSwipeTrace - ) -> Result<(sample: Data, stableSinceMs: Double, settled: Bool), RunnerAppScreenCaptureFailure> { + ) -> Result { + let started = currentUptimeMs() var runStart: (sample: Data, ms: Double)? - var last = Data() - for index in 0.. 0 { sleepFor(Self.backSwipeVerificationDelay) } - trace.settleSamples = index + 1 + while true { + if trace.settleSamples > 0 { sleepFor(Self.backSwipeSettleSampleInterval) } + trace.settleSamples += 1 + let sample: Data switch timedBackCapture(app: app, context: context, trace: &trace) { - case .failure(let refusal): - return .failure(refusal) - case .success(let sample): - let now = currentUptimeMs() - last = sample - guard let start = runStart, !Self.visualSamplesDiffer(start.sample, sample) else { - runStart = (sample, now) - continue - } - if now - start.ms >= Self.backSwipeRequiredStillMs(maxCaptureMs: trace.maxCaptureMs) { - trace.settled = true - return .success((sample, start.ms, true)) - } + case .failure(let refusal): return .failure(refusal) + case .success(let captured): sample = captured + } + let now = currentUptimeMs() + let start = runStart.flatMap { Self.visualSamplesDiffer($0.sample, sample) ? nil : $0 } ?? (sample: sample, ms: now) + runStart = start + if now - start.ms >= Self.backSwipeRequiredStillMs(maxCaptureMs: trace.maxCaptureMs) { + trace.settled = true + return .success(SettledBackSample(sample: sample, stableSinceMs: start.ms, sampledMs: now, settled: true)) + } + if now - started >= Self.backSwipeSettleBudgetMs(maxCaptureMs: trace.maxCaptureMs) { + return .success(SettledBackSample(sample: sample, stableSinceMs: start.ms, sampledMs: now, settled: false)) } } - return .success((last, runStart?.ms ?? currentUptimeMs(), false)) } #endif @@ -368,12 +723,6 @@ extension RunnerTests { afterMs - justBeforeMs <= justBeforeMs - stableSinceMs } - /// Whether the swipe's effect can be measured: the screen settled, and the sample taken right - /// before the swipe still matches the settled baseline. - static func backSwipeBaselineHolds(settled: Bool, baseline: Data, justBefore: Data) -> Bool { - settled && !visualSamplesDiffer(baseline, justBefore) - } - /// What an observation of the swipe's before/after samples concludes. Kept apart from the sleep /// and the capture so the three-way decision is testable without a running app. static func backSwipeOutcome(observation: NavigationVisualObservation) -> InAppBackOutcome { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b0b3e8f569..09bfe4e010 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -218,6 +218,17 @@ final class RunnerTests: XCTestCase { /// Records an XCTest issue during the lit-panel window walk, as a window that vanishes mid-walk does. var litPanelWindowWalkIssueForTesting = false +#if os(iOS) + /// Runs at the end of every navigation bar read of iOS `back`, and records an XCTest issue with + /// the description it returns, as a bar that vanishes mid-transition, or a hung query, does. It + /// can also sleep, as a read on a slow host does. + var navigationBarBackReadForTesting: ((NavigationBarBackReadPhase) -> String?)? + + /// What the last iOS `back` saw of the navigation bar back button, so a test can tell the button + /// path from the swipe when both navigate. + var lastNavigationBarBackTraceForTesting: NavigationBarBackTrace? +#endif + static let injectedTapFailureFlagPathForTesting = "/tmp/agent-device-inject-tap-recorded-failure-for-testing" @@ -315,6 +326,14 @@ final class RunnerTests: XCTestCase { "Failed to resolve query" ] + /// XCTest's wording for a query that hung. It must keep reaching `record(_:)`: a read that holds + /// its issues records this one again. + static let hungQueryIssueWording = "Timed out while evaluating UI query" + + static func isHungQueryIssueDescription(_ description: String) -> Bool { + description.contains(hungQueryIssueWording) + } + /// True for the one recorded-issue class the runner deliberately mutes: an AX-server error /// (`kAXError*`, deliberately covering kAXErrorIllegalArgument and its sibling codes) inside one /// of the fetch wordings above. Variants without that token MUST keep recording: the timeout diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 6b58f0820e..4c8029843f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -1,3 +1,4 @@ +import AgentDeviceSnapshotPresentation import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) import ObjectiveC.runtime @@ -44,9 +45,10 @@ extension RunnerTests { XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) } - /// A screen that changes on its own every 0.5 s is never still for the required span (the contact - /// allowance, one 0.25 s delay, and the slowest capture, over 0.5 s on a simulator), so it never - /// settles and back reports an unknown outcome instead of crediting the swipe. + /// A screen that changes on its own every 0.3 s, three verification delays apart, is never still + /// for the required span (the contact allowance, one 0.1 s delay, and the slowest capture, about + /// 0.4 s on a simulator), so it never settles and back reports an unknown outcome instead of + /// crediting the swipe. @MainActor func testInAppBackOnAScreenChangingSlowerThanOneDelayIsNotPerformed() throws { _ = try launchBackFixture(["--agent-device-back-regression", "--slow-toggle"]) @@ -58,6 +60,227 @@ extension RunnerTests { XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) } + /// The navigation bar's system back button is pressed from the live tree: the fixture's + /// interactive pop is off, so only that button can pop the detail screen. + @MainActor + func testInAppBackPressesTheNavigationBarBackButton() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + + guard case .performed = try performInAppBack(app: app) else { + return XCTFail("a visible navigation bar back button must be pressed") + } + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true, "the button, not the swipe, popped") + let popped = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "Back pressed"), object: status) + XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed) + } + + /// A delivered tap on the system back button is not a navigation: when the bar does not change, + /// back reports an unknown outcome and never swipes after the tap. + @MainActor + func testInAppBackReportsAPressedBackButtonThatDoesNotNavigateAsUnverified() throws { + guard #available(iOS 16.0, *) else { throw XCTSkip("UINavigationItem.backAction needs iOS 16") } + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--blocked-back"]) + defer { endBackFixture() } + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a back button tap that leaves the bar unchanged must not read as performed") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertTrue(app.navigationBars["Detail"].exists) + } + + /// A bar title that counts down on its own is not a navigation: a tap that navigates nowhere on + /// such a bar reports an unknown outcome, though the title changes within the confirmation window. + @MainActor + func testInAppBackDoesNotCreditATickingBarTitleToAPressedBackButton() throws { + guard #available(iOS 16.0, *) else { throw XCTSkip("UINavigationItem.backAction needs iOS 16") } + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--blocked-back", "--ticking-title"]) + defer { endBackFixture() } + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a ticking bar title must not confirm a back button tap that navigated nowhere") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + XCTAssertEqual(app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).count, 1) + } + + /// A back button that only moves after the tap is not a navigation: a tap that navigates nowhere + /// but shifts the bar, as a large-title collapse does, reports an unknown outcome. + @MainActor + func testInAppBackDoesNotCreditAMovedBackButtonToAPressedBackButton() throws { + guard #available(iOS 16.0, *) else { throw XCTSkip("UINavigationItem.backAction needs iOS 16") } + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--blocked-back", "--shifting-back"]) + defer { endBackFixture() } + let backButton = app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).firstMatch + let frameBefore = backButton.frame + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a back button that only moved must not confirm a tap that navigated nowhere") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + XCTAssertTrue(app.navigationBars["Detail"].exists) + XCTAssertNotEqual(backButton.frame.minY, frameBefore.minY, "the fixture must move the back button") + } + + /// An XCTest issue while the back button is looked up, such as a button gone mid-transition, means + /// no button: back answers from the swipe path. An issue that escaped the read would fail this + /// test as it fails the command with a recorded failure. The swipe's own outcome is not asserted: + /// iOS 27 pops this fixture on the edge swipe even with its interactive pop turned off. + @MainActor + func testInAppBackWithAnIssueInTheButtonLookupAnswersFromTheSwipe() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + navigationBarBackReadForTesting = { $0 == .lookup ? Self.vanishedBackButtonIssue : nil } + defer { navigationBarBackReadForTesting = nil } + + let response = try execute(command: try runnerCommandFixture( + #"{"command":"backInApp","commandId":"back-lookup-issue","appBundleId":"com.callstack.agentdevice.runner"}"# + )) + + XCTAssertNotEqual(response.error?.code, "XCTEST_RECORDED_FAILURE") + let trace = try XCTUnwrap(lastNavigationBarBackTraceForTesting) + XCTAssertEqual(trace.found, "unreadable") + XCTAssertFalse(trace.pressed, "back answered from the swipe path") + } + + /// A hung query before the tap stays visible to XCTest, as `record(_:)` requires, and back sends + /// nothing: neither the button tap nor the swipe follows a hung query. + @MainActor + func testInAppBackWithAHungQueryInTheButtonLookupSendsNothing() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + defer { navigationBarBackReadForTesting = nil } + + let outcome = try expectingHungQueryRecorded { + navigationBarBackReadForTesting = { $0 == .lookup ? Self.hungQueryIssue : nil } + return try performInAppBack(app: app) + } + + guard case .refused(let payload) = outcome else { return XCTFail("a hung lookup must send nothing") } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNotDispatched) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.found, "hung") + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, false) + navigationBarBackReadForTesting = nil + XCTAssertTrue(app.navigationBars["Detail"].exists, "nothing popped the detail screen") + } + + /// A hung query after a delivered tap stays visible to XCTest, and back answers unverified at once + /// instead of querying the hung tree again. + @MainActor + func testInAppBackWithAHungQueryInTheConfirmationAnswersUnverified() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + defer { navigationBarBackReadForTesting = nil } + + let outcome = try expectingHungQueryRecorded { + navigationBarBackReadForTesting = { $0 == .confirmation ? Self.hungQueryIssue : nil } + return try performInAppBack(app: app) + } + + guard case .unconfirmed(let payload) = outcome else { return XCTFail("a hung confirmation must not read as performed") } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.confirmed, "hung") + } + + /// Two hittable system back buttons, one per stack, are ambiguous: back presses neither and + /// answers from the swipe path. + @MainActor + func testInAppBackPressesNoneOfTwoNavigationBarBackButtons() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--two-stacks"]) + defer { endBackFixture() } + XCTAssertEqual(app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).count, 2) + + _ = try performInAppBack(app: app) + + let trace = try XCTUnwrap(lastNavigationBarBackTraceForTesting) + XCTAssertEqual(trace.found, "ambiguous") + XCTAssertFalse(trace.pressed, "back answered from the swipe path") + } + + /// A penalized XCTest snapshot channel is not read: back skips the button lookup and swipes. + @MainActor + func testInAppBackSkipsTheButtonLookupOnAPenalizedChannel() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + penalizeSnapshotXCTestChannel(bundleId: mainOwned.bundleId, reason: "navigation-test") + defer { clearSnapshotXCTestChannelPenalty(reason: "navigation-test") } + + _ = try performInAppBack(app: app) + + let trace = try XCTUnwrap(lastNavigationBarBackTraceForTesting) + XCTAssertEqual(trace.found, "skipped") + XCTAssertFalse(trace.pressed, "back answered from the swipe path") + XCTAssertEqual(trace.lookupMs, 0) + } + + /// An XCTest issue in every read after a delivered tap proves nothing: back answers unverified and + /// never swipes after the tap. An escaped issue would fail this test. + @MainActor + func testInAppBackWithAnIssueInTheConfirmationAnswersUnverified() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + navigationBarBackReadForTesting = { $0 == .lookup ? nil : Self.vanishedBackButtonIssue } + defer { navigationBarBackReadForTesting = nil } + + let response = try execute(command: try runnerCommandFixture( + #"{"command":"backInApp","commandId":"back-confirmation-issue","appBundleId":"com.callstack.agentdevice.runner"}"# + )) + + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + navigationBarBackReadForTesting = nil + let popped = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "Back pressed"), object: status) + XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed, "the tap itself was delivered") + } + + /// A slow host can spend the whole confirmation budget on reads of the bar that is vanishing mid-pop, + /// each one an issue that proves nothing. Every read that starts in the budget fails here, so only + /// the read after the budget, which sees the settled screen, can confirm the pop. + @MainActor + func testInAppBackConfirmsAPopWhoseReadsInTheBudgetAllFail() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar"]) + defer { endBackFixture() } + var budgetReads = 0 + var finalReads = 0 + navigationBarBackReadForTesting = { phase in + switch phase { + case .lookup: return nil + case .confirmation: + budgetReads += 1 + return Self.vanishedBackButtonIssue + case .finalConfirmation: + finalReads += 1 + return nil + } + } + defer { navigationBarBackReadForTesting = nil } + + guard case .performed = try performInAppBack(app: app) else { + return XCTFail("a pop read after the budget must read as performed") + } + XCTAssertGreaterThan(budgetReads, 0, "the reads in the budget ran and failed") + XCTAssertEqual(finalReads, 1, "the one read after the budget confirmed the pop") + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.confirmed, "yes") + } + + /// A bar button titled "Back" is not the system back button, so back never presses it: it swipes, + /// and the swipe that changes nothing reports so. + @MainActor + func testInAppBackNeverPressesABarButtonOnlyTitledBack() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--custom-back-item"]) + defer { endBackFixture() } + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a bar button titled Back must not count as the back button") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNoChange) + XCTAssertEqual(status.label, "Navigation root") + } + @MainActor func testInAppBackDoesNotSwipeOffTheLitPanel() throws { let status = try launchBackFixture(["--agent-device-back-regression"]) @@ -266,6 +489,38 @@ extension RunnerTests { XCTAssertTrue(status.waitForFixtureExistence(timeout: appExistenceTimeout)) return status } + static let vanishedBackButtonIssue = "Back button vanished during the navigation bar read" + static let hungQueryIssue = "Timed out while evaluating UI query." + + /// Runs `body` and requires the hung query it records to reach XCTest exactly once. + @MainActor + private func expectingHungQueryRecorded(_ body: () throws -> T) throws -> T { + let options = XCTExpectedFailure.Options() + var observed = 0 + options.issueMatcher = { issue in + guard issue.compactDescription == Self.hungQueryIssue else { return false } + observed += 1 + return true + } + var result: Result? + XCTExpectFailure("The hung query is the condition under test", options: options) { + result = Result { try body() } + } + XCTAssertEqual(observed, 1, "a hung query reaches XCTest once, and no query follows it") + return try XCTUnwrap(result).get() + } + + @MainActor + private func launchBackNavigationFixture(_ arguments: [String]) throws -> XCUIElement { + app.launchArguments = arguments + app.launch() + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + clearSnapshotXCTestChannelPenalty(reason: "navigation-test") + XCTAssertTrue(app.navigationBars.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) + return app.staticTexts["agent-device-back-navigation-status"] + } + @MainActor private func endBackFixture() { invalidateCachedTarget(reason: "unit_test_cleanup") @@ -439,14 +694,65 @@ extension RunnerTests { XCTAssertGreaterThanOrEqual(slow, Self.backSwipeDurationMs + Self.backSwipeVerificationDelay * 1000 + 600) } - /// A screen that moved between settling and the swipe has no baseline to credit a change against. - func testBackSwipeBaselineHoldsOnlyWhenSettledAndUnmovedBeforeTheSwipe() { - let baseline = grid(100) - let moved = grid(255) +#if os(iOS) + /// #3899: a placed iPad window's snapshot frames start at the window's own origin, while the live + /// button frame is in screen points. The still bar holds the live button only through the window's + /// screen origin. + func testNavigationBarBackStateHoldsTheLiveButtonOfAPlacedWindow() { + let snapshotButton = CGRect(x: 8, y: 54, width: 44, height: 44) + let state = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [snapshotButton]) + let placed = SurfaceGeometry( + windowOnScreen: SurfaceRect(CGRect(x: 224, y: 128, width: 386, height: 954)), + panelNativeSize: nil, + orientation: RunnerInterfaceOrientation.portrait + ) + let liveButton = CGRect(x: 232, y: 182, width: 44, height: 44) + + XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: liveButton, geometry: placed)) + XCTAssertFalse(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: placed)) + + let fullScreen = SurfaceGeometry( + windowOnScreen: SurfaceRect(CGRect(x: 0, y: 0, width: 402, height: 874)), + panelNativeSize: nil, + orientation: RunnerInterfaceOrientation.portrait + ) + XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: fullScreen)) + } + + /// After the tap, only a change of the bar count, the back button count, or a back button label + /// shows a pop. A back button that only moved, such as by a large-title collapse, shows none, but + /// it is never a still bar before the tap. + func testNavigationBarBackPoppedOnlyOnABarOrBackButtonChange() { + let button = CGRect(x: 8, y: 54, width: 44, height: 44) + let detail = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button]) + + let moved = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button.offsetBy(dx: 0, dy: 30)]) + XCTAssertFalse(Self.navigationBarBackPopped(from: detail, to: moved)) + XCTAssertNotEqual(moved, detail) + let relabeled = NavigationBarBackState(backButtonLabels: [["Settings"]], backButtonFrames: [button]) + XCTAssertTrue(Self.navigationBarBackPopped(from: detail, to: relabeled)) + let atRoot = NavigationBarBackState(backButtonLabels: [[]], backButtonFrames: []) + XCTAssertTrue(Self.navigationBarBackPopped(from: detail, to: atRoot)) + let barGone = NavigationBarBackState(backButtonLabels: [], backButtonFrames: []) + XCTAssertTrue(Self.navigationBarBackPopped(from: atRoot, to: barGone)) + } +#endif - XCTAssertTrue(Self.backSwipeBaselineHolds(settled: true, baseline: baseline, justBefore: baseline)) - XCTAssertFalse(Self.backSwipeBaselineHolds(settled: true, baseline: baseline, justBefore: moved)) - XCTAssertFalse(Self.backSwipeBaselineHolds(settled: false, baseline: baseline, justBefore: baseline)) + /// A still screen settles before the budget runs out at any capture time: the settle loop's + /// timeline, replayed with fixed captures, reaches the required still span first. On a simulator + /// the base budget holds; a 1 s capture extends it. + func testBackSwipeSettleBudgetLetsAStillScreenSettleOnASlowHost() { + XCTAssertEqual(Self.backSwipeSettleBudgetMs(maxCaptureMs: 100), Self.backSwipeSettleBaseBudgetMs) + for captureMs in stride(from: 50.0, through: 2000.0, by: 50.0) { + let required = Self.backSwipeRequiredStillMs(maxCaptureMs: captureMs) + let budget = Self.backSwipeSettleBudgetMs(maxCaptureMs: captureMs) + var now = captureMs + let runStart = now + while now - runStart < required { + XCTAssertLessThan(now, budget, "capture \(captureMs) ms ran out of budget before settling") + now += Self.backSwipeSettleSampleInterval * 1000 + captureMs + } + } } func testBackSwipeOutcomeNamesNoChangeAndAnUnsampledScreenApart() { diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 477c72dc3b..d88201deee 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -134,42 +134,90 @@ than silently applied: { "command": "backInApp", "appBundleId": "com.example.app" } ``` -`backInApp` is the only back command. On iOS the runner answers it with the left-edge back swipe and -verifies a change of the app's screen. Its failure codes are: +`backInApp` is the only back command. On iOS the runner first reads the current tree, in the same +call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a +navigation bar. It taps that button with a synthesized tap only when exactly one live, hittable +candidate is found in a readable tree, and two reads in a row of the bar state, with a 0.05 s pause +between them, agree and contain that button's frame; each round re-reads the buttons. The match +places the snapshot frames at the window's screen origin, because a placed iPad window reports them +from its own top-left. No other button +counts, whatever its label. +The bar state is the labels of each navigation bar's `BackButton` children, one list per bar in +bar order, and the frames of those buttons; one XCTest snapshot per bar. The still check before the +tap compares labels and frames. Bar titles and identifiers are left out, because a bar can retitle +itself (a countdown) without navigating. After the tap it reads that state again with a 0.05 s +pause between reads. Only these changes from the state read before the tap answer `ok`: the number +of bars, the number of `BackButton` children (such as none at the root), or a `BackButton` label. +Frames are left out after the tap, because a tap that navigates nowhere can still move the button +(a large-title collapse, a bar button that appears). A pop that keeps the same `BackButton` labels +and bar count, such as one between two screens whose back buttons have the same title, answers +`BACK_SWIPE_UNVERIFIED`; so can a pop on a +back button whose title updates itself, misread as a navigation. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is +never followed by the swipe. Before and after the tap, reads start for up to 1 s each, checked +before each bar's snapshot; a query in flight finishes or ends at XCTest's own query timeout. +After the tap, when no read in the 1 s saw a change or hung, the runner starts one more read after +the 1 s, with no deadline of its own: a snapshot of a bar that vanished mid-pop can spend the 1 s in +XCTest's retries, and this read can still see the screen after the pop. That read can also fail, +which answers `BACK_SWIPE_UNVERIFIED`, or hang, which ends at XCTest's own query timeout and also +answers `BACK_SWIPE_UNVERIFIED`. A read in flight can run well past the 1 s: one bar snapshot took +about 2 s on a CI simulator. + +Every bar read holds its XCTest issues. A held issue means the read proved nothing. A hung query +("Timed out while evaluating UI query") is recorded again, and no further query follows it: before +the tap the runner answers `BACK_SWIPE_NOT_DISPATCHED` with nothing sent, and after it +`BACK_SWIPE_UNVERIFIED`. + +The runner swipes instead when no tap was sent: + +- The tree read is skipped: the window resolution marked the tree unreadable, accessibility is + unavailable, or the XCTest snapshot channel is penalized for the app. +- The read fails or XCTest rejects one of its queries. +- No candidate is found, or two or more are (such as a sheet over a stack). +- The bar state does not hold still within 1 s. +- The synthesized tap is refused before anything is sent. + +A tap whose outcome is unknown after it may have been sent throws, and neither the bar check nor +the swipe follows. The swipe verifies a change of the app's screen. When XCTest cannot resolve the +app's window, it samples the sole screen only on an iPhone, or on an iPad while the app provably +fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure codes are: | Code | When | Sent | | --- | --- | --- | | `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, `APP_SCREEN_OFF_LIT_PANEL` | The screen could not be sampled before the swipe. | Nothing | -| `BACK_SWIPE_NOT_DISPATCHED` | The swipe was refused before any touch went down, or the platform has no back gesture (visionOS; the daemon refuses it first). | Nothing | +| `BACK_SWIPE_NOT_DISPATCHED` | The swipe was refused before any touch went down, an XCTest query on the navigation bar hung before the tap, or the platform has no back command (visionOS; the daemon refuses it first). | Nothing | | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | -| `BACK_SWIPE_UNVERIFIED` | The swipe may have run; the outcome is unknown. | Possibly the swipe | +| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or on one more read after it or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | -The settle window is a run of samples taken 0.25 s apart that agree for as long as the gap the -swipe will be credited across: a contact allowance, one 0.25 s delay, and the slowest capture seen -while settling. A slow host therefore settles for longer. The swipe is then resolved up to its -contact, and one more sample, taken right before the swipe, must still match the run: +The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a +0.05 s pause between captures plus each capture's own time, that agree for as long as the gap the swipe will be credited across: a contact +allowance, one 0.1 s delay, and the slowest capture seen while settling. A slow host therefore +settles for longer. The last sample of that run is the one the change is measured from, and the +swipe follows it at once: -| Platform | Resolved before the last sample | Left in the gap | Contact allowance | +| Platform | Resolved before settling | Left in the gap | Contact allowance | | --- | --- | --- | --- | | Simulator | the edge-touch plan, its display, and the HID touchscreen sender | the 100 ms edge contact | 0.2 s | | Device | the synthesized fling plan (2 pt inside the leading edge to 0.9 of the width, no hold), with every tree and window read it needs | the 200 ms fling | 0.4 s | -On both, every tree and window read happens during that preparation, before the just-before sample. +On both, every tree and window read happens during that preparation, before the first sample. -Settling takes at most 8 samples, so the wait before the swipe is at most 1.75 s of delays plus -eight captures, then the preparation and the last sample: about 2.6 s with the 0.1 s captures of a -local simulator, and about 5 s with the 0.4 s captures measured on one physical iPhone (iPhone 17 Pro, -iOS 27.0.1; https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541). Usually a still screen -settles after 3 samples. +Settling samples for at most 2 s, plus the capture in flight. On a slow host the budget grows to the first capture, the still span, and one more pause and capture, so a still screen settles instead of always answering `BACK_SWIPE_UNVERIFIED`. A still screen usually settles after +4 samples, about 0.45 s on a local simulator with 0.1 s captures; with the 0.4 s captures measured +on one physical iPhone (iPhone 17 Pro, iOS 27.0.1; +https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541) the still span is 0.9 s. -Only a change from the last sample to the sample after the swipe counts, and only when the time +Only a change from the last sample to the sample taken 0.1 s after the swipe counts, and only when the time between those two does not exceed the time the screen had been still; otherwise the result is `BACK_SWIPE_UNVERIFIED`. Then the swipe may or may not have navigated: run `snapshot` to see the current screen before acting again. A screen that animates on its own more slowly than the settle window can still be misread as a back navigation; any finite window has that limit, so also run `snapshot` after `back` on a screen with auto-advancing content. -Each iOS back logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its +Each iOS back whose window context resolved logs one `AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON` +line on every exit of the button path, a thrown tap included, with `found` (`skipped`, +`unreadable`, `hung`, `moving`, `no`, `ambiguous`, or `yes`), `pressed`, `confirmed` (`yes`, `no`, +`hung`, or `-` before a tap), `lookupMs` (`0` when the read was +skipped), and `confirmMs`. Each swipe logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its `outcome` (`performed`, `refused`, `unconfirmed`, or `error` for a thrown failure), `code`, `changed`, `stableMs`, `justBeforeToAfterMs`, `gestureMs`, `maxCaptureMs`, `settleSamples`, and `settled`. diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index e03b62befb..284edc3106 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -264,7 +264,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ setValue: 'in-app', usageLabel: '--in-app', usageDescription: - 'Back: use app-provided back UI when available; on iOS, the verified left-edge back swipe', + 'Back: use app-provided back UI when available; on iOS, the one hittable navigation bar back button, confirmed by a bar change, else the verified left-edge back swipe', projectConfig: true, recorded: false, }, @@ -276,7 +276,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ setValue: 'system', usageLabel: '--system', usageDescription: - 'Back: use system back input or gesture when available; on iOS, the same left-edge back swipe as --in-app', + 'Back: use system back input or gesture when available; on iOS, the same back as --in-app', projectConfig: true, recorded: false, }, diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 39594afc8e..293b2fe6de 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes swipe from the left edge and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref.'; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index f12218945c..56c8eeb510 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -148,7 +148,8 @@ export function createAppleInteractor( dispatched: 'no', }); } - // iOS has one back: the verified left-edge swipe, whichever mode was asked. + // iOS has one back, whichever mode was asked: the navigation bar back button, else the verified + // left-edge swipe. await runAppleRunnerCommand( device, { command: 'backInApp', appBundleId: runnerContext.appBundleId }, diff --git a/packages/platform-apple/src/navigation/runtime.ts b/packages/platform-apple/src/navigation/runtime.ts index 9d30160747..08740751e7 100644 --- a/packages/platform-apple/src/navigation/runtime.ts +++ b/packages/platform-apple/src/navigation/runtime.ts @@ -33,10 +33,10 @@ const backOsUnavailable = Object.freeze({ const backVisionOsUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', - hint: 'iOS back is a left-edge swipe; visionOS has none. Press the visible back control by its ref.', + hint: 'visionOS has no back command. Press the visible back control by its ref.', } as const); -/** tvOS presses the remote's Menu button, macOS taps the window back control, and iOS swipes from - * the left edge. watchOS and visionOS have no back. */ +/** tvOS presses the remote's Menu button, macOS taps the window back control, and iOS presses the + * navigation bar back button or swipes from the left edge. watchOS and visionOS have no back. */ function appleBackFact(device: DeviceInfo): RuntimeOperationFact { if (device.kind !== 'simulator' && device.kind !== 'device') return backKindUnavailable; const os = resolveDeviceAppleOs(device); diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 49f720b1d1..146ab7a220 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -528,7 +528,7 @@ function expectNavigationAndKeyboardFacts( if (device.appleOs === 'visionos') { expect(binding.facts.operations.back).toMatchObject({ reason: 'unsupported-platform-leaf', - hint: expect.stringContaining('visionOS has none'), + hint: expect.stringContaining('visionOS has no back command'), }); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d68250fd82..a93e539340 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,14 +124,14 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. It succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.25 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing: up to 8 samples 0.25 s apart, which is 1.75 s plus eight screen captures: about 2.6 s on a simulator, and about 5 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. -- `reason: back_swipe_unverified`: the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. -- `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, so nothing reached the device. +- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. +- `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. - If the app's screen cannot be captured before the swipe, `back` does not swipe: it refuses with `COMMAND_FAILED` and `details.runnerErrorCode` set to `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, or `APP_SCREEN_OFF_LIT_PANEL`. Nothing is sent. -- A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. -- An app with a left-edge drawer may open the drawer instead of going back. When a back or close control is visible, press it by its ref. -- `back --system` sends system back input: the Android back keyevent, the iOS back swipe, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. +- A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back command and refuses before dispatch with `UNSUPPORTED_OPERATION`. +- An app with a left-edge drawer may open the drawer instead of going back. When a custom back or close control is visible, press it by its ref. +- `back --system` sends system back input: the Android back keyevent, the same iOS back as the default, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. - `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. In both default and scoped simulator sets, an iPhone Duo classified with more than one integrated panel refuses `portrait-upside-down` and rotates through the fold helper. It succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. Unresolved guest reads are retried inside the settle loop; an unresolved final read reports `orientation-panel-table-unavailable`. The picker offsets were measured on iPhone Duo; a new multi-panel model needs its own mapping. A simulator classified with one panel, or whose first classification read is unresolved, uses the standard runner rotation. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`. - `action-button` presses the iPhone or iPad Action Button once. It takes no arguments, no hold duration, and no `--settle`, and it doesn't activate the session app or drive Camera Control. Other platforms refuse it, and a model without the button (an iPhone SE, most iPad simulators) fails with `UNSUPPORTED_OPERATION`. @@ -169,7 +169,7 @@ Remote daemons and tenants: ```bash agent-device open "https://example.com" --platform ios # open link in web browser agent-device open MyApp "myapp://screen/to" --platform ios # open deep link to MyApp -agent-device back --platform ios # left-edge back swipe, verified by a screen change +agent-device back --platform ios # navigation bar back button, else a verified left-edge swipe agent-device reinstall MyApp /path/to/app-debug.apk --platform android --serial emulator-5554 agent-device open com.example.myapp --platform android --serial emulator-5554 --session my-session --relaunch agent-device metro reload