From 47ff6dbad735df57660eceadcab0318e9dc80286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 15 Nov 2026 09:52:12 +0100 Subject: [PATCH] fix(ios): find the navigation bar back button outside the bar's subtree On the iPhone Duo the system back button floats below the navigation bar, in another subtree, so the bar-scoped lookup found nothing and back swiped. The lookup reads BackButton in the app's windows and presses it only when it is by a navigation bar: inside one, or, when none is, below one within a third of the window height. Rounds snapshot the candidates until still; the bar count and, for an unlabeled button, the bar titles must agree with a live read before the tap, and a button with no label under bars with no title is not pressed. A covered button is never read as a pop. Co-Authored-By: Claude Sonnet 5.5 --- .../AgentDeviceRunnerAppFixtures.m | 80 ++++- .../RunnerTests+Navigation.swift | 281 ++++++++++++++---- .../RunnerTests+NavigationTests.swift | 153 +++++++++- apple/runner/RUNNER_PROTOCOL.md | 68 +++-- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 5 +- 6 files changed, 484 insertions(+), 105 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 91b9002f0f..f6df76a93c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -43,13 +43,19 @@ @interface AgentDeviceRunnerFixtureViewController : AgentDeviceRunnerViewControl // so the bar shows UIKit's system back button; the interactive pop is off, so only that button can // pop it. `--blocked-back` keeps that button but replaces its action, so pressing it navigates // nowhere. `--shifting-back` makes that press move the bar down instead, so the back button moves -// without any navigation. `--ticking-title` counts the detail screen's title down every 0.2 s, as a timer does, +// without any navigation. `--ticking-title` counts the detail screen's title down every 0.2 s, as a timer does (`--fast-ticking-title`: every 0.02 s), // so its bar changes without any navigation. `--custom-back-item` instead puts a bar button titled -// "Back" on the root, which is not the system back button and must never be pressed. The root's -// label reports what happened. +// "Back" on the root, which is not the system back button and must never be pressed. +// `--floating-back` hides the bar's back button and floats an unlabeled one in the screen's content +// beside the bar, as the iPhone Duo does; `--hidden-bar` also hides the bar, so that button is not +// a navigation bar's. `--content-back-button` adds an in-content button identified `BackButton` at the +// bottom of the detail screen, which is not the system back button; `--moving-content-back` also moves +// it every 0.05 s; `--top-content-back` puts it just below the bar instead. The root's label reports +// what happened. `--alert-back` makes the back press present an alert over the screen instead of popping. @interface AgentDeviceBackNavigationFixture : UINavigationController @property(nonatomic, strong) UILabel *status; @property(nonatomic, strong) NSTimer *titleTimer; +@property(nonatomic, strong) NSTimer *contentTimer; @end @implementation AgentDeviceBackNavigationFixture @@ -87,10 +93,74 @@ - (instancetype)initWithCustomBackItem:(BOOL)customBackItem blockedBack:(BOOL)bl }]; } } - if ([NSProcessInfo.processInfo.arguments containsObject:@"--ticking-title"]) { + if ([NSProcessInfo.processInfo.arguments containsObject:@"--floating-back"]) { + detail.navigationItem.hidesBackButton = YES; + UIButton *floating = [UIButton buttonWithType:UIButtonTypeSystem]; + floating.accessibilityIdentifier = @"BackButton"; + floating.accessibilityLabel = @""; + [floating setImage:[UIImage systemImageNamed:@"chevron.left"] forState:UIControlStateNormal]; + floating.translatesAutoresizingMaskIntoConstraints = NO; + __weak UINavigationController *weakSelf = self; + [floating addAction:[UIAction actionWithHandler:^(__kindof UIAction *action) { + [weakSelf popViewControllerAnimated:NO]; + }] forControlEvents:UIControlEventTouchUpInside]; + [detail.view addSubview:floating]; + [NSLayoutConstraint activateConstraints:@[ + [floating.topAnchor constraintEqualToAnchor:detail.view.safeAreaLayoutGuide.topAnchor constant:8], + [floating.trailingAnchor constraintEqualToAnchor:detail.view.trailingAnchor constant:-16], + [floating.widthAnchor constraintEqualToConstant:44], + [floating.heightAnchor constraintEqualToConstant:44], + ]]; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--hidden-bar"]) { + [self setNavigationBarHidden:YES animated:NO]; + } + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--alert-back"]) { + if (@available(iOS 16.0, *)) { + __weak UIViewController *weakDetail = detail; + detail.navigationItem.backAction = [UIAction actionWithHandler:^(__kindof UIAction *action) { + UIAlertController *alert = [UIAlertController alertControllerWithTitle:@"Discard?" message:nil + preferredStyle:UIAlertControllerStyleAlert]; + [alert addAction:[UIAlertAction actionWithTitle:@"Stay" style:UIAlertActionStyleCancel handler:nil]]; + [weakDetail presentViewController:alert animated:NO completion:nil]; + }]; + } + } + BOOL movingContent = [NSProcessInfo.processInfo.arguments containsObject:@"--moving-content-back"]; + BOOL topContent = [NSProcessInfo.processInfo.arguments containsObject:@"--top-content-back"]; + if (movingContent || topContent || [NSProcessInfo.processInfo.arguments containsObject:@"--content-back-button"]) { + UIButton *content = [UIButton buttonWithType:UIButtonTypeSystem]; + content.accessibilityIdentifier = @"BackButton"; + [content setTitle:@"Content" forState:UIControlStateNormal]; + content.translatesAutoresizingMaskIntoConstraints = NO; + [detail.view addSubview:content]; + // The constraint on the edge the button hangs from: the top one under `--top-content-back`, else the bottom. + CGFloat rest = topContent ? 16 : -16; + CGFloat moved = topContent ? 60 : -60; + NSLayoutConstraint *edge = topContent + ? [content.topAnchor constraintEqualToAnchor:detail.view.safeAreaLayoutGuide.topAnchor constant:rest] + : [content.bottomAnchor constraintEqualToAnchor:detail.view.safeAreaLayoutGuide.bottomAnchor constant:rest]; + [NSLayoutConstraint activateConstraints:@[ + edge, + [content.centerXAnchor constraintEqualToAnchor:detail.view.centerXAnchor], + [content.widthAnchor constraintEqualToConstant:120], + [content.heightAnchor constraintEqualToConstant:44], + ]]; + if (movingContent) { + __block BOOL up = NO; + __weak UIViewController *weakDetail = detail; + self.contentTimer = [NSTimer scheduledTimerWithTimeInterval:0.05 repeats:YES block:^(NSTimer *timer) { + up = !up; + edge.constant = up ? moved : rest; + [weakDetail.view layoutIfNeeded]; + }]; + } + } + BOOL fastTicking = [NSProcessInfo.processInfo.arguments containsObject:@"--fast-ticking-title"]; + if (fastTicking || [NSProcessInfo.processInfo.arguments containsObject:@"--ticking-title"]) { __block NSInteger remaining = 59; __weak UIViewController *weakDetail = detail; - self.titleTimer = [NSTimer scheduledTimerWithTimeInterval:0.2 repeats:YES block:^(NSTimer *timer) { + self.titleTimer = [NSTimer scheduledTimerWithTimeInterval:(fastTicking ? 0.02 : 0.2) repeats:YES block:^(NSTimer *timer) { remaining = remaining > 0 ? remaining - 1 : 59; weakDetail.title = [NSString stringWithFormat:@"0:%02ld", (long)remaining]; }]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 01d098d2cb..1e4b4397fb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -3,6 +3,8 @@ import XCTest extension RunnerTests { /// UIKit's identifier for a navigation bar's system back button, in UIKit and SwiftUI stacks alike. + /// The button is not always inside the bar's subtree: a floating one, such as the detail column's on + /// the iPhone Duo, is a sibling of the bar. It is therefore read from the app's windows. static let navigationBarBackButtonIdentifier = "BackButton" /// The wait from the end of the swipe's contact to the sample it is compared against. An /// interactive pop follows the contact across most of the width, so the screen has already moved. @@ -191,8 +193,10 @@ extension RunnerTests { } } - /// Presses the navigation bar's system back button when exactly one hittable one is on screen, - /// and confirms that the bar changed. It is read only where the tree answers queries: on a tree + /// Presses the system back button when exactly one hittable one is on screen and the app shows a + /// navigation bar, and confirms that the state changed. The button is read from the app's windows: + /// a floating one, such as the iPhone Duo's, is not inside the bar. One with no label under bars + /// with no title could not be confirmed, so it is not pressed. It is read only where the tree answers queries: on a tree /// XCTest cannot serialize, the query would grind through XCTest's retries, so back swipes there. /// Two candidates, such as a sheet over a stack, are ambiguous and fall back to the swipe too, as /// does a bar still moving after the read budget. Nil when nothing was sent, so the swipe is still @@ -248,6 +252,7 @@ extension RunnerTests { let confirmation = navigationBarBackConfirmation( app: app, from: before, + windowHeight: context.geometry.windowOnScreen.rect.height, deadlineMs: tapped + Self.navigationBarBackReadBudgetMs ) trace.confirmMs = currentUptimeMs() - tapped @@ -263,7 +268,7 @@ extension RunnerTests { struct NavigationBarBackTrace { /// `skipped` when the tree was not read, `unreadable` when a read failed, `hung` when an XCTest /// query hung, `moving` when the bar did not hold still within the read budget, `no`, - /// `ambiguous`, or `yes`. + /// `ambiguous`, `unprovable` (a back button with no label under bars with no title), or `yes`. var found = "skipped" var pressed = false /// `yes`, `no`, or `hung`; nil until a tap was delivered. @@ -283,20 +288,43 @@ extension RunnerTests { } } - /// The labels of each navigation bar's system back buttons, one list per bar in bar order, and - /// the frames of those buttons. Both must agree for a still bar before the tap. Bar titles and - /// identifiers are left out: a bar can retitle itself, such as a countdown, without any navigation. + /// The number of navigation bars, the labels of the system back buttons in tree order, and the + /// frames of those buttons. The buttons' labels and frames must agree for a still bar before the tap. + /// Bar titles are part of the state only for a back button with no label, such as the iPhone Duo's + /// floating one, which shows no pop on its own: the state then holds each bar's title (its + /// identifier, which UIKit sets to the title, or its label when the identifier is empty), the titles + /// must agree before the tap, and a title change after it shows a pop. For a labeled button titles + /// are left out, because a bar can retitle itself, such as a countdown, without any navigation. The + /// state read decides this itself, so the state before the tap and each one after it follow one rule. + /// Limit: on the Duo no structural signal shows every pop (closed, the button and one bar stay after + /// a pop). So under an unlabeled button a bar that retitles itself can be misread as a pop: one that + /// retitles more slowly than the still check samples passes it, and after the tap a single changed + /// read counts as a pop for the whole confirmation window. struct NavigationBarBackState: Equatable { - let backButtonLabels: [[String]] + var barCount: Int + let backButtonLabels: [String] let backButtonFrames: [CGRect] + var barTitles: [String] = [] + + var hasUnlabeledBackButton: Bool { backButtonLabels.contains(where: \.isEmpty) } + + func hasSameButtons(as other: NavigationBarBackState) -> Bool { + backButtonLabels == other.backButtonLabels && backButtonFrames == other.backButtonFrames + } + + /// Whether a pop can show in a later read of this rule: every back button has a label, or a bar + /// has a title. Without it a tap could not be confirmed, so it is not sent. + var canShowAPop: Bool { !hasUnlabeledBackButton || barTitles.contains { !$0.isEmpty } } } - /// Whether the bar after the tap shows a pop: the bar count, the back button count, or a back - /// button label changed. Frames are left out: a tap that navigates nowhere can still move the - /// button, such as by a large-title collapse or a bar button that appears. A pop that keeps the - /// same back button labels and bar count answers unverified. + /// Whether the bar after the tap shows a pop: the bar count, the back button count, a back + /// button label, or, for a back button with no label, a bar title changed. Frames are left out: a + /// tap that navigates nowhere can still move the button, such as by a large-title collapse or a bar button that appears. A pop that keeps the + /// same back button labels and bar count answers unverified, except under an unlabeled button, + /// where a title change shows it. static func navigationBarBackPopped(from before: NavigationBarBackState, to after: NavigationBarBackState) -> Bool { - after.backButtonLabels != before.backButtonLabels + after.barCount != before.barCount || after.backButtonLabels != before.backButtonLabels + || after.barTitles != before.barTitles } /// The pause between two reads of the bar, and how long reads may start, both before the tap @@ -331,33 +359,52 @@ extension RunnerTests { case hung } - /// Reads the hittable back buttons, then the bar state until two reads in a row, one read interval - /// apart, agree and the button's frame is in them, so a push still animating is not read - /// as the state a pop changes. Each round re-reads the buttons. Reads start only until `deadlineMs`. + /// Reads the back buttons by a navigation bar, one snapshot per button, until two reads in a row, one + /// read interval apart, agree, so a push still animating is not read as the state a pop changes. A + /// button that is not by a bar is no candidate: it is neither counted nor compared. The bars (their + /// frames, count and, under an unlabeled candidate, titles) are read by the first round and then by + /// the live read, which checks that exactly one candidate is hittable and that its frame is in the + /// state; the two bar reads must agree too, so a title that changes on its own is not still. + /// Rounds start only until `deadlineMs`; each live read has a read budget of its own. @MainActor private func stillNavigationBarBackButton( app: XCUIApplication, geometry: SurfaceGeometry, deadlineMs: Double ) -> NavigationBarBackLookup { + let windowHeight = geometry.windowOnScreen.rect.height var previous: NavigationBarBackState? + var barsRead: (state: NavigationBarBackState, bars: [NavigationBarSnapshot])? while true { - let button: CGRect - switch navigationBarBackButtonFrames(app: app, deadlineMs: deadlineMs) { + let sample: NavigationBarBackSample + switch readNavigationBarBack( + app: app, phase: .lookup, deadlineMs: deadlineMs, windowHeight: windowHeight, knownBars: barsRead?.bars + ) { case .hung: return .hung case .unreadable: return .notFound("unreadable") - case .read(let frames) where frames.isEmpty: return .notFound("no") - case .read(let frames) where frames.count > 1: return .notFound("ambiguous") - case .read(let frames): button = frames[0] + case .read(let read): sample = read } - let current: NavigationBarBackState - switch readNavigationBarBackState(app: app, phase: .lookup, deadlineMs: deadlineMs) { - case .hung: return .hung - case .unreadable: return .notFound("unreadable") - case .read(let state): current = state + let candidates = sample.candidates(by: sample.bars) + if candidates.isEmpty { return .notFound("no") } + let current = sample.state(of: candidates, bars: nil) + let buttonsStill = previous.map { current.hasSameButtons(as: $0) } ?? false + if sample.barsWereRead { + barsRead = (sample.state(of: candidates, bars: sample.bars), sample.bars) + } else if !buttonsStill { + barsRead = nil } - if current == previous, Self.navigationBarBackState(current, holdsButton: button, geometry: geometry) { - return .still(current, button: button) + if buttonsStill, let barsBefore = barsRead?.state { + // The live read is one bounded read of its own: it must not inherit what the rounds spent. + let liveDeadlineMs = currentUptimeMs() + Self.navigationBarBackReadBudgetMs + switch liveNavigationBarBackButton(of: sample, geometry: geometry, deadlineMs: liveDeadlineMs) { + case .hung: return .hung + case .unreadable: return .notFound("unreadable") + case .read(.absent(let found)): return .notFound(found) + case .read(.found(let state, let button, let bars)): + if state == barsBefore { return .still(state, button: button) } + barsRead = (state, bars) + case .read(.moving): break + } } if currentUptimeMs() >= deadlineMs { return .notFound("moving") } previous = current @@ -365,6 +412,37 @@ extension RunnerTests { } } + /// What the live read of a still state found. + enum NavigationBarBackLive { + case found(NavigationBarBackState, CGRect, [NavigationBarSnapshot]) + case absent(String) + case moving + } + + /// One live read of the sampled buttons: the bars, the candidates by them, and which candidates are + /// hittable. A pop that the candidates could not show is not tapped. + @MainActor + private func liveNavigationBarBackButton( + of sample: NavigationBarBackSample, + geometry: SurfaceGeometry, + deadlineMs: Double + ) -> NavigationBarBackRead { + heldNavigationBarRead(.lookup) { + let bars = sample.roots.flatMap { $0.descendants(matching: .navigationBar).allElementsBoundByIndex } + guard let snapshots = barSnapshots(bars, deadlineMs: deadlineMs) else { return nil } + let candidates = sample.candidates(by: snapshots) + let hittable = candidates.filter { sample.buttons[$0].isHittable } + if hittable.isEmpty { return .absent("no") } + if hittable.count > 1 { return .absent("ambiguous") } + let button = sample.buttons[hittable[0]].frame + guard SnapshotGeometry.isPositiveFinite(button) else { return nil } + let state = sample.state(of: candidates, bars: snapshots) + if !state.canShowAPop { return .absent("unprovable") } + return Self.navigationBarBackState(state, holdsButton: button, geometry: geometry) + ? .found(state, button, snapshots) : .moving + } + } + /// Whether `state` holds the live button. A live frame is in screen points, and a snapshot frame /// is in the window's own space, which a placed iPad window starts off the screen origin (#3899). static func navigationBarBackState( @@ -401,14 +479,21 @@ extension RunnerTests { private func navigationBarBackConfirmation( app: XCUIApplication, from before: NavigationBarBackState, + windowHeight: CGFloat, deadlineMs: Double ) -> NavigationBarBackConfirmation { while true { let isFinalRead = currentUptimeMs() >= deadlineMs let readDeadlineMs = isFinalRead ? Double.infinity : deadlineMs let phase: NavigationBarBackReadPhase = isFinalRead ? .finalConfirmation : .confirmation - switch readNavigationBarBackState(app: app, phase: phase, deadlineMs: readDeadlineMs) { - case .read(let after) where Self.navigationBarBackPopped(from: before, to: after): return .changed + switch readNavigationBarBack( + app: app, phase: phase, deadlineMs: readDeadlineMs, windowHeight: windowHeight, knownBars: nil + ) { + case .read(let after) + where Self.navigationBarBackPopped( + from: before, to: after.state(of: after.candidates(by: after.bars), bars: after.bars) + ): + return .changed case .hung: return .hung case .read, .unreadable: break } @@ -459,53 +544,119 @@ extension RunnerTests { return .read(value) } - /// The frames of the hittable system back buttons; unreadable when the tree did not answer or the - /// read budget had run out. - @MainActor - private func navigationBarBackButtonFrames(app: XCUIApplication, deadlineMs: Double) -> NavigationBarBackRead<[CGRect]> { - guard currentUptimeMs() < deadlineMs else { return .unreadable } - return heldNavigationBarRead(.lookup) { - guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } - let frames = bars - .flatMap { $0.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex } - .filter { $0.isHittable } - .map(\.frame) - return frames.allSatisfy(SnapshotGeometry.isPositiveFinite) ? frames : nil + /// One read of every `BackButton` and the bars: the buttons and their snapshots, and the bars the + /// candidates are judged by. A candidate is a button by a bar, so an in-content `BackButton` is + /// neither counted nor compared. The elements stay so that a live read does not query them again. + struct NavigationBarBackSample { + let buttons: [XCUIElement] + let labels: [String] + /// Snapshot frames, in the same space as the bars'. + let frames: [CGRect] + let roots: [XCUIElement] + let bars: [NavigationBarSnapshot] + let barsWereRead: Bool + let windowHeight: CGFloat + + func candidates(by bars: [NavigationBarSnapshot]) -> [Int] { + RunnerTests.backButtonCandidates(frames, barFrames: bars.map(\.frame), windowHeight: windowHeight) + } + + /// The candidates' state, with the bars read when given. + func state(of candidates: [Int], bars: [NavigationBarSnapshot]?) -> NavigationBarBackState { + let state = NavigationBarBackState( + barCount: 0, backButtonLabels: candidates.map { labels[$0] }, backButtonFrames: candidates.map { frames[$0] }) + return bars.map { RunnerTests.addingBars($0, to: state) } ?? state } } - /// One XCTest snapshot per bar, so a read costs a query per bar rather than one per attribute. - /// Unreadable when the tree did not answer or the read budget ran out before a bar's snapshot. + /// One walk of the app's windows, then one query for the back buttons and one snapshot per button, + /// so a read stays well inside the read budget. The bars are read too unless `knownBars` gives them: + /// the rounds that wait for a still state read them once, and the live read reads them again. + /// Unreadable when the tree did not answer or the read budget ran out before a snapshot. @MainActor - private func readNavigationBarBackState( + private func readNavigationBarBack( app: XCUIApplication, phase: NavigationBarBackReadPhase, - deadlineMs: Double - ) -> NavigationBarBackRead { - heldNavigationBarRead(phase) { - guard let bars = try? elementQueries(app: app, matching: .navigationBar) else { return nil } - var backButtonLabels: [[String]] = [] - var backButtonFrames: [CGRect] = [] - for bar in bars.flatMap({ $0.allElementsBoundByIndex }) { - guard currentUptimeMs() < deadlineMs, let snapshot = try? bar.snapshot() else { return nil } - let buttons = Self.navigationBarBackButtons(snapshot) - backButtonLabels.append(buttons.map(\.label)) - backButtonFrames.append(contentsOf: buttons.map(\.frame)) + deadlineMs: Double, + windowHeight: CGFloat, + knownBars: [NavigationBarSnapshot]? + ) -> NavigationBarBackRead { + guard currentUptimeMs() < deadlineMs else { return .unreadable } + return heldNavigationBarRead(phase) { + guard let roots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else { return nil } + let buttons = roots.flatMap { + $0.descendants(matching: .button).matching(identifier: Self.navigationBarBackButtonIdentifier).allElementsBoundByIndex + } + var labels: [String] = [] + var frames: [CGRect] = [] + for button in buttons { + guard currentUptimeMs() < deadlineMs, let snapshot = try? button.snapshot() else { return nil } + labels.append(snapshot.label) + frames.append(snapshot.frame) } - return NavigationBarBackState(backButtonLabels: backButtonLabels, backButtonFrames: backButtonFrames) + var bars = knownBars + if bars == nil { + let elements = roots.flatMap { $0.descendants(matching: .navigationBar).allElementsBoundByIndex } + bars = barSnapshots(elements, deadlineMs: deadlineMs) + } + guard let bars else { return nil } + return NavigationBarBackSample( + buttons: buttons, labels: labels, frames: frames, roots: roots, bars: bars, + barsWereRead: knownBars == nil, windowHeight: windowHeight + ) } } - static func navigationBarBackButtons(_ bar: XCUIElementSnapshot) -> [(label: String, frame: CGRect)] { - var buttons: [(label: String, frame: CGRect)] = [] - var pending = bar.children - while let element = pending.popLast() { - if element.identifier == navigationBarBackButtonIdentifier { - buttons.append((element.label, element.frame)) - } - pending.append(contentsOf: element.children) + /// Whether a back button is by a navigation bar, both frames in one coordinate space: inside it, as + /// on an iPhone, or below it within a third of the window's height from the bar's top, where the + /// iPhone Duo floats its button (43 pt below the bar when open, 93 pt when closed), horizontally + /// within the bar. An in-content button elsewhere, or in an app with no bar, is not the system back + /// button. + static func backButtonBelongsToABar(_ button: CGRect, barFrames: [CGRect], windowHeight: CGFloat) -> Bool { + barFrames.contains { bar in + button.midX >= bar.minX && button.midX <= bar.maxX + && button.minY >= bar.minY && button.minY <= bar.minY + windowHeight / 3 + } + } + + /// The back buttons by a navigation bar, in two tiers decided by geometry alone, so the state read + /// before the tap and after it never depends on what covers a button. First the buttons inside a bar, + /// which is the rule a button in a bar's subtree always met. Only when none is inside one, the + /// buttons below a bar where the iPhone Duo floats its button. A content `BackButton` near the top + /// therefore never competes with the bar's own button. Hittability only picks the tap target. + static func backButtonCandidates(_ frames: [CGRect], barFrames: [CGRect], windowHeight: CGFloat) -> [Int] { + let inside = frames.indices.filter { index in + barFrames.contains { $0.contains(CGPoint(x: frames[index].midX, y: frames[index].midY)) } } - return buttons + if !inside.isEmpty { return inside } + return frames.indices.filter { + backButtonBelongsToABar(frames[$0], barFrames: barFrames, windowHeight: windowHeight) + } + } + + struct NavigationBarSnapshot { + let frame: CGRect + /// The bar's identifier, which UIKit sets to the title, or its label when the identifier is empty. + let title: String + } + + /// One snapshot per bar. Nil when the read budget ran out before one or one failed. + func barSnapshots(_ bars: [XCUIElement], deadlineMs: Double) -> [NavigationBarSnapshot]? { + var snapshots: [NavigationBarSnapshot] = [] + for bar in bars { + guard currentUptimeMs() < deadlineMs, let snapshot = try? bar.snapshot() else { return nil } + snapshots.append( + NavigationBarSnapshot(frame: snapshot.frame, title: snapshot.identifier.isEmpty ? snapshot.label : snapshot.identifier)) + } + return snapshots + } + + /// `state` with the bars read: their count, and their titles when a back button has no label. + static func addingBars(_ bars: [NavigationBarSnapshot], to state: NavigationBarBackState) -> NavigationBarBackState { + var state = state + state.barCount = bars.count + if state.hasUnlabeledBackButton { state.barTitles = bars.map(\.title) } + return state } @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 2b63e7fc60..d31c33fefc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -77,6 +77,73 @@ extension RunnerTests { XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed) } + /// A floating back button with no label, outside the bar's subtree as on the iPhone Duo, is pressed + /// and its pop is confirmed through the bar's title. + @MainActor + func testInAppBackPressesAFloatingUnlabeledBackButtonAndConfirmsThroughTheBarTitle() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--floating-back"]) + defer { endBackFixture() } + XCTAssertEqual(app.navigationBars.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).count, 0) + + try assertBackPressesTheButtonAndPops(status, "a floating back button beside a navigation bar must be pressed and confirmed") + } + + /// A bar that retitles itself, such as a countdown, never holds still for an unlabeled back button, + /// whose pop the title alone shows: nothing is tapped. + @MainActor + func testInAppBackNeverPressesAnUnlabeledBackButtonUnderATickingBarTitle() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--floating-back", "--fast-ticking-title"]) + defer { endBackFixture() } + + _ = try performInAppBack(app: app) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, false) + XCTAssertTrue(["moving", "unreadable"].contains(lastNavigationBarBackTraceForTesting?.found ?? ""), "the lookup never holds still") + } + + /// An in-content button identified `BackButton` is no candidate: the bar's own back button is still + /// the one pressed, not counted as ambiguous. + @MainActor + func testInAppBackPressesTheBarBackButtonBesideAnInContentBackButton() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--content-back-button"]) + defer { endBackFixture() } + XCTAssertEqual(app.buttons.matching(identifier: Self.navigationBarBackButtonIdentifier).count, 2) + + try assertBackPressesTheButtonAndPops(status, "the bar's back button must be pressed beside an in-content button of the same identifier") + } + + /// An in-content `BackButton` just below the bar, in the region a floating button can take, does not + /// make the lookup ambiguous: the button inside the bar is pressed. + @MainActor + func testInAppBackPressesTheBarBackButtonBesideAnInContentBackButtonBelowTheBar() throws { + let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--top-content-back"]) + defer { endBackFixture() } + try assertBackPressesTheButtonAndPops(status, "an in-content button below the bar must not make the bar's button ambiguous") + } + + /// An in-content `BackButton` that keeps moving does not hold the lookup back: it is not compared. + @MainActor + func testInAppBackIsNotBlockedByAMovingInContentBackButton() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--moving-content-back"]) + defer { endBackFixture() } + + guard case .performed = try performInAppBack(app: app) else { + return XCTFail("a moving in-content button must not block the bar's back button") + } + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.found, "yes") + } + + /// A back button identified `BackButton` in an app with no navigation bar is not a system back + /// button: nothing is pressed. + @MainActor + func testInAppBackNeverPressesABackButtonWhenTheAppShowsNoNavigationBar() throws { + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--floating-back", "--hidden-bar"]) + defer { endBackFixture() } + + _ = try performInAppBack(app: app) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.found, "no") + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, false) + } + /// A delivered tap on the system back button is not a navigation: when the bar does not change, /// back reports an unknown outcome and never swipes after the tap. @MainActor @@ -92,6 +159,21 @@ extension RunnerTests { XCTAssertTrue(app.navigationBars["Detail"].exists) } + /// A back tap that presents an alert covers the bar's back button instead of popping: the button + /// stays in the tree, so the outcome is unknown, not a pop. + @MainActor + func testInAppBackReportsABackTapThatPresentsAnAlertAsUnverified() throws { + guard #available(iOS 16.0, *) else { throw XCTSkip("UINavigationItem.backAction needs iOS 16") } + _ = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--alert-back"]) + defer { endBackFixture() } + + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("an alert over the bar must not read as a pop") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + } + /// A bar title that counts down on its own is not a navigation: a tap that navigates nowhere on /// such a bar reports an unknown outcome, though the title changes within the confirmation window. @MainActor @@ -519,6 +601,17 @@ extension RunnerTests { return try XCTUnwrap(result).get() } + /// `back` presses a back button, confirms the pop, and the fixture's root reports it. + @MainActor + private func assertBackPressesTheButtonAndPops(_ status: XCUIElement, _ failure: String) throws { + guard case .performed = try performInAppBack(app: app) else { return XCTFail(failure) } + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.found, "yes") + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.pressed, true) + XCTAssertEqual(lastNavigationBarBackTraceForTesting?.confirmed, "yes") + let popped = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "Back pressed"), object: status) + XCTAssertEqual(XCTWaiter().wait(for: [popped], timeout: 5), .completed) + } + @MainActor private func launchBackNavigationFixture(_ arguments: [String]) throws -> XCUIElement { app.launchArguments = arguments @@ -526,7 +619,8 @@ extension RunnerTests { mainOwned.app = app mainOwned.bundleId = "com.callstack.agentdevice.runner" clearSnapshotXCTestChannelPenalty(reason: "navigation-test") - XCTAssertTrue(app.navigationBars.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) + let ready = arguments.contains("--hidden-bar") ? app.buttons[Self.navigationBarBackButtonIdentifier] : app.navigationBars.firstMatch + XCTAssertTrue(ready.waitForFixtureExistence(timeout: appExistenceTimeout)) return app.staticTexts["agent-device-back-navigation-status"] } @@ -733,7 +827,7 @@ extension RunnerTests { /// screen origin. func testNavigationBarBackStateHoldsTheLiveButtonOfAPlacedWindow() { let snapshotButton = CGRect(x: 8, y: 54, width: 44, height: 44) - let state = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [snapshotButton]) + let state = NavigationBarBackState(barCount: 1, backButtonLabels: ["Root"], backButtonFrames: [snapshotButton]) let placed = SurfaceGeometry( windowOnScreen: SurfaceRect(CGRect(x: 224, y: 128, width: 386, height: 954)), panelNativeSize: nil, @@ -752,22 +846,61 @@ extension RunnerTests { XCTAssertTrue(Self.navigationBarBackState(state, holdsButton: snapshotButton, geometry: fullScreen)) } - /// After the tap, only a change of the bar count, the back button count, or a back button label - /// shows a pop. A back button that only moved, such as by a large-title collapse, shows none, but + /// After the tap, only a change of the bar count, the back button count, a back button label, or, + /// for a back button with no label, a bar title shows a pop. A back button that only moved, such as by a large-title collapse, shows none, but /// it is never a still bar before the tap. - func testNavigationBarBackPoppedOnlyOnABarOrBackButtonChange() { + func testNavigationBarBackPoppedOnlyOnABarBackButtonOrUnlabeledBarTitleChange() { let button = CGRect(x: 8, y: 54, width: 44, height: 44) - let detail = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button]) + let detail = NavigationBarBackState(barCount: 1, backButtonLabels: ["Root"], backButtonFrames: [button]) - let moved = NavigationBarBackState(backButtonLabels: [["Root"]], backButtonFrames: [button.offsetBy(dx: 0, dy: 30)]) + let moved = NavigationBarBackState(barCount: 1, backButtonLabels: ["Root"], backButtonFrames: [button.offsetBy(dx: 0, dy: 30)]) XCTAssertFalse(Self.navigationBarBackPopped(from: detail, to: moved)) XCTAssertNotEqual(moved, detail) - let relabeled = NavigationBarBackState(backButtonLabels: [["Settings"]], backButtonFrames: [button]) + let relabeled = NavigationBarBackState(barCount: 1, backButtonLabels: ["Settings"], backButtonFrames: [button]) XCTAssertTrue(Self.navigationBarBackPopped(from: detail, to: relabeled)) - let atRoot = NavigationBarBackState(backButtonLabels: [[]], backButtonFrames: []) + let atRoot = NavigationBarBackState(barCount: 1, backButtonLabels: [], backButtonFrames: []) XCTAssertTrue(Self.navigationBarBackPopped(from: detail, to: atRoot)) - let barGone = NavigationBarBackState(backButtonLabels: [], backButtonFrames: []) + let barGone = NavigationBarBackState(barCount: 0, backButtonLabels: [], backButtonFrames: []) XCTAssertTrue(Self.navigationBarBackPopped(from: atRoot, to: barGone)) + + // A back button with no label shows a pop only through the bar title. + let unlabeled = NavigationBarBackState(barCount: 1, backButtonLabels: [""], backButtonFrames: [button], barTitles: ["About"]) + let unlabeledPopped = NavigationBarBackState(barCount: 1, backButtonLabels: [""], backButtonFrames: [button], barTitles: ["General"]) + XCTAssertTrue(Self.navigationBarBackPopped(from: unlabeled, to: unlabeledPopped)) + XCTAssertTrue(unlabeled.canShowAPop) + // Without a bar title an unlabeled button could never confirm its pop, so it is not tapped. + XCTAssertFalse(NavigationBarBackState(barCount: 1, backButtonLabels: [""], backButtonFrames: [button], barTitles: [""]).canShowAPop) + XCTAssertTrue(NavigationBarBackState(barCount: 1, backButtonLabels: ["Root"], backButtonFrames: [button]).canShowAPop) + } + + /// The Duo's floating back button (measured on iOS 27.1 in all three poses) and an iPhone's button + /// inside its bar belong to a bar. A content button far below the bar, or outside the bar's + /// horizontal span, does not. + func testBackButtonBelongsToABarOnlyWhenItIsByTheBar() { + XCTAssertTrue(Self.backButtonBelongsToABar( + CGRect(x: 884, y: 125, width: 38, height: 38), barFrames: [CGRect(x: 0, y: 24, width: 951, height: 58)], windowHeight: 669)) + XCTAssertTrue(Self.backButtonBelongsToABar( + CGRect(x: 399, y: 175, width: 38, height: 38), barFrames: [CGRect(x: 0, y: 24, width: 466, height: 58)], windowHeight: 678)) + XCTAssertTrue(Self.backButtonBelongsToABar( + CGRect(x: 8, y: 60, width: 44, height: 44), barFrames: [CGRect(x: 0, y: 54, width: 402, height: 44)], windowHeight: 874)) + let bar = CGRect(x: 0, y: 54, width: 402, height: 44) + XCTAssertFalse(Self.backButtonBelongsToABar(CGRect(x: 8, y: 700, width: 44, height: 44), barFrames: [bar], windowHeight: 874)) + XCTAssertFalse(Self.backButtonBelongsToABar(CGRect(x: 8, y: 20, width: 44, height: 20), barFrames: [bar], windowHeight: 874)) + XCTAssertFalse(Self.backButtonBelongsToABar( + CGRect(x: 500, y: 60, width: 44, height: 44), barFrames: [CGRect(x: 0, y: 54, width: 320, height: 44)], windowHeight: 874)) + XCTAssertFalse(Self.backButtonBelongsToABar(CGRect(x: 8, y: 60, width: 44, height: 44), barFrames: [], windowHeight: 874)) + } + + /// A button inside a bar is the candidate even when a content button sits below the bar; the + /// floating button is one when no button is inside a bar. + func testBackButtonCandidatesPreferTheButtonInsideTheBar() { + let bar = CGRect(x: 0, y: 54, width: 402, height: 44) + let inBar = CGRect(x: 8, y: 60, width: 44, height: 30) + let below = CGRect(x: 100, y: 120, width: 120, height: 44) + let content = CGRect(x: 100, y: 800, width: 120, height: 44) + XCTAssertEqual(Self.backButtonCandidates([inBar, below, content], barFrames: [bar], windowHeight: 874), [0]) + XCTAssertEqual(Self.backButtonCandidates([below, content], barFrames: [bar], windowHeight: 874), [0]) + XCTAssertEqual(Self.backButtonCandidates([content], barFrames: [bar], windowHeight: 874), []) } #endif diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index bf1ab37a04..15aee46a14 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -135,31 +135,53 @@ than silently applied: ``` `backInApp` is the only back command. On iOS the runner first reads the current tree, in the same -call, for the navigation bar's system back button: a button with the identifier `BackButton` inside a -navigation bar. It taps that button with a synthesized tap only when exactly one live, hittable -candidate is found in a readable tree, and two reads in a row of the bar state, with a 0.05 s pause -between them, agree and contain that button's frame; each round re-reads the buttons. The match -places the snapshot frames at the window's screen origin, because a placed iPad window reports them -from its own top-left. No other button -counts, whatever its label. -The bar state is the labels of each navigation bar's `BackButton` children, one list per bar in -bar order, and the frames of those buttons; one XCTest snapshot per bar. The still check before the -tap compares labels and frames. Bar titles and identifiers are left out, because a bar can retitle -itself (a countdown) without navigating. After the tap it reads that state again with a 0.05 s -pause between reads. Only these changes from the state read before the tap answer `ok`: the number -of bars, the number of `BackButton` children (such as none at the root), or a `BackButton` label. +call, for the system back button: a button with the identifier `BackButton` in the app's windows +(the lit panel's windows when a panel is booked). A floating one, such as the iPhone Duo's, is a +sibling of the navigation bar rather than inside it. The runner taps that button with a +synthesized tap only when all of these hold: + +- The tree is readable. +- A candidate is a button inside a navigation bar. Only when none is, a candidate is a button + horizontally within a bar and below it, within a third of the window height from the bar's top. +- Two reads in a row of the `BackButton` buttons by a navigation bar (the candidates), with a 0.05 s + pause between them, agree on their labels and frames, hittable or not. A `BackButton` that is not + by a bar is neither counted nor compared. +- One live read of those buttons then finds exactly one hittable. Its frame is in the state. The match + places the snapshot frames at the window's screen origin, because a placed iPad window reports them + from its own top-left. +- The bar state read by the first round and by the live read agree (below). +- A pop can show in the state: every candidate has a label, or a bar has a title. + +No other button counts, whatever its label. + +The state is the number of navigation bars, the labels of the candidates in tree order, +and the frames of those buttons. The runner reads one XCTest snapshot per button. Bar titles are not +part of the state, because a bar can retitle itself (a countdown) without navigating. One exception +applies: a state with a candidate that has no label, such as the Duo's, also holds the title of +each bar. The title is the bar's identifier, which UIKit sets to the title, or its label when the +identifier is empty. The label of such a button cannot show a pop. The state read applies this rule +itself, so the state before the tap and each state after it follow it. The state before the tap must +hold still: the bar count and, with an unlabeled button, the titles must agree between the first +round and the live read, so a title that changes on its own is never still. When such a state has no +non-empty title, the pop could not be confirmed, so nothing is tapped and the runner swipes +(`found=unprovable`). After the tap the runner reads that state again with a 0.05 s pause between +reads. Only these changes from the state read before the tap answer `ok`: the number of bars, the +number of `BackButton` buttons (such as none at the root), a `BackButton` label, or, for an +unlabeled button, a bar title. Frames are left out after the tap, because a tap that navigates nowhere can still move the button (a large-title collapse, a bar button that appears). A pop that keeps the same `BackButton` labels and bar count, such as one between two screens whose back buttons have the same title, answers -`BACK_SWIPE_UNVERIFIED`; so can a pop on a -back button whose title updates itself, misread as a navigation. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is +`BACK_SWIPE_UNVERIFIED`; so can a pop on a back button whose title updates itself, misread as a +navigation. Under an unlabeled button the same misread applies to a bar that retitles itself: +after the tap, any single changed title read within the confirmation window answers `ok`. No change, or reads that fail, answer `BACK_SWIPE_UNVERIFIED`; a delivered tap is never followed by the swipe. Before and after the tap, reads start for up to 1 s each, checked -before each bar's snapshot; a query in flight finishes or ends at XCTest's own query timeout. +before each button's snapshot and each bar's snapshot; the live read before the tap counts as a read of +its own. A query in flight finishes or ends at XCTest's own query timeout. After the tap, when no read in the 1 s saw a change or hung, the runner starts one more read after -the 1 s, with no deadline of its own: a snapshot of a bar that vanished mid-pop can spend the 1 s in -XCTest's retries, and this read can still see the screen after the pop. That read can also fail, +the 1 s, with no deadline of its own: a snapshot of a button that vanished mid-pop can spend the 1 s +in XCTest's retries, and this read can still see the screen after the pop. That read can also fail, which answers `BACK_SWIPE_UNVERIFIED`, or hang, which ends at XCTest's own query timeout and also -answers `BACK_SWIPE_UNVERIFIED`. A read in flight can run well past the 1 s: one bar snapshot took +answers `BACK_SWIPE_UNVERIFIED`. A read in flight can run well past the 1 s: one snapshot took about 2 s on a CI simulator. Every bar read holds its XCTest issues. A held issue means the read proved nothing. A hung query @@ -172,8 +194,10 @@ The runner swipes instead when no tap was sent: - The tree read is skipped: the window resolution marked the tree unreadable, accessibility is unavailable, or the XCTest snapshot channel is penalized for the app. - The read fails or XCTest rejects one of its queries. -- No candidate is found, or two or more are (such as a sheet over a stack). -- The bar state does not hold still within 1 s. +- No `BackButton` is found, no `BackButton` is hittable, or two or more are (such as a sheet over a + stack), or the hittable one is not by a navigation bar, or a `BackButton` has no label and no bar + has a title. +- The back buttons or the bar state do not hold still within 1 s. - The synthesized tap is refused before anything is sent. A tap whose outcome is unknown after it may have been sent throws, and neither the bar check nor @@ -217,7 +241,7 @@ window can still be misread as a back navigation; any finite window has that lim Each iOS back whose window context resolved logs one `AGENT_DEVICE_RUNNER_BACK_NAVIGATION_BAR_BUTTON` line on every exit of the button path, a thrown tap included, with `found` (`skipped`, -`unreadable`, `hung`, `moving`, `no`, `ambiguous`, or `yes`), `pressed`, `confirmed` (`yes`, `no`, +`unreadable`, `hung`, `moving`, `no`, `ambiguous`, `unprovable`, or `yes`), `pressed`, `confirmed` (`yes`, `no`, `hung`, or `-` before a tap), `lookupMs` (`0` when the read was skipped), and `confirmMs`. Each swipe logs one runner line, `AGENT_DEVICE_RUNNER_BACK_SWIPE_VERIFICATION`, on every exit, with its `outcome` (`performed`, `refused`, `unconfirmed`, or `error` for a thrown failure), `code`, `changed`, `stableMs`, `justBeforeToAfterMs`, `gestureMs`, `maxCaptureMs`, diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 07de4cac1e..10834c6d55 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -286,7 +286,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (for a back button with a title, a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap, and a bar that retitles itself can still be misread as a pop; when no bar has a title or a title keeps changing, that button is not pressed); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 888e4b04fe..e73e49687c 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -125,9 +125,10 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. macOS finds that control by the app's own back identifier or "Back" label; an app that exposes neither refuses with `error.details.reason: in_app_back_control_unavailable` under `COMMAND_FAILED` and `dispatched: no`, pressing nothing, so press the app's visible back control by its ref. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), a 0.25 s margin, and the slowest capture: about 0.55 s in all on a simulator. The margin keeps a screen that repaints itself about every 0.5 s, such as a carousel, from looking still, so its next repaint is not read as a back; it does not depend on the 0.1 s wait after the swipe. A screen that repaints on its own more often than the still span never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.45 s. A still screen usually settles in about 0.6 s on a simulator, and in about 1.4 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar, or, when none is, one floating below it, such as on the iPhone Duo), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the back buttons and navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. A back button with no title of its own, such as the Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap; when no bar has a title, or a title keeps changing, that button is not pressed and `back` swipes. If none changes, it reports `back_swipe_unverified` and does not swipe. For a back button that has its own title, a bar's title never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself, or a navigation bar that retitles itself under a back button with no title, can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), a 0.25 s margin, and the slowest capture: about 0.55 s in all on a simulator. The margin keeps a screen that repaints itself about every 0.5 s, such as a carousel, from looking still, so its next repaint is not read as a back; it does not depend on the 0.1 s wait after the swipe. A screen that repaints on its own more often than the still span never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.45 s. A still screen usually settles in about 0.6 s on a simulator, and in about 1.4 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. -- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. +- `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons (or, for a back button with no title, the bar's title) did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. +- Runner log `found=unprovable`: the back button by a navigation bar has no title and no navigation bar has one, so a pop could not be confirmed. The button is not pressed and `back` swipes instead. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. - If the app's screen cannot be captured before the swipe, `back` does not swipe: it refuses with `COMMAND_FAILED` and `details.runnerErrorCode` set to `APP_SCREEN_WINDOW_UNRESOLVED`, `APP_SCREEN_UNRESOLVED`, `APP_SCREEN_CAPTURE_UNRENDERABLE`, or `APP_SCREEN_OFF_LIT_PANEL`. Nothing is sent. - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back command and refuses before dispatch with `UNSUPPORTED_OPERATION`.