diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 408d81c48a..91b9002f0f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -784,15 +784,27 @@ - (void)viewDidLoad { } }]; } - // `--slow-toggle` repaints the whole screen every 0.3 s, slower than one verification delay, as - // a carousel does: one quiet pair of samples must not pass it for settled. - if ([NSProcessInfo.processInfo.arguments containsObject:@"--slow-toggle"]) { - __weak UIView *view = self.view; - [self.animationTimer invalidate]; - self.animationTimer = [NSTimer scheduledTimerWithTimeInterval:0.3 repeats:YES block:^(NSTimer *timer) { - view.backgroundColor = [view.backgroundColor isEqual:UIColor.blackColor] ? UIColor.whiteColor : UIColor.blackColor; - }]; - } + } + // `--slow-toggle` repaints the whole screen every 0.45 s, as a carousel does: a run of quiet samples + // shorter than the back swipe's still span must not pass it for settled. Without + // `--agent-device-back-regression` nothing answers the swipe, so only the repaint changes the screen. + // The repaints fall on whole multiples of 0.45 s of the wall clock, which the test process shares, + // so a test can start at a chosen phase of the cycle. The shade steps through 8 grays by wall-clock + // period, so two samples on either side of a stall of a few periods still differ. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--slow-toggle"]) { + __weak UIView *view = self.view; + NSTimeInterval period = 0.45; + NSDate *firstRepaint = + [NSDate dateWithTimeIntervalSince1970:(floor(NSDate.date.timeIntervalSince1970 / period) + 1) * period]; + [self.animationTimer invalidate]; + self.animationTimer = [[NSTimer alloc] initWithFireDate:firstRepaint interval:period repeats:YES block:^(NSTimer *timer) { + // NSTimer fires at or after its slot, never before: floor names the slot a late callback + // belongs to, so a late callback and the next on-time one never share a shade. The 1e-3 + // keeps float error just below a slot boundary in that slot. + long long step = (long long)floor(NSDate.date.timeIntervalSince1970 / period + 1e-3) % 8; + view.backgroundColor = [UIColor colorWithWhite:step / 7.0 alpha:1]; + }]; + [NSRunLoop.mainRunLoop addTimer:self.animationTimer forMode:NSRunLoopCommonModes]; } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 3919ecdcc5..9bec27637d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -6,6 +6,7 @@ extension RunnerTests { static let navigationBarBackButtonIdentifier = "BackButton" /// The wait from the end of the swipe's contact to the sample it is compared against. An /// interactive pop follows the contact across most of the width, so the screen has already moved. + /// It only bounds how soon that sample is taken; the still span does not depend on it. static let backSwipeVerificationDelay: TimeInterval = 0.1 /// The pause between two settle samples; a capture itself takes about 0.1 s on a simulator. static let backSwipeSettleSampleInterval: TimeInterval = 0.05 @@ -20,6 +21,12 @@ extension RunnerTests { /// The least time settling may sample before back swipes anyway and reports the outcome /// unverified; `backSwipeSettleBudgetMs` extends it on a host that captures slowly. static let backSwipeSettleBaseBudgetMs = 2000.0 + /// The still span's margin beyond the contact and the slowest capture. A screen that repaints + /// itself less often than the span looks still, and its next repaint after the swipe reads as a + /// back. 250 ms makes the span about 0.55 s on a simulator, so such a repaint every 0.5 s is + /// caught as changing. It is deliberately independent of `backSwipeVerificationDelay`, so a + /// faster after-sample does not weaken the guard. + static let backSwipeStillMarginMs = 250.0 /// How long settling may sample: the base budget, or, when longer, the time a still screen needs /// to settle at the slowest capture seen: the first capture, the required still span, and one more @@ -33,12 +40,13 @@ extension RunnerTests { ) } - /// How long the screen must agree with itself before the swipe: as long as the gap it will be - /// credited across, the contact allowance, one verification delay, and the slowest capture seen - /// while settling. A slow host captures slowly, so it settles for longer instead of reading a real - /// pop as unverified. `backSwipeGapIsCovered` still refuses a swipe whose actual gap outran it. + /// How long the screen must agree with itself before the swipe: the contact allowance, the still + /// margin, and the slowest capture seen while settling. That is longer than the gap the swipe is + /// credited across (contact, verification delay, capture). A slow host captures slowly, so it + /// settles for longer instead of reading a real pop as unverified. `backSwipeGapIsCovered` still + /// refuses a swipe whose actual gap outran it. static func backSwipeRequiredStillMs(maxCaptureMs: Double) -> Double { - backGestureAllowanceMs + backSwipeVerificationDelay * 1000 + maxCaptureMs + backGestureAllowanceMs + backSwipeStillMarginMs + maxCaptureMs } /// What one `back` concluded. `.refused` dispatched nothing. `.unconfirmed` ran the back gesture, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 4c8029843f..2007576d2b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -45,19 +45,21 @@ extension RunnerTests { XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) } - /// A screen that changes on its own every 0.3 s, three verification delays apart, is never still - /// for the required span (the contact allowance, one 0.1 s delay, and the slowest capture, about - /// 0.4 s on a simulator), so it never settles and back reports an unknown outcome instead of - /// crediting the swipe. + /// A screen that repaints itself every 0.45 s, with a swipe that does nothing, is never still for + /// the required span (about 0.55 s on a simulator), so its own repaint after the swipe is never + /// read as a back. Attempt k starts k * 0.09 s after a repaint, five phases across the cycle. @MainActor - func testInAppBackOnAScreenChangingSlowerThanOneDelayIsNotPerformed() throws { - _ = try launchBackFixture(["--agent-device-back-regression", "--slow-toggle"]) + func testInAppBackNeverCreditsASlowSelfRepaintToANoOpSwipe() throws { + _ = try launchBackFixture(["--slow-toggle"]) defer { endBackFixture() } - guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { - return XCTFail("a screen that toggles on its own must not read as performed") + for offset in (0..<5).map({ Double($0) * Self.slowTogglePeriod / 5 }) { + waitForSlowToggleOffset(offset) + guard case .unconfirmed(let payload) = try performInAppBack(app: app) else { + return XCTFail("a repaint \(offset) s before the attempt must not read as a back") + } + XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified, "offset \(offset) s") } - XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeUnverified) } /// The navigation bar's system back button is pressed from the live tree: the fixture's @@ -473,6 +475,13 @@ extension RunnerTests { wait(for: [performed], timeout: 1) } + /// Waits until `offset` seconds have passed since the `--slow-toggle` fixture's last repaint. + private func waitForSlowToggleOffset(_ offset: TimeInterval) { + let sinceRepaint = Date().timeIntervalSince1970.truncatingRemainder(dividingBy: Self.slowTogglePeriod) + let wait = offset - sinceRepaint + sleepFor(wait < 0 ? wait + Self.slowTogglePeriod : wait) + } + /// Launches the runner app and waits for its fixture surface. With /// `--agent-device-back-regression` a left-edge swipe sets the status to "Back performed", and /// without it the app ignores the swipe. @@ -694,6 +703,21 @@ extension RunnerTests { XCTAssertGreaterThanOrEqual(slow, Self.backSwipeDurationMs + Self.backSwipeVerificationDelay * 1000 + 600) } + /// The `--slow-toggle` fixture's repaint period; it repaints on whole multiples of it of the wall + /// clock, which this process shares. + static let slowTogglePeriod: TimeInterval = 0.45 + + /// The still span outlasts a 0.5 s self-repaint at a simulator's 0.1 s capture, and the + /// `--slow-toggle` fixture's period by at least 100 ms. It stays longer than the after-sample gap + /// (contact, verification delay, capture), whatever the delay. + func testBackSwipeRequiredStillSpanOutlastsAHalfSecondRepaint() { + let required = Self.backSwipeRequiredStillMs(maxCaptureMs: 100) + XCTAssertGreaterThan(required, 500) + XCTAssertGreaterThanOrEqual(required, Self.slowTogglePeriod * 1000 + 100) + XCTAssertEqual(required, Self.backGestureAllowanceMs + Self.backSwipeStillMarginMs + 100) + XCTAssertGreaterThan(required, Self.backGestureAllowanceMs + Self.backSwipeVerificationDelay * 1000 + 100) + } + #if os(iOS) /// #3899: a placed iPad window's snapshot frames start at the window's own origin, while the live /// button frame is in screen points. The still bar holds the live button only through the window's diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index d88201deee..5d3281ab63 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -189,9 +189,11 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure | `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or on one more read after it or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a -0.05 s pause between captures plus each capture's own time, that agree for as long as the gap the swipe will be credited across: a contact -allowance, one 0.1 s delay, and the slowest capture seen while settling. A slow host therefore -settles for longer. The last sample of that run is the one the change is measured from, and the +0.05 s pause between captures plus each capture's own time, that agree for the still span: a contact +allowance, a 0.25 s margin, and the slowest capture seen while settling, about 0.55 s on a simulator. +The margin keeps a screen that repaints itself about every 0.5 s from looking still, so its next +repaint is not read as the swipe's effect; it does not depend on the 0.1 s wait before the sample +after the swipe. A slow host therefore settles for longer. The last sample of that run is the one the change is measured from, and the swipe follows it at once: | Platform | Resolved before settling | Left in the gap | Contact allowance | @@ -202,9 +204,9 @@ swipe follows it at once: On both, every tree and window read happens during that preparation, before the first sample. Settling samples for at most 2 s, plus the capture in flight. On a slow host the budget grows to the first capture, the still span, and one more pause and capture, so a still screen settles instead of always answering `BACK_SWIPE_UNVERIFIED`. A still screen usually settles after -4 samples, about 0.45 s on a local simulator with 0.1 s captures; with the 0.4 s captures measured +5 samples, about 0.6 s on a local simulator with 0.1 s captures; with the 0.4 s captures measured on one physical iPhone (iPhone 17 Pro, iOS 27.0.1; -https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541) the still span is 0.9 s. +https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541) the still span is 1.05 s. Only a change from the last sample to the sample taken 0.1 s after the swipe counts, and only when the time between those two does not exceed the time the screen had been still; otherwise the result is diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a93e539340..f499e11bf7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,7 +124,7 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. -- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), the 0.1 s wait after the swipe, and the slowest capture: about 0.4 s in all on a simulator. A screen that repaints on its own more often than that never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.3 s. A still screen usually settles in about 0.45 s on a simulator, and in about 1 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). +- On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. If none changes, it reports `back_swipe_unverified` and does not swipe. A bar's title alone never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), a 0.25 s margin, and the slowest capture: about 0.55 s in all on a simulator. The margin keeps a screen that repaints itself about every 0.5 s, such as a carousel, from looking still, so its next repaint is not read as a back; it does not depend on the 0.1 s wait after the swipe. A screen that repaints on its own more often than the still span never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.45 s. A still screen usually settles in about 0.6 s on a simulator, and in about 1.4 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). - `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device.