From 0aa7976e0f3bc63f7062984190ddbf004f5c5ba3 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 04:07:27 +0000 Subject: [PATCH 1/3] fix(ios): bound the idle wait of keyboard dismiss and keyboard enter `keyboard dismiss` and `keyboard enter` tapped the keyboard's control with a plain XCUIElement.tap() outside withBoundedInteractionIdleTimeoutIfSupported, which `type`, `swipe` and taps already use. XCTest then waited for the app to idle with its own timeout. An app that keeps animating (Safari right after an iPhone Duo unfold) held the runner main thread past its 30 s watchdog: the command failed with "main thread execution timed out" and the next commands were refused until XCTest gave up. Both commands now run inside the bounded wait with both quiescence waits skipped. dismissKeyboard still waits for the keyboard to go and the screen to settle on its own. The host app's --agent-device-keyboard-busy fixture keeps a UIView animation in flight while its field has focus, until a backstop, and records whether its Done accessory and its return key landed while the app was still busy. Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerAppFixtures.m | 91 +++++++++++++++++++ .../RunnerTests+CommandExecution.swift | 16 +++- .../UnitTests/RunnerTests+KeyboardTests.swift | 56 ++++++++++++ 3 files changed, 161 insertions(+), 2 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 1df45efe69..55924d6ab4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -75,6 +75,9 @@ @interface AgentDeviceRunnerFixtureViewController () *arguments = NSProcessInfo.processInfo.arguments; + NSUInteger flag = [arguments indexOfObject:@"--agent-device-keyboard-busy"]; + return flag + 1 < arguments.count ? arguments[flag + 1].doubleValue : 0; +} + +- (void)startKeyboardBusy { + if (self.keyboardBusyBackstop != nil) { + return; + } + self.keyboardBusyBackstop = [NSTimer scheduledTimerWithTimeInterval:AgentDeviceKeyboardBusyWindow() + target:self + selector:@selector(stopKeyboardBusy) + userInfo:nil + repeats:NO]; + [UIView animateWithDuration:0.4 + delay:0 + options:(UIViewAnimationOptionRepeat | UIViewAnimationOptionAutoreverse) + animations:^{ + self.keyboardBusyAnswer.transform = CGAffineTransformMakeTranslation(0, 8); + } + completion:nil]; +} + +- (void)stopKeyboardBusy { + [self.keyboardBusyAnswer.layer removeAllAnimations]; + self.keyboardBusyAnswer.transform = CGAffineTransformIdentity; + [self.keyboardBusyBackstop invalidate]; + self.keyboardBusyBackstop = nil; +} + +- (void)recordKeyboardBusyAnswer:(NSString *)control { + self.keyboardBusyAnswer.text = [NSString + stringWithFormat:@"%@ %@", control, self.keyboardBusyBackstop != nil ? @"while busy" : @"after busy"]; + [self stopKeyboardBusy]; + [self.keyboardBusyField resignFirstResponder]; +} + +- (void)keyboardBusyDonePressed { + [self recordKeyboardBusyAnswer:@"Done"]; +} + +- (void)keyboardBusyReturnPressed { + [self recordKeyboardBusyAnswer:@"Return"]; +} + // A banner from this app, shown over its own alert and re-posted before the previous one expires so // one is on screen for as long as the first alert is unanswered. XCTest treats such a banner as an // interruption of every event aimed at the app (#2546's late tap, from the banner side). @@ -652,6 +706,43 @@ - (void)viewDidLoad { self.alertActivationBusyAnswer = busyAnswer; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-keyboard-busy"]) { + UITextField *field = [[UITextField alloc] init]; + field.accessibilityIdentifier = @"agent-device-keyboard-busy-input"; + field.placeholder = @"Busy while editing"; + field.borderStyle = UITextBorderStyleRoundedRect; + UIToolbar *accessory = [[UIToolbar alloc] initWithFrame:CGRectMake(0, 0, 320, 44)]; + accessory.items = @[ + [[UIBarButtonItem alloc] initWithBarButtonSystemItem:UIBarButtonSystemItemFlexibleSpace target:nil action:nil], + [[UIBarButtonItem alloc] initWithTitle:@"Done" + style:UIBarButtonItemStyleDone + target:self + action:@selector(keyboardBusyDonePressed)], + ]; + field.inputAccessoryView = accessory; + [field addTarget:self action:@selector(startKeyboardBusy) forControlEvents:UIControlEventEditingDidBegin]; + [field addTarget:self + action:@selector(keyboardBusyReturnPressed) + forControlEvents:UIControlEventEditingDidEndOnExit]; + field.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:field]; + UILabel *answer = [[UILabel alloc] init]; + answer.text = @"Unanswered"; + answer.accessibilityIdentifier = @"agent-device-keyboard-busy-answer"; + answer.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:answer]; + [NSLayoutConstraint activateConstraints:@[ + [field.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [field.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24], + [field.widthAnchor constraintEqualToConstant:240], + [field.heightAnchor constraintEqualToConstant:44], + [answer.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [answer.topAnchor constraintEqualToAnchor:field.bottomAnchor constant:12], + ]]; + self.keyboardBusyField = field; + self.keyboardBusyAnswer = answer; + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-native-states"]) { for (NSUInteger index = 0; index < 3; index++) { UITextField *field = [[UITextField alloc] initWithFrame:CGRectMake(40, 120 + index * 60, 260, 44)]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 1d002e9056..55f90b074e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -587,7 +587,13 @@ extension RunnerTests { } return Response(ok: true, data: DataPayload(message: command.command.rawValue)) case .keyboardDismiss: - let result = dismissKeyboard(app: activeApp) + var dismissal: (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?)? + withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { + dismissal = dismissKeyboard(app: activeApp) + } + guard let result = dismissal else { + return Response(ok: false, error: ErrorPayload(message: "keyboardDismiss produced no result")) + } if result.wasVisible && !result.dismissed { return Response( ok: false, @@ -610,7 +616,13 @@ extension RunnerTests { ) ) case .keyboardReturn: - let result = pressKeyboardReturn(app: activeApp) + var press: (wasVisible: Bool, pressed: Bool, visible: Bool)? + withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { + press = pressKeyboardReturn(app: activeApp) + } + guard let result = press else { + return Response(ok: false, error: ErrorPayload(message: "keyboardReturn produced no result")) + } if !result.pressed { return Response( ok: false, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift index 06c3aacf0a..32d1cb72a4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift @@ -327,6 +327,62 @@ extension RunnerTests { private func textEntryInputCount() -> Int { app.textFields.count + app.secureTextFields.count + app.searchFields.count + app.textViews.count } + + // `keyboard dismiss` and `keyboard enter` tap a keyboard control. An app that never stops animating + // made that tap wait for XCTest's idle timeout, past the runner's main-thread watchdog, so the command + // failed with "main thread execution timed out" and the runner refused the next commands. The fixture + // keeps an animation in flight while its field has focus, until its backstop, and records whether + // each control landed while the app was still busy. + static let keyboardBusyBackstop: TimeInterval = 20 + + @MainActor + func testKeyboardDismissTapsItsDismissKeyWhileTheAppIsBusy() throws { + try withBusyKeyboardField { + let command = try runnerCommandFixture(#"{"command":"keyboardDismiss","commandId":"dismiss-busy"}"#) + let response = try execute(command: command) + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-keyboard-busy-answer"].label, "Done while busy") + } + } + + @MainActor + func testKeyboardEnterPressesReturnWhileTheAppIsBusy() throws { + try withBusyKeyboardField { + let command = try runnerCommandFixture(#"{"command":"keyboardReturn","commandId":"return-busy"}"#) + let response = try execute(command: command) + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-keyboard-busy-answer"].label, "Return while busy") + } + } + + @MainActor + private func withBusyKeyboardField(_ body: () throws -> Void) throws { + app.launchArguments = ["--agent-device-keyboard-busy", String(Self.keyboardBusyBackstop)] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let field = app.textFields["agent-device-keyboard-busy-input"] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + mainOwned.processIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) + // Focus starts the busy animation, so this tap must not wait for the app to idle either. + withBoundedInteractionIdleTimeoutIfSupported(app, waits: .bothSkipped) { field.tap() } + let keyboardDeadline = Date().addingTimeInterval(5) + while !isKeyboardVisible(app: app) && Date() < keyboardDeadline { + sleepFor(0.2) + } + try XCTSkipUnless( + isKeyboardVisible(app: app), + "software keyboard is down: this simulator cannot exercise a keyboard control" + ) + XCTAssertEqual(app.staticTexts["agent-device-keyboard-busy-answer"].label, "Unanswered") + try body() + } #endif } #endif From 90fcc5ae0c3f756987aeedf014ae04356df146eb Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 04:51:52 +0000 Subject: [PATCH 2/3] fix(ios): require busy keyboard coverage and settle Return --- .../RunnerTests+CommandExecution.swift | 2 +- .../UnitTests/RunnerTests+KeyboardTests.swift | 14 ++++++++++---- apple/runner/README.md | 9 +++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 55f90b074e..bc88aee555 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -617,7 +617,7 @@ extension RunnerTests { ) case .keyboardReturn: var press: (wasVisible: Bool, pressed: Bool, visible: Bool)? - withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { + withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .preEventSkipped) { press = pressKeyboardReturn(app: activeApp) } guard let result = press else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift index 32d1cb72a4..ecd62afcc2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift @@ -342,7 +342,9 @@ extension RunnerTests { let response = try execute(command: command) XCTAssertTrue(response.ok, String(describing: response.error)) - XCTAssertEqual(app.staticTexts["agent-device-keyboard-busy-answer"].label, "Done while busy") + let answer = app.staticTexts["agent-device-keyboard-busy-answer"].label + XCTAssertEqual(answer, "Done while busy") + NSLog("AGENT_DEVICE_KEYBOARD_BUSY command=dismiss response.ok=%@ answer=%@", response.ok ? "true" : "false", answer) } } @@ -353,7 +355,11 @@ extension RunnerTests { let response = try execute(command: command) XCTAssertTrue(response.ok, String(describing: response.error)) - XCTAssertEqual(app.staticTexts["agent-device-keyboard-busy-answer"].label, "Return while busy") + XCTAssertEqual(response.data?.wasVisible, true) + XCTAssertEqual(response.data?.visible, false) + let answer = app.staticTexts["agent-device-keyboard-busy-answer"].label + XCTAssertEqual(answer, "Return while busy") + NSLog("AGENT_DEVICE_KEYBOARD_BUSY command=enter response.ok=%@ visible=%@ answer=%@", response.ok ? "true" : "false", String(describing: response.data?.visible), answer) } } @@ -376,9 +382,9 @@ extension RunnerTests { while !isKeyboardVisible(app: app) && Date() < keyboardDeadline { sleepFor(0.2) } - try XCTSkipUnless( + XCTAssertTrue( isKeyboardVisible(app: app), - "software keyboard is down: this simulator cannot exercise a keyboard control" + "Enable the simulator software keyboard before running the busy keyboard tests; see apple/runner/README.md" ) XCTAssertEqual(app.staticTexts["agent-device-keyboard-busy-answer"].label, "Unanswered") try body() diff --git a/apple/runner/README.md b/apple/runner/README.md index d9dca31e42..c919112b26 100644 --- a/apple/runner/README.md +++ b/apple/runner/README.md @@ -33,6 +33,15 @@ Protocol and maintenance references: ## UITest Runner File Map +### Busy keyboard regression tests + +Run the two `testKeyboard*WhileTheAppIsBusy` tests with the simulator software keyboard enabled. +Before you boot a throwaway simulator, run `defaults write com.apple.iphonesimulator ConnectHardwareKeyboard -bool NO`. +Restore the previous preference after the run. +If the software keyboard does not appear, the tests fail instead of skipping. +The tests assert command success and the fixture's `Done while busy` or `Return while busy` label. +The Return test also asserts the keyboard visibility reported after submission. + `AgentDeviceRunnerUITests/RunnerTests` is split into focused files to reduce context size for contributors and LLM agents. - `RunnerTests.swift`: shared state/constants, `setUp()`, and `testCommand()` entry flow. From 27197a7527fbaee0ea73da19332995c88fc040ab Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 05:04:17 +0000 Subject: [PATCH 3/3] docs(ios): restore Simulator keyboard preference after tests --- apple/runner/README.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/apple/runner/README.md b/apple/runner/README.md index c919112b26..a2a3626332 100644 --- a/apple/runner/README.md +++ b/apple/runner/README.md @@ -33,15 +33,6 @@ Protocol and maintenance references: ## UITest Runner File Map -### Busy keyboard regression tests - -Run the two `testKeyboard*WhileTheAppIsBusy` tests with the simulator software keyboard enabled. -Before you boot a throwaway simulator, run `defaults write com.apple.iphonesimulator ConnectHardwareKeyboard -bool NO`. -Restore the previous preference after the run. -If the software keyboard does not appear, the tests fail instead of skipping. -The tests assert command success and the fixture's `Done while busy` or `Return while busy` label. -The Return test also asserts the keyboard visibility reported after submission. - `AgentDeviceRunnerUITests/RunnerTests` is split into focused files to reduce context size for contributors and LLM agents. - `RunnerTests.swift`: shared state/constants, `setUp()`, and `testCommand()` entry flow. @@ -65,6 +56,31 @@ The Return test also asserts the keyboard visibility reported after submission. - `UnitTests/RunnerTests+Tests.swift`: the `AGENT_DEVICE_RUNNER_UNIT_TESTS` tests for each source file. The packaged runner source omits this directory. +### Busy keyboard regression tests + +Run the two `testKeyboard*WhileTheAppIsBusy` tests with the simulator software keyboard enabled. +Before you boot a throwaway simulator, save the current hardware-keyboard preference: + +```sh +keyboard_preference=$(defaults read com.apple.iphonesimulator ConnectHardwareKeyboard 2>/dev/null) || keyboard_preference=absent +defaults write com.apple.iphonesimulator ConnectHardwareKeyboard -bool NO +``` + +After the tests, restore the preference in the same shell, including when the tests fail. +If the key was absent, delete it: + +```sh +if [ "$keyboard_preference" = absent ]; then + defaults delete com.apple.iphonesimulator ConnectHardwareKeyboard +else + defaults write com.apple.iphonesimulator ConnectHardwareKeyboard -bool "$keyboard_preference" +fi +``` + +If the software keyboard does not appear, the tests fail instead of skipping. +The tests assert command success and the fixture's `Done while busy` or `Return while busy` label. +The Return test also asserts the keyboard visibility reported after submission. + ## Snapshot Strategy iOS snapshots have two explicit public capture modes: