diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 4368e2f15..4edaf6c30 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -318,7 +318,7 @@ The port installs the skill with `npx skills add hardkoded/e2e-dotnet` and regis - Upstream's `device.unstable_raw` (`feat(mobile): device.unstable_raw exposes the test's agent-device session`, `c3c4238`, tester-army/e2e#1380) is `Device.UnstableRaw`. Upstream's `DeviceSession` interface (`device.ts`) is `RawDeviceSession`, a .NET-only name, because the port's `DeviceSession` is the surface (upstream's `AgentDeviceSurface`) and a public type of that name exists. Its `client` is the session's `IDeviceClient` (the port's stand-in for `AgentDeviceClient`), `selection` is `DeviceSelection`, and `platform` is `MobilePlatform`. `BoundDevice` (`surface.ts`) is internal, set by the boot from the `id` and `device` of the `devices.boot` answer (`DeviceSession.BoundDevice`, upstream's `boundDevice`), and cleared when the session closes. Reading `UnstableRaw`, or `Client` of a handle taken earlier, after the session closed is `INVALID_STATE`. The client is `DeviceCommands.Translate`, which refuses a method it does not list, so `DeviceMethods.AppsList` (`apps.list`, the daemon's `apps` command) is added for upstream's reference example; the other methods of upstream's client stay unsupported until code calls them. Tests: the three `it` of `describe('unstable_raw')` in `Engine/DeviceFixtureUnstableRawTests`; the first is `Partial_`, since the port's fixture has no map of declared operations for `expect(declared).not.toHaveProperty('unstable_raw')`. Not ported: `unstable_raw reaches the test session through agent-device` (`apps/mobile-benchmark/tests/device-fixture.e2e.ts`), which needs the mobile benchmark app, a React Native app the port lacks. - Upstream's known-viewport fix (`fix(mobile): keep the known viewport when the app reads smaller than the screen`, `7674a2c`, tester-army/e2e#1291) is ported. `DeviceSession.Project` learns a snapshot's viewport only when it is plainly the screen's: the first one the session reads, the one it knows, or that one rotated (`SameScreen`, upstream's `sameScreen`; two sizes within a point are one screen, since rects from the tree carry fractions). `ViewportForAsync` (upstream's `viewportFor`) checks a tree of any other size against the device's screenshot, in either orientation, before it replaces the known one: a screenshot reports the portrait size even in landscape, so it confirms a size but cannot pick the orientation. The size it checks against is `DeviceSupport.LogicalScreenSize` (upstream's `logicalScreenSize`): the screenshot's logical dimensions, else its pixel dimensions over its density. A simulator's screenshot reports those; a real device's reports neither, so there the tree's size is taken as it comes. A screenshot that fails during that check fails the observation, as upstream's does. `TraceScreen` and `MaskedScreenshotAsync` size the trace screen and the secure-field masks by the known viewport, not the tree's. On iOS, a swipe from a sheet's grabber can pull down Notification Center, and the app's tree behind it reported a third of the screen (130x281 on a 390x844 device), which misplaced every rect and tap after it. Upstream records this only in its changeset; its skill references and docs say nothing about it, so `SKILL.md` and `docs/` have nothing to mirror. Tests: in `Engine/ObservationTests`, 'keeps the known viewport for an app drawn smaller than the screen, and follows a rotation' (new) and 'measures the screen in the viewport the snapshot publishes, and in its tree when it publishes none' (split into two harnesses, as upstream's); in `Engine/DeviceFixtureDismissingAKeyboardThatHasNoDismissKeyTests`, 'drags at the centre of a screen rotated since the last observation' (new), since the keyboard drag measures its start through `ViewportForAsync` too. - Upstream's input budget (`fix(mobile): bound each input by its budget`, `14b77d3`, tester-army/e2e#1073) is ported. An input (a tap or other node action, a point action, typing, a key, `back`, a keyboard dismissal, and the node `LongPressAsync`, `HoverAsync`, `DragToAsync`, and `SwipeAsync` the port has in place of upstream's `longPress`, `hover`, `dragTo`, and `swipe` actions) runs its device commands under `InputBudget` (`DeviceSupport.cs`, upstream's `support.ts`): with no budget left the input is not sent and fails with `OPERATION_TIMEOUT`, an input that does not answer in time fails with `ACTION_MAY_HAVE_COMMITTED`, one sent with almost no budget left still gets 250 ms to answer, a hold (a long press or timed swipe) at least as long as the budget left is refused with `NOT_ACTIONABLE`, and each command of a multi-command input (a key on an unfocused field) is sent on its own. Reads, launches, artifacts, and the `Device` fixture calls (including `device.dismissKeyboard`, which passes no budget, as upstream's `dismissKeyboard(signal)` does) are unbounded. Differences: upstream reads the budget per call, from `operation.timeoutMs`, which is the call's `timeout` or `actionTimeout`. The port's session calls carry no timeout, so every input gets `DeviceSession.ActionTimeout`, which `MobileStartInfo.ActionTimeout` sets (`EngineStartOptions.ActionTimeout` fills it), as the web engine's operations get the action timeout. `DeviceSession.InputBudget` is upstream's `inputBudget` and `DeviceSession.SendInputAsync` its `sendInput`. `InputBudget`'s `track` is an `Action` that registers the task (upstream's returns the promise it is given); see [In-flight commands](#in-flight-commands). The `OperationContext.signal` doc change in `docs/reference/engine.mdx` and `contract.ts` has no counterpart: the port has no `OperationContext`. Tests: in `Engine/ObservationTests`, 'ends %s, hung on %s, at its budget as possibly committed' (a theory of the same seven entries; the screen swipe is a `perform` on the root), 'sends nothing once a settling target spent the budget, and no keystroke once a focus tap used it up', and 'refuses a long press that holds longer than its timeout leaves, before sending it' (also on the node `LongPressAsync`); in `Support/SupportHelpersTests`, 'sends an input with what is left of its budget, and reports one cut off as possibly committed'. These run on real timers of a few hundred milliseconds, where upstream's fake timers do not wait. 'lets a locate outlive its budget' was already ported. -- Upstream's Compose Multiplatform text fields on iOS (`mobile reads Compose Multiplatform text fields on iOS`, `75ae2ab`, tester-army/e2e#1427) are ported in part. `SnapshotProjection.ProjectSnapshot` reads the placeholder an empty iOS field folds into its label (upstream's `foldedPlaceholder` in `nodes.ts`, `FoldedPlaceholder` here): for an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text, that text is the field's `Placeholder`, the child is left out of the tree, and the name is the label before `, ` (a non-empty `ContentDescription` stays the name, and that label is then the `Text`). A field of any other shape keeps its label and children as reported. The private record `Folded` is a .NET-only name for upstream's inline `{ label, placeholder, child }` object. Test: 'reads the placeholder an empty Compose Multiplatform field folds into its label' in `Nodes/SnapshotProjectionTests`. Not ported: `mayTakeText` and `NodeBinding.takesText` (`nodes.ts`, `surface.ts`), with the three `it`s that use them in `describe('input freshness')` of `engine.test.ts` ('acts on an unclassified iOS field', 'rejects an unclassified iOS element', 'rejects an unclassified iOS element that gained a control before typing'). The port's `fill` and `clear` have no check of the node's role and send the text to any node with a ref, so an iOS `other` is already typed into; the check upstream widens does not exist here. Adding it would be a new refusal, not this change; it is tracked in #850. +- Upstream's Compose Multiplatform text fields on iOS (`mobile reads Compose Multiplatform text fields on iOS`, `75ae2ab`, tester-army/e2e#1427) are ported in part. `SnapshotProjection.ProjectSnapshot` reads the placeholder an empty iOS field folds into its label (upstream's `foldedPlaceholder` in `nodes.ts`, `FoldedPlaceholder` here): for an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text, that text is the field's `Placeholder`, the child is left out of the tree, and the name is the label before `, ` (a non-empty `ContentDescription` stays the name, and that label is then the `Text`). A field of any other shape keeps its label and children as reported. The private record `Folded` is a .NET-only name for upstream's inline `{ label, placeholder, child }` object. Test: 'reads the placeholder an empty Compose Multiplatform field folds into its label' in `Nodes/SnapshotProjectionTests`. `mayTakeText` and `NodeBinding.takesText` (`nodes.ts`, `surface.ts`) are ported (#850): `SnapshotProjection.MayTakeText` and `NodeBinding.TakesText`, set in `Bind` from the node with its children. `DeviceActions.PerformAsync` refuses a `fill` or `clear` whose node does not take text with `UNSUPPORTED_CAPABILITY` (the message of upstream's `refuseFill`, `RefuseFill` here), before the action and again on the binding found by a fresh snapshot taken just before typing (upstream's `relocated`; `DeviceSession.RelocatedAsync`, with the transition wait split out as `WaitForTransitionAsync`, upstream's `waitForTransition`). Tests: 'Acts_on_an_unclassified_iOS_field', 'Rejects_an_unclassified_iOS_element', and 'Rejects_an_unclassified_iOS_element_that_gained_a_control_before_typing' in `Engine/InputFreshnessTests`. Differences: a fill or clear now reads the field again, so it costs one more `capture.snapshot` than before. Upstream's `relocated` also refuses, as `NODE_STALE`, a truncated capture or a changed input purpose, disabled, secure, or hidden state before typing; the port has no such check yet, so only the `takesText` re-check of this change is ported. The other `it`s of upstream's `describe('input freshness')` are not part of this change. - Upstream's WebDriver element-type fix (`fix(mobile): read XCUIElementType class names as their XCTest types`, `3f9a7d8`, tester-army/e2e#1133) is ported. `SnapshotProjection.NormalizeKind` is upstream's `normalizeKind` in `nodes.ts`: it drops `^XCUIElementType(?=[A-Z])` (`XcuiElementTypePrefix`, a .NET `[GeneratedRegex]` for upstream's inline regex) before it maps the type, so the `XCUIElementTypeButton` that agent-device's WebDriver runtime reports on hosted Appium clouds reads as `Button`. `DeviceSession.KeyboardShowing` is upstream's `keyboardShowing` in `surface.ts`: it routes node types through `NormalizeKind`, so an `XCUIElementTypeKeyboard` counts as a shown keyboard. Upstream records this only in its changeset; its skill references and docs say nothing about it, so `SKILL.md` and `docs/` have nothing to mirror. Tests: in `Nodes/SnapshotProjectionTests`, 'reads the XCUIElementType class names a WebDriver session reports as the native XCTest types'; in `Engine/DeviceFixtureDismissingAKeyboardThatHasNoDismissKeyTests`, 'reads an XCUIElementTypeKeyboard from a WebDriver session as a shown keyboard'. - Upstream's held-device skip (`mobile: skip booted devices another agent-device session holds`, `9e53c40`, tester-army/e2e#1379) is ported. `BootedDevices` (`DevicePool.cs`, upstream's class in `pool.ts`) skips a booted device whose inventory entry has a `claimedBy.session` that is not one of this target's own slot sessions, logs the skipped ones, and fails `prepare` with a non-retryable `ENGINE_FAILURE` when every booted device is held. `BootedDevice` is upstream's interface of that name, as a record. `LifecycleTests.Skips_a_booted_device_another_agent_device_session_holds_keeps_one_its_own_slot_session_holds_and_fails_when_every_booted_one_is_held` ports the `lifecycle` `it` of `engine.test.ts`, whole. - Not ported: the tool-pixels test (needs the full runner and a scripted model), and the `device.openApp` check of an unknown permission name. diff --git a/docs-site/src/content/docs/get-started/mobile.mdx b/docs-site/src/content/docs/get-started/mobile.mdx index 8f7ed64bd..e9f6fc737 100644 --- a/docs-site/src/content/docs/get-started/mobile.mdx +++ b/docs-site/src/content/docs/get-started/mobile.mdx @@ -82,7 +82,7 @@ public sealed class GreetingTests : MobileAppTest } ``` -`GetByTestId` finds a control by its accessibility identifier. A field's hint is its `Placeholder`, shown or not, so `GetByPlaceholder` finds it. Compose Multiplatform reports no placeholder on iOS: an empty field appends the text it draws to its label (`Name, Ada Lovelace`) over one `StaticText` child. For an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text (`Name, Ada Lovelace`), that text is the field's `Placeholder`, the child is left out of the tree, and the name is the label before `, ` (a non-empty `ContentDescription` stays the name, and that label is then the `Text`). A field of any other shape keeps its label and children as reported. So `GetByPlaceholder("Ada Lovelace")` finds such a field and `GetByText("Ada Lovelace")` does not. `Device` is the device fixture: `HomeAsync`, `OpenAppAsync`, `ForegroundAppAsync`, and `SetAppearanceAsync(Appearance.Dark)`. +`GetByTestId` finds a control by its accessibility identifier. A field's hint is its `Placeholder`, shown or not, so `GetByPlaceholder` finds it. Compose Multiplatform reports no placeholder on iOS: an empty field appends the text it draws to its label (`Name, Ada Lovelace`) over one `StaticText` child. For an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text (`Name, Ada Lovelace`), that text is the field's `Placeholder`, the child is left out of the tree, and the name is the label before `, ` (a non-empty `ContentDescription` stays the name, and that label is then the `Text`). A field of any other shape keeps its label and children as reported. So `GetByPlaceholder("Ada Lovelace")` finds such a field and `GetByText("Ada Lovelace")` does not. `FillAsync` and `ClearAsync` read the field again before typing into it, and refuse anything that is not an editable field with `UNSUPPORTED_CAPABILITY`, a button or a text view included. An iOS `other`, the type of an element the platform did not classify, is typed into when it holds a value or only text, since a custom-drawn field (Compose Multiplatform, Flutter) can read that way; an `other` with no value and no children, such as a React Native `Pressable`, or one that holds a control, stays refused. `Device` is the device fixture: `HomeAsync`, `OpenAppAsync`, `ForegroundAppAsync`, and `SetAppearanceAsync(Appearance.Dark)`. ```csharp await Device.HomeAsync(); diff --git a/skills/e2e-dotnet/SKILL.md b/skills/e2e-dotnet/SKILL.md index 3df4c3a02..4a35eb50d 100644 --- a/skills/e2e-dotnet/SKILL.md +++ b/skills/e2e-dotnet/SKILL.md @@ -35,7 +35,7 @@ trip. The installed tool prints the same text with `e2e guide ` | Running tests: `dotnet test` filters, the `e2e` tool, `report.json` for scripts, exit codes, CI, test filters (not wired into `dotnet test`: the `ITestFilter` seam and the `run.changes` and `run.filters` report fields) | `running` | [references/running.md](references/running.md) | | Handling a failed run: the trace page, `report.json` for scripts, `E2ESession.Report()` for a session you run yourself, error codes and their fixes, a visible browser, replay misses, phase timings, the agent step table, every model call | `debugging` | [references/debugging.md](references/debugging.md) | | Exploring an app toward a goal without a test file, with `e2e explore` or `Explore.ExploreAsync` | `explore` | [references/explore.md](references/explore.md) | -| Testing an iOS or Android app: `MobileEngine`, the `Device` fixture, picking a simulator (without `MobileOptions.Device`, `MobileEngine` skips booted devices another agent-device session holds, and `prepare` fails with `ENGINE_FAILURE` when every booted one is held), video of the device screen, `Device.UnstableRaw` (the test's agent-device session: name, device id and name, selection, and a raw `IDeviceClient` whose calls are not steps, replayed, or redacted; unstable; `INVALID_STATE` after the session is disposed), Limrun hosted devices, a MAUI app through the DevFlow agent, known limits, `Locator.PressAsync("Enter")` tapping an unfocused field first, and the agent's `open_app` returning to the app under test when it names no app, a Compose Multiplatform field on iOS (for an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text, that text is the field's `Placeholder`, the child is left out, and the name is the label before `, `; a non-empty `ContentDescription` stays the name, and that label is then the `Text`; any other field keeps its label and children) | `mobile` | [references/mobile.md](references/mobile.md) | +| Testing an iOS or Android app: `MobileEngine`, the `Device` fixture, picking a simulator (without `MobileOptions.Device`, `MobileEngine` skips booted devices another agent-device session holds, and `prepare` fails with `ENGINE_FAILURE` when every booted one is held), video of the device screen, `Device.UnstableRaw` (the test's agent-device session: name, device id and name, selection, and a raw `IDeviceClient` whose calls are not steps, replayed, or redacted; unstable; `INVALID_STATE` after the session is disposed), Limrun hosted devices, a MAUI app through the DevFlow agent, known limits, `Locator.PressAsync("Enter")` tapping an unfocused field first, and the agent's `open_app` returning to the app under test when it names no app, a Compose Multiplatform field on iOS (for an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text, that text is the field's `Placeholder`, the child is left out, and the name is the label before `, `; a non-empty `ContentDescription` stays the name, and that label is then the `Text`; any other field keeps its label and children), and `FillAsync` and `ClearAsync` refusing a node that is not an editable field with `UNSUPPORTED_CAPABILITY` (an iOS `other` that holds a value or only text is typed into) | `mobile` | [references/mobile.md](references/mobile.md) | | Testing a Windows desktop app: `WinAppEngine`, `WinAppTest`, the `Window` fixture, the `windows` target in `e2e explore` and `e2e mcp`, the interactive-desktop requirement, known limits | `windows` | [references/windows.md](references/windows.md) | | Asked to bug bash or hunt for bugs: proving each bug with a repro test | `bug-bash` | [references/bug-bash.md](references/bug-bash.md) | diff --git a/skills/e2e-dotnet/references/mobile.md b/skills/e2e-dotnet/references/mobile.md index 532fa8335..4b3051c9e 100644 --- a/skills/e2e-dotnet/references/mobile.md +++ b/skills/e2e-dotnet/references/mobile.md @@ -168,6 +168,7 @@ Limits of the preview: the tests attach to the running app, so `Device.InstallAp ## Known limits - Compose Multiplatform reports no placeholder on iOS: an empty field appends the text it draws to its label (`Name, Ada Lovelace`) over one `StaticText` child. For an empty (no value), editable, non-secure iOS field that reports no placeholder of its own, whose only child is a `StaticText` with no children, and whose label is a non-empty name, then `, `, then that child's text (`Name, Ada Lovelace`), that text is the field's `Placeholder`, the child is left out of the tree, and the name is the label before `, ` (a non-empty `ContentDescription` stays the name, and that label is then the `Text`). A field of any other shape keeps its label and children as reported. So `GetByPlaceholder("Ada Lovelace")` finds such a field and `GetByText("Ada Lovelace")` does not. +- `FillAsync` and `ClearAsync` read the field again before typing into it, and refuse anything that is not an editable field with `UNSUPPORTED_CAPABILITY`, a button or a text view included. An iOS `other`, the type of an element the platform did not classify, is typed into when it holds a value or only text, since a custom-drawn field (Compose Multiplatform, Flutter) can read that way; an `other` with no value and no children, such as a React Native `Pressable`, or one that holds a control, stays refused. Typing into a button or a text view that worked before now fails. - On iOS, a plain React Native `View` has no children in the tree, so `Filter(has: ...)` on it and queries scoped to it match nothing, while `Filter(hasNot: ...)` keeps every such row, including one you meant to exclude. Scope to the screen's `ScrollView` (`Device.Locator("role=scroll-area")`), or give the children test ids. - `Locator.PressAsync` takes `Enter`, `Space`, or one character, sent through the soft keyboard. A field that is not focused is tapped first, for `Enter` as for a character, so the keyboard is up. Other named keys and every modifier fail with `UNSUPPORTED_CAPABILITY`. - Android needs `adb` from the Android SDK Platform-Tools on `PATH`, locally and for the Limrun tunnel. diff --git a/src/E2E.Mobile/DeviceActions.cs b/src/E2E.Mobile/DeviceActions.cs index 305371cfe..03a65a7b0 100644 --- a/src/E2E.Mobile/DeviceActions.cs +++ b/src/E2E.Mobile/DeviceActions.cs @@ -91,6 +91,12 @@ public async Task PerformAsync(DeviceSession session, SemanticNode node, Locator } var entry = session.Resolve(node); + var filling = action is LocatorAction.Fill or LocatorAction.Clear; + if (filling && !entry.TakesText) + { + throw RefuseFill(action, entry); + } + var label = $"perform {KindOf(action)}"; // A toggle already in the wanted state sends nothing, so it neither waits for a transition nor counts as an @@ -108,13 +114,35 @@ await session.ActAsync( async (client, token) => { var settle = session.ActionSettle(); - var target = session.Origin == ActionOrigin.Test ? await session.SettledAsync(entry, token).ConfigureAwait(false) : entry; + NodeBinding target; + if (filling) + { + // The field is read again before it is typed into, and what it reads as may have changed. + if (session.Origin == ActionOrigin.Test) + { + await session.WaitForTransitionAsync(entry, token).ConfigureAwait(false); + } + + target = await session.RelocatedAsync(entry, token).ConfigureAwait(false); + if (!target.TakesText) + { + throw RefuseFill(action, target); + } + } + else + { + target = session.Origin == ActionOrigin.Test ? await session.SettledAsync(entry, token).ConfigureAwait(false) : entry; + } + await RunAsync(session, client, target, action, settle, input, token).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false); session.MarkAction(before); } + private static EngineException RefuseFill(LocatorAction action, NodeBinding binding) => + DeviceSupport.Unsupported($"agent-device can only {KindOf(action)} editable fields; node {binding.Id} is {binding.Node.Role ?? "unknown"}"); + private static async Task RunAsync(DeviceSession session, IDeviceClient client, NodeBinding target, LocatorAction action, JsonObject settle, InputBudget input, CancellationToken cancellationToken) { switch (action) diff --git a/src/E2E.Mobile/DeviceSession.Screen.cs b/src/E2E.Mobile/DeviceSession.Screen.cs index a57a7d466..2cf887384 100644 --- a/src/E2E.Mobile/DeviceSession.Screen.cs +++ b/src/E2E.Mobile/DeviceSession.Screen.cs @@ -21,8 +21,9 @@ public enum ActionOrigin /// /// Action data only: a bound node never retains a projected snapshot through parent links. RefsGeneration is the /// capture Ref and ControlRef came from, pinned on every action so agent-device refuses them once a newer capture supersedes it. +/// TakesText is whether fill and clear may type into the node, judged with the children Node leaves out. /// -internal sealed record NodeBinding(string Id, string Ref, SemanticNode Node, string ControlRef, IReadOnlyDictionary States, long? RefsGeneration); +internal sealed record NodeBinding(string Id, string Ref, SemanticNode Node, string ControlRef, IReadOnlyDictionary States, long? RefsGeneration, bool TakesText); public sealed partial class DeviceSession { @@ -276,7 +277,7 @@ private static int DepthBelow(ProjectedNode entry, ProjectedNode ancestor) /// Copies the fields actions use and pre-resolves a toggle's inner control. private static NodeBinding Bind(ProjectedNode entry, IReadOnlyList snapshot, long? refsGeneration) => - new(entry.Id, entry.Ref, entry.Node, ControlOf(entry, snapshot).Ref, entry.States, refsGeneration); + new(entry.Id, entry.Ref, entry.Node, ControlOf(entry, snapshot).Ref, entry.States, refsGeneration, SnapshotProjection.MayTakeText(entry.Node)); /// The same control in the newest capture: the binding itself when it came from that capture, else found there again where it was. internal NodeBinding Current(NodeBinding entry) @@ -348,18 +349,27 @@ a is not null && b is not null /// the control landed: Android reports a window's frames while it slides in, so a ref resolved mid-flight would /// send the action where the control was. /// - internal async Task SettledAsync(NodeBinding entry, CancellationToken cancellationToken) + internal async Task SettledAsync(NodeBinding entry, CancellationToken cancellationToken) => + await WaitForTransitionAsync(entry, cancellationToken).ConfigureAwait(false) + ? await RelocatedAsync(entry, cancellationToken).ConfigureAwait(false) + : entry; + + /// + /// Waits out the remainder of the transition budget when the control came with the last action. True when it waited, + /// so the binding may be stale; false when the control was already in place. + /// + internal async Task WaitForTransitionAsync(NodeBinding entry, CancellationToken cancellationToken) { var since = _engine.Clock.GetUtcNow() - _lastActionAt; var remaining = TimeSpan.FromMilliseconds(Options.TransitionMilliseconds) - since; if (remaining <= TimeSpan.Zero) { - return entry; + return false; } if (_indexBeforeAction is { } before && SamePlace(entry, before) is not null) { - return entry; + return false; } try @@ -378,11 +388,11 @@ internal async Task SettledAsync(NodeBinding entry, CancellationTok throw DeviceSupport.Cancelled("transition wait cancelled"); } - return await RelocatedAsync(entry, cancellationToken).ConfigureAwait(false); + return true; } /// The same control in a fresh snapshot. A snapshot that no longer lists it is a retryable NODE_STALE, so the locator resolves again. - private async Task RelocatedAsync(NodeBinding entry, CancellationToken cancellationToken) + internal async Task RelocatedAsync(NodeBinding entry, CancellationToken cancellationToken) { var projected = Project(await SnapshotOrEmptyAsync(false, cancellationToken).ConfigureAwait(false)); return Refind(entry, projected.Index) is { } found diff --git a/src/E2E.Mobile/SnapshotProjection.cs b/src/E2E.Mobile/SnapshotProjection.cs index 5f5c557c9..726670c18 100644 --- a/src/E2E.Mobile/SnapshotProjection.cs +++ b/src/E2E.Mobile/SnapshotProjection.cs @@ -641,6 +641,34 @@ private static bool ShowsHint(RawNode source, bool android, bool editable) return new Folded(label[..^(placeholder.Length + 2)], placeholder, child); } + /// + /// Whether a node may take typed text: a text input role, or an other (only iOS reports one, for an element it + /// did not classify) that holds a value or nothing but text. agent-device 0.21.33's simulator accessibility tree + /// reports a custom-drawn field (Compose Multiplatform, Flutter) that way: a filled one with its text as the value, + /// an empty one over the placeholder it draws. A childless, valueless other, such as a React Native + /// Pressable, and a container of controls stay refused. + /// + public static bool MayTakeText(SemanticNode node) + { + if (node.Role is null) + { + return false; + } + + if (TextInputRoles.Contains(node.Role)) + { + return true; + } + + if (node.Role != "other") + { + return false; + } + + return node.Value is not null + || (node.Children.Count > 0 && node.Children.All(child => child.Role == "text" && child.Children.Count == 0)); + } + /// The string itself, or null for an absent or empty one. private static string? NonEmpty(string? value) => string.IsNullOrEmpty(value) ? null : value; diff --git a/tests/E2E.Mobile.Tests/Engine/InputFreshnessTests.cs b/tests/E2E.Mobile.Tests/Engine/InputFreshnessTests.cs new file mode 100644 index 000000000..427115158 --- /dev/null +++ b/tests/E2E.Mobile.Tests/Engine/InputFreshnessTests.cs @@ -0,0 +1,132 @@ +// Copyright 2026 TesterArmy. +// Modified by Dario Kondratiuk. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text.Json.Nodes; +using E2E.Engine; +using static E2E.Mobile.Tests.RawNodes; +using static E2E.Mobile.Tests.Screens; + +namespace E2E.Mobile.Tests.Engine; + +/// Upstream's describe('input freshness') of engine.test.ts; only the commit's three its are here. +public sealed class InputFreshnessTests +{ + /// A uniquely identified field and its neighbor, as upstream's form, with the field's type and what it holds named. + private static JsonObject Form(RawNode field, RawNode? inside = null) + { + var account = field with { Ref = "@e20", Index = 1, ParentIndex = 0, Depth = 1, Label = "Account", Identifier = "account", Rect = R(40, 237, 322, 49) }; + List nodes = + [ + SettingsNodes[0], + account, + N("@e21", index: 2, parentIndex: 0, depth: 1, type: "text-field", label: "Username", identifier: "username", rect: R(40, 218, 322, 49)), + ]; + if (inside is not null) + { + nodes.Add(inside); + } + + return FakeDeviceClient.SnapshotOf(nodes); + } + + // agent-device 0.21.33's simulator accessibility tree reads a Compose Multiplatform field as an unclassified + // `Other`: an empty one over the placeholder it draws, a filled one with its text as the value. + private static JsonObject Unclassified(string? value = null, RawNode? inside = null) => Form(N(type: "Other", value: value), inside); + + private static RawNode Child(string type, string label) => + N("@e22", index: 3, parentIndex: 1, depth: 2, type: type, label: label, rect: R(52, 252, 101, 18)); + + private static LocatorAction Act(string kind) => kind == "fill" ? new LocatorAction.Fill("example") : new LocatorAction.Clear(); + + public static TheoryData Cases(params string[] names) + { + var data = new TheoryData(); + foreach (var name in names) + { + foreach (var origin in new[] { ActionOrigin.Test, ActionOrigin.Agent }) + { + foreach (var kind in new[] { "fill", "clear" }) + { + data.Add(origin, kind, name); + } + } + } + + return data; + } + + public static TheoryData ActsCases() => Cases("empty, over its placeholder", "filled"); + + public static TheoryData RejectsCases() => + Cases("childless and valueless, like a React Native Pressable", "holding a control"); + + [Theory] + [MemberData(nameof(ActsCases))] + public async Task Acts_on_an_unclassified_iOS_field(ActionOrigin origin, string kind, string field) + { + var h = Harness.Create(); + var session = await OpenAttempt(h); + session.Origin = origin; + var snapshot = field == "filled" ? Unclassified("Grace") : Unclassified(inside: Child("static-text", "Ada Lovelace")); + h.Fake.Respond(DeviceMethods.CaptureSnapshot, _ => snapshot.DeepClone()); + var observed = await Observed(session, "Account"); + + await session.PerformAsync(observed, Act(kind), CancellationToken.None); + + JsonAssert.Args(kind == "fill" ? """{"ref":"@e20","text":"example"}""" : """{"ref":"@e20","text":""}""", h.Fake.LastArgs("interactions.fill") is { } args ? Strip(args) : null); + } + + [Theory] + [MemberData(nameof(RejectsCases))] + public async Task Rejects_an_unclassified_iOS_element(ActionOrigin origin, string kind, string element) + { + var h = Harness.Create(); + var session = await OpenAttempt(h); + session.Origin = origin; + var snapshot = element == "holding a control" ? Unclassified(inside: Child("button", "Clear")) : Unclassified(); + h.Fake.Respond(DeviceMethods.CaptureSnapshot, _ => snapshot.DeepClone()); + var observed = await Observed(session, "Account"); + + var failure = await Assert.ThrowsAsync(() => session.PerformAsync(observed, Act(kind), CancellationToken.None)); + + Assert.Equal("UNSUPPORTED_CAPABILITY", failure.Code); + Assert.False(failure.Retryable); + Assert.DoesNotContain("interactions.fill", h.Fake.Methods()); + } + + [Theory] + [MemberData(nameof(GainedCases))] + public async Task Rejects_an_unclassified_iOS_element_that_gained_a_control_before_typing(ActionOrigin origin, string kind) + { + var h = Harness.Create(); + var session = await OpenAttempt(h); + session.Origin = origin; + h.Fake.Respond(DeviceMethods.CaptureSnapshot, _ => Unclassified(inside: Child("static-text", "Loading"))); + var observed = await Observed(session, "Account"); + h.Fake.Respond(DeviceMethods.CaptureSnapshot, _ => Unclassified(inside: Child("button", "Loading"))); + + var failure = await Assert.ThrowsAsync(() => session.PerformAsync(observed, Act(kind), CancellationToken.None)); + + Assert.Equal("UNSUPPORTED_CAPABILITY", failure.Code); + Assert.False(failure.Retryable); + Assert.DoesNotContain("interactions.fill", h.Fake.Methods()); + } + + public static TheoryData GainedCases() + { + var data = new TheoryData(); + foreach (var origin in new[] { ActionOrigin.Test, ActionOrigin.Agent }) + { + foreach (var kind in new[] { "fill", "clear" }) + { + data.Add(origin, kind); + } + } + + return data; + } + + /// The ref and text of a fill call, the way upstream's toMatchObject reads them: settle options are not part of the assertion. + private static JsonObject Strip(JsonObject args) => new() { ["ref"] = args["ref"]?.DeepClone(), ["text"] = args["text"]?.DeepClone() }; +} diff --git a/tests/E2E.Mobile.Tests/Engine/PerformTests.cs b/tests/E2E.Mobile.Tests/Engine/PerformTests.cs index 73398c5cd..f5f98cbc0 100644 --- a/tests/E2E.Mobile.Tests/Engine/PerformTests.cs +++ b/tests/E2E.Mobile.Tests/Engine/PerformTests.cs @@ -45,7 +45,11 @@ public async Task Maps_every_supported_action_onto_one_agent_device_command() ("interactions.longPress", """{"ref":"@e4","settle":true,"settleQuietMs":150,"durationMs":1000}"""), ("interactions.press", """{"ref":"@e7","settle":true,"settleQuietMs":150}"""), ("interactions.hover", """{"ref":"@e4"}"""), + + // A fill or clear reads the field again before typing into it. + ("capture.snapshot", """{"interactiveOnly":false}"""), ("interactions.fill", """{"ref":"@e7","text":"blue","settle":true,"settleQuietMs":150}"""), + ("capture.snapshot", """{"interactiveOnly":false}"""), ("interactions.fill", """{"ref":"@e7","text":"","settle":true,"settleQuietMs":150}"""), ("interactions.press", """{"ref":"@e6","settle":true,"settleQuietMs":150}"""),