diff --git a/CHANGELOG.md b/CHANGELOG.md index caddc6b..19f0f7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- **`set_value` checks that the value took.** On a Chrome drop-down list (``) do, so expand the list +and select the option, or click it. A field that takes the text and clears itself, +such as a tag field, fails the same way although it took the text, so inspect +before you retry. A control that reformats the value succeeds, and its result's +`value` is what the control now reads. Successful action submission is not proof that an application completed its work; inspect again to confirm the result, including possible modal dialogs. diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index 203563a..7aaef36 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -178,7 +178,7 @@ is still restoring when the operation's deadline passes, it fails with `errorCod | `desktop.windows` | `query`, `pid` | `{windows: [{windowId, pid, process, title, bounds, foreground, minimized, maximized}], summary}` | | `desktop.observe` | `windowId`, `maxElements`, `maxDepth`, `maxTextChars`, `includeOffscreen`, `includeScreenshot`, `maxWidth` | `{windowId, snapshotId, expiresInSeconds, window, elements, observedAt, truncated, warnings, screenshot?, screenshotError?, summary}` | | `desktop.window` | `windowId`, `action`, optional move geometry `x`, `y`, `width`, `height` | `{requested, note, summary}` | -| `desktop.element` | `snapshotId`, `elementId`, `action`, optional `value`, `number`, `direction`, `amount` | `{performed, note, summary}` | +| `desktop.element` | `snapshotId`, `elementId`, `action`, optional `value`, `number`, `direction`, `amount` | `{performed, note, value?, summary}`; `value` is what the control reads after a `set_value` it reformatted | | `desktop.capabilities` | `probeCapture` (default true) | `{version, session, workingDirectory, screen, capture, input, supported, limitations, summary}` | | `desktop.wait` | `windowId`, selectors `automationId`, `name`, `role`, `textContains`, `state`, `waitMs` | `{matched, waitElapsedMs, ...observation, summary}` | diff --git a/src/Anode/Cli/DesktopChecks.cs b/src/Anode/Cli/DesktopChecks.cs index 1889d50..b71482b 100644 --- a/src/Anode/Cli/DesktopChecks.cs +++ b/src/Anode/Cli/DesktopChecks.cs @@ -119,13 +119,10 @@ public static string Presentation() return "HTML report escapes untrusted app text; text summary keeps automation IDs, bounds and capture geometry"; } - /// - /// A hidden window, never shown, whose accessibility provider says it is an AppBar (UIA control type 50040), as File - /// Explorer's command bar does. .NET's UI Automation client has no name for that type. - /// - private sealed class AppBarWindow : NativeWindow, IRawElementProviderSimple + /// A hidden window, never shown, that answers UI Automation with its own provider. + private abstract class HiddenWindow : NativeWindow, IRawElementProviderSimple { - private const int WmGetObject = 0x003D, UiaRootObjectId = -25, AppBarControlTypeId = 50040; + private const int WmGetObject = 0x003D, UiaRootObjectId = -25; public const int Quit = 0x8002; // WM_APP + 2 protected override void WndProc(ref Message m) @@ -140,59 +137,160 @@ protected override void WndProc(ref Message m) } public ProviderOptions ProviderOptions => ProviderOptions.ServerSideProvider; - public object? GetPatternProvider(int patternId) => null; - public object? GetPropertyValue(int propertyId) => + public abstract object? GetPatternProvider(int patternId); + public abstract object? GetPropertyValue(int propertyId); + public IRawElementProviderSimple HostRawElementProvider => AutomationInteropProvider.HostProviderFromHandle(Handle); + } + + /// A control that says it is an AppBar (UIA control type 50040), as File Explorer's command bar does. + private sealed class AppBarWindow : HiddenWindow + { + private const int AppBarControlTypeId = 50040; + public override object? GetPatternProvider(int patternId) => null; + public override object? GetPropertyValue(int propertyId) => propertyId == AutomationElement.ControlTypeProperty.Id ? AppBarControlTypeId : propertyId == AutomationElement.NameProperty.Id ? "Command bar" : null; - public IRawElementProviderSimple HostRawElementProvider => AutomationInteropProvider.HostProviderFromHandle(Handle); + } + + /// + /// A drop-down list, or a text field, that takes set_value the way it is told to: applies it, ignores it as Chrome's + /// <select> does, applies it a moment later as Chrome's text fields do, reformats it, or goes away. + /// + private sealed class ValueWindow : HiddenWindow, IValueProvider + { + private volatile string _mode = "apply", _value = "Free"; + private volatile bool _sent, _field; + public void Use(string mode, bool field = false) { _mode = mode; _field = field; _value = "Free"; _sent = false; } + public override object? GetPatternProvider(int patternId) => patternId == ValuePattern.Pattern.Id ? this : null; + public override object? GetPropertyValue(int propertyId) => + propertyId == AutomationElement.ControlTypeProperty.Id ? (_field ? ControlType.Edit.Id : ControlType.ComboBox.Id) + : propertyId == AutomationElement.NameProperty.Id ? "Plan" : null; + public bool IsReadOnly => false; + public string Value => _mode == "vanish" && _sent ? throw new ElementNotAvailableException() : _value; + public void SetValue(string value) + { + _sent = true; + switch (_mode) + { + case "ignore": break; + case "late": _ = Task.Delay(300).ContinueWith(_ => _value = value, TaskScheduler.Default); break; + case "reformat": _value = value.ToUpperInvariant(); break; + default: _value = value; break; + } + } } [System.Runtime.InteropServices.DllImport("user32.dll")] [return: System.Runtime.InteropServices.MarshalAs(System.Runtime.InteropServices.UnmanagedType.Bool)] private static extern bool PostMessage(IntPtr window, int message, IntPtr wParam, IntPtr lParam); - public static string UnknownControlTypes() + /// + /// Runs a check against a hidden window of this process on its own STA thread. The window is this process's, so it + /// passes the seat's session checks; the check runs off that thread, since UI Automation must not run on an STA + /// thread, as in the seat's worker. + /// + private static string Hosted(HiddenWindow window, string caption, Func check) { - Require(AccessibilityReader.Role(ControlType.Button) == "Button", "a known control type lost its name"); - var window = new AppBarWindow(); using var created = new ManualResetEventSlim(); Exception? failed = null; var thread = new Thread(() => { - try { window.CreateHandle(new CreateParams { Caption = "Anode unknown control type check", Style = unchecked((int)0x80000000) }); } // WS_POPUP, never visible + try { window.CreateHandle(new CreateParams { Caption = caption, Style = unchecked((int)0x80000000) }); } // WS_POPUP, never visible catch (Exception ex) { failed = ex; created.Set(); return; } created.Set(); Application.Run(); window.DestroyHandle(); - }) { IsBackground = true, Name = "unknown control type window" }; + }) { IsBackground = true, Name = caption }; thread.SetApartmentState(ApartmentState.STA); thread.Start(); Require(created.Wait(5000) && failed is null, "the check's hidden window was not created: " + failed?.Message); - T Within(Func work) => Task.Run(work).WaitAsync(TimeSpan.FromSeconds(10)).GetAwaiter().GetResult(); try { - // UI Automation must not run on an STA thread, as in the seat's worker. The window is this process's, so it - // passes the seat's session checks. var target = WindowAccess.Identify(window.Handle); - var observation = Within(() => AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 1 })); - var root = observation["elements"]?.AsArray().FirstOrDefault()?.AsObject() - ?? throw new InvalidOperationException("the observation has no elements: " + observation.ToJsonString()); + return Task.Run(() => check(target)).WaitAsync(TimeSpan.FromSeconds(20)).GetAwaiter().GetResult(); + } + finally + { + PostMessage(window.Handle, HiddenWindow.Quit, IntPtr.Zero, IntPtr.Zero); + thread.Join(5000); + } + } + + private static JsonObject Root(JsonObject observation) => observation["elements"]?.AsArray().FirstOrDefault()?.AsObject() + ?? throw new InvalidOperationException("the observation has no elements: " + observation.ToJsonString()); + + public static string UnknownControlTypes() + { + Require(AccessibilityReader.Role(ControlType.Button) == "Button", "a known control type lost its name"); + return Hosted(new AppBarWindow(), "Anode unknown control type check", target => + { + var root = Root(AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 1 })); string expected = AccessibilityReader.Role(ControlType.LookupById(50040)); Require(root.Str("role") == expected && root.Str("name") == "Command bar", $"an AppBar control was described as {root.Str("role")} \"{root.Str("name")}\", not {expected} \"Command bar\""); // Acting on it compares the role again; an action it doesn't offer must be refused after that comparison. try { - Within(() => AccessibilityReader.Act(target, root, new JsonObject { ["action"] = "invoke" })); + AccessibilityReader.Act(target, root, new JsonObject { ["action"] = "invoke" }); throw new InvalidOperationException("an action the control does not offer was performed"); } catch (InvalidOperationException error) when (error.Message.Contains("not offered", StringComparison.Ordinal)) { } return $"a control type .NET's client can't name (AppBar, as in File Explorer) is observed as {expected}, and actions on it reach their checks"; - } - finally + }); + } + + public static string IgnoredValues() + { + var window = new ValueWindow(); + return Hosted(window, "Anode set_value check", target => { - PostMessage(window.Handle, AppBarWindow.Quit, IntPtr.Zero, IntPtr.Zero); - thread.Join(5000); - } + var list = Root(AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 0 })); + Require(list["actions"]!.AsArray().Any(action => action?.GetValue() == "set_value"), "the list does not offer set_value"); + JsonObject Set(string mode, JsonObject? control = null) + { + window.Use(mode, field: control is not null); + return AccessibilityReader.Act(target, control ?? list, new JsonObject { ["action"] = "set_value", ["value"] = "Team" }); + } + foreach (string mode in new[] { "apply", "late" }) + { + var done = Set(mode); + Require(done.Str("performed") == "set_value" && done["value"] is null && window.Value == "Team", + $"a value the control {(mode == "late" ? "applied a moment later" : "applied")} was reported as {done.ToJsonString()}"); + } + var reformatted = Set("reformat"); + Require(reformatted.Str("performed") == "set_value" && reformatted.Str("value") == "TEAM" + && DesktopPresentation.Summary(reformatted).EndsWith("It now reads: \"TEAM\"", StringComparison.Ordinal), + "a value the control reformatted was not reported as it now reads: " + reformatted.ToJsonString()); + var watch = Stopwatch.StartNew(); + try + { + var ignored = Set("ignore"); + throw new InvalidOperationException("a value the control ignored was reported as " + ignored.ToJsonString()); + } + catch (InvalidOperationException error) when (error.Message.Contains("still showed its old value", StringComparison.Ordinal)) + { + // Errors are logged, so the message must not quote the control's text; a list gets the way to choose. + Require(!error.Message.Contains("Free", StringComparison.Ordinal) && error.Message.Contains("expand the list", StringComparison.Ordinal), + "the failure quoted the control's text or left out the drop-down advice: " + error.Message); + } + Require(window.Value == "Free" && watch.ElapsedMilliseconds >= AccessibilityReader.SetValueSettleMs - 100, + "an ignored value was refused before the control had time to apply it"); + // The value went out before the control went away, so that must not read as "nothing happened". + var unread = Set("vanish"); + Require(unread.Str("performed") == "set_value" && unread.Str("note")!.Contains("could not be read back", StringComparison.Ordinal), + "a control that went away after set_value was reported as " + unread.ToJsonString()); + window.Use("ignore", field: true); + var field = Root(AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 0 })); + try + { + var ignored = Set("ignore", field); + throw new InvalidOperationException("a value the text field ignored was reported as " + ignored.ToJsonString()); + } + catch (InvalidOperationException error) when (error.Message.Contains("still showed its old value", StringComparison.Ordinal)) + { + Require(!error.Message.Contains("expand the list", StringComparison.Ordinal), "a text field was told to expand a list: " + error.Message); + } + return "set_value reads the control back: an ignored value (as Chrome's