From f08518fee793ffd183c1d8d406504b4b82a2d9ba Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:24:05 +0100 Subject: [PATCH 1/3] Fix: set_value reports "performed" when the control ignores it (B-026) seat_element set_value on a Chrome `), `seat_element` + `set_value` reported `performed` while the list kept its old choice, so an agent could submit a + form with the wrong option. Anode now reads the control back: if it still shows its old value 1.5 + seconds later, the action fails and says to expand the list and select the option. A control that + reformats the value succeeds and returns what it now reads. - **Packaged apps' Anode folders.** A daemon started by a desktop app that redirects AppData, such as the Claude desktop app, keeps its log and state in that app's `%LOCALAPPDATA%\Packages\\LocalCache\Local\Anode`, which the privacy, install and troubleshooting diff --git a/docs/DESKTOP-TOOLS.md b/docs/DESKTOP-TOOLS.md index 1277a94..f135081 100644 --- a/docs/DESKTOP-TOOLS.md +++ b/docs/DESKTOP-TOOLS.md @@ -63,6 +63,11 @@ by `--`, for example `--direction down --amount large` or `--number 50`. Only use actions listed on that control. Read-only fields do not offer value changes. Password controls omit their values and offer no element actions. +`set_value` reads the control back. If it still shows its old value 1.5 seconds +later, the action fails instead of reporting success: some controls accept the +change and ignore it. Chrome's drop-down lists (`) fails instead of reporting performed; applied, late and reformatted values pass"; + }); } } diff --git a/src/Anode/Cli/SelfTest.cs b/src/Anode/Cli/SelfTest.cs index deedcc2..7ca4ba8 100644 --- a/src/Anode/Cli/SelfTest.cs +++ b/src/Anode/Cli/SelfTest.cs @@ -131,6 +131,7 @@ public static int Run(string[] args) Check(results, "desktop worker deadline", () => DesktopChecks.WorkerTimeout().GetAwaiter().GetResult()); Check(results, "desktop report escaping", DesktopChecks.Presentation); Check(results, "desktop unknown control types", DesktopChecks.UnknownControlTypes); + Check(results, "desktop ignored set_value", DesktopChecks.IgnoredValues); Check(results, "execution output", DevelopmentChecks.Output); Check(results, "execution jobs", () => DevelopmentChecks.Jobs().GetAwaiter().GetResult()); Check(results, "cancelled execution", () => DevelopmentChecks.CancelledStart().GetAwaiter().GetResult()); diff --git a/src/Anode/Core/Desktop/AccessibilityReader.cs b/src/Anode/Core/Desktop/AccessibilityReader.cs index 17aeeae..8e9ac01 100644 --- a/src/Anode/Core/Desktop/AccessibilityReader.cs +++ b/src/Anode/Core/Desktop/AccessibilityReader.cs @@ -192,7 +192,7 @@ T Pattern(AutomationPattern id) where T : class => element.TryGetCurrentPatte { case "invoke": Pattern(InvokePattern.Pattern).Invoke(); break; case "focus": element.SetFocus(); break; - case "set_value": Pattern(ValuePattern.Pattern).SetValue(request.Str("value")!); break; + case "set_value": return SetValue(Pattern(ValuePattern.Pattern), request.Str("value")!); case "toggle": Pattern(TogglePattern.Pattern).Toggle(); break; case "select": Pattern(SelectionItemPattern.Pattern).Select(); break; case "expand": Pattern(ExpandCollapsePattern.Pattern).Expand(); break; @@ -208,7 +208,53 @@ T Pattern(AutomationPattern id) where T : class => element.TryGetCurrentPatte case "set_range": Pattern(RangeValuePattern.Pattern).SetValue(request["number"]!.GetValue()); break; default: throw new ArgumentException("Unknown element action."); } - return new JsonObject { ["performed"] = action, ["note"] = "Observation consumed. Inspect again before the next action." }; + return Performed(action); + } + + private static JsonObject Performed(string action) => + new() { ["performed"] = action, ["note"] = "Observation consumed. Inspect again before the next action." }; + + /// How long set_value watches for the control to take its new value. Chrome shows a change a moment later. + internal const int SetValueSettleMs = 1500; + + /// + /// Sets a value and reads it back. Some controls accept the call and ignore it, as Chrome's drop-down lists + /// (<select>) do, and "performed" would then let an agent submit a form with the old choice. + /// + private static JsonObject SetValue(ValuePattern pattern, string value) + { + string before = pattern.Current.Value ?? ""; + pattern.SetValue(value); + string now; + var watch = Stopwatch.StartNew(); + try + { + now = pattern.Current.Value ?? ""; + while (now == before && now != value && watch.ElapsedMilliseconds < SetValueSettleMs) + { + Thread.Sleep(50); + now = pattern.Current.Value ?? ""; + } + } + catch (ElementNotAvailableException) + { + var gone = Performed("set_value"); + gone["note"] = "The control went away after the change, so its new value wasn't read back. Inspect again before the next action."; + return gone; + } + if (now == value) return Performed("set_value"); + if (now != before) + { + // An app may reformat what it was given, for example a number field that adds its separators. + var changed = Performed("set_value"); + changed["value"] = Clip(now, 500); + changed["note"] = "The control changed the value it was given; value is what it reads now. Observation consumed. Inspect again before the next action."; + return changed; + } + throw new InvalidOperationException( + $"The control accepted set_value but still read \"{Clip(before, 200)}\" after {SetValueSettleMs} ms, so the new value " + + "wasn't applied, or the control doesn't show it. Some controls ignore set_value, as Chrome's drop-down lists do: " + + "expand the list and select the option, or click it. Inspect again before you retry."); } /// The role of a control whose type .NET's UI Automation client can't name. diff --git a/src/Anode/Mcp/Tools.cs b/src/Anode/Mcp/Tools.cs index 6267445..2098805 100644 --- a/src/Anode/Mcp/Tools.cs +++ b/src/Anode/Mcp/Tools.cs @@ -144,6 +144,7 @@ private sealed record Tool(string Name, string Title, string Op, bool StartsDaem "Act on one control from a fresh seat_observe result. Use only an action listed on that control. " + "Supports invoke, focus, set_value, toggle, select, expand, collapse, scroll_into_view, scroll and set_range. " + "Verifies the window process and control identity again before acting. Password controls are refused. " + + "set_value reads the control back and fails if it still shows its old value; Chrome's drop-down lists ignore it, so expand one and select the option. " + "The observation is consumed even if the app times out. Never blindly replay a timed-out action; observe again.", Schema(("snapshotId", "string", "Observation ID returned by seat_observe.", true), ("elementId", "string", "Control ID from that observation, for example e7.", true), From b884144acb1a86828ac94eb372995af2e9e67136 Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:33:40 +0100 Subject: [PATCH 2/3] set_value: keep control text out of the failure, and say when a change may have taken (review) From the second review of B-026's fix: - The failure quoted up to 200 characters of the control's old value, and the pipe server logs every handler error to anode.log, so a field's contents could reach the log. The message no longer quotes the control. - It now says the control may have taken the change and kept its old text (a tag field, a value it rewrites, a change still pending), and gives the "expand the list" advice only for a ComboBox. - A COMException, timeout or InvalidOperationException while reading back, after the value was sent, now answers "performed" with a note instead of a failure that reads as "nothing happened". - The text summary shows what a reformatting control now reads (sanitised), and PROTOCOL.md lists the optional `value`. Co-Authored-By: Claude Opus 5.5 --- docs/DESKTOP-TOOLS.md | 6 +++-- docs/PROTOCOL.md | 2 +- src/Anode/Cli/DesktopChecks.cs | 10 +++++-- src/Anode/Core/Desktop/AccessibilityReader.cs | 26 +++++++++++-------- src/Anode/Core/Desktop/DesktopPresentation.cs | 6 ++++- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/DESKTOP-TOOLS.md b/docs/DESKTOP-TOOLS.md index f135081..8b83862 100644 --- a/docs/DESKTOP-TOOLS.md +++ b/docs/DESKTOP-TOOLS.md @@ -66,8 +66,10 @@ changes. Password controls omit their values and offer no element actions. `set_value` reads the control back. If it still shows its old value 1.5 seconds later, the action fails instead of reporting success: some controls accept the change and ignore it. Chrome's drop-down lists (`) fails instead of reporting performed; applied, late and reformatted values pass"; diff --git a/src/Anode/Core/Desktop/AccessibilityReader.cs b/src/Anode/Core/Desktop/AccessibilityReader.cs index 8e9ac01..256c3df 100644 --- a/src/Anode/Core/Desktop/AccessibilityReader.cs +++ b/src/Anode/Core/Desktop/AccessibilityReader.cs @@ -192,7 +192,7 @@ T Pattern(AutomationPattern id) where T : class => element.TryGetCurrentPatte { case "invoke": Pattern(InvokePattern.Pattern).Invoke(); break; case "focus": element.SetFocus(); break; - case "set_value": return SetValue(Pattern(ValuePattern.Pattern), request.Str("value")!); + case "set_value": return SetValue(Pattern(ValuePattern.Pattern), request.Str("value")!, state.ControlType == ControlType.ComboBox); case "toggle": Pattern(TogglePattern.Pattern).Toggle(); break; case "select": Pattern(SelectionItemPattern.Pattern).Select(); break; case "expand": Pattern(ExpandCollapsePattern.Pattern).Expand(); break; @@ -219,9 +219,10 @@ private static JsonObject Performed(string action) => /// /// Sets a value and reads it back. Some controls accept the call and ignore it, as Chrome's drop-down lists - /// (<select>) do, and "performed" would then let an agent submit a form with the old choice. + /// (<select>) do, and "performed" would then let an agent submit a form with the old choice. The failure never + /// quotes the control's text: the pipe server logs errors, and a field can hold anything. /// - private static JsonObject SetValue(ValuePattern pattern, string value) + private static JsonObject SetValue(ValuePattern pattern, string value, bool list) { string before = pattern.Current.Value ?? ""; pattern.SetValue(value); @@ -236,11 +237,13 @@ private static JsonObject SetValue(ValuePattern pattern, string value) now = pattern.Current.Value ?? ""; } } - catch (ElementNotAvailableException) + catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or TimeoutException + or System.Runtime.InteropServices.COMException) { - var gone = Performed("set_value"); - gone["note"] = "The control went away after the change, so its new value wasn't read back. Inspect again before the next action."; - return gone; + // The value was already sent, so this must not read as "nothing happened". + var unread = Performed("set_value"); + unread["note"] = "The value was sent, but the control could not be read back (it may have gone away). Inspect again before the next action."; + return unread; } if (now == value) return Performed("set_value"); if (now != before) @@ -248,13 +251,14 @@ private static JsonObject SetValue(ValuePattern pattern, string value) // An app may reformat what it was given, for example a number field that adds its separators. var changed = Performed("set_value"); changed["value"] = Clip(now, 500); - changed["note"] = "The control changed the value it was given; value is what it reads now. Observation consumed. Inspect again before the next action."; + changed["note"] = "The control changed the value it was given. Observation consumed. Inspect again before the next action."; return changed; } throw new InvalidOperationException( - $"The control accepted set_value but still read \"{Clip(before, 200)}\" after {SetValueSettleMs} ms, so the new value " - + "wasn't applied, or the control doesn't show it. Some controls ignore set_value, as Chrome's drop-down lists do: " - + "expand the list and select the option, or click it. Inspect again before you retry."); + $"The control accepted set_value but still showed its old value {SetValueSettleMs} ms later. It ignored the change, or " + + "took it and kept its old text (a tag field that adds a tag, a value it rewrites, a change still pending). " + + (list ? "Drop-down lists such as Chrome's ignore set_value: expand the list and select the option, or click it. " : "") + + "Inspect again before you retry."); } /// The role of a control whose type .NET's UI Automation client can't name. diff --git a/src/Anode/Core/Desktop/DesktopPresentation.cs b/src/Anode/Core/Desktop/DesktopPresentation.cs index baa86d6..8397aa5 100644 --- a/src/Anode/Core/Desktop/DesktopPresentation.cs +++ b/src/Anode/Core/Desktop/DesktopPresentation.cs @@ -66,7 +66,11 @@ public static string Summary(JsonObject result) if (result["warnings"] is JsonArray warnings) foreach (var warning in warnings) text.AppendLine(Line(warning?.ToString())); } - else text.Append(result.Str("note") ?? "ok"); + else + { + text.Append(result.Str("note") ?? "ok"); + if (result.Str("value") is { } value) text.Append(" It now reads: " + Line(value)); + } return text.ToString().TrimEnd(); } From 1f7d5a1b9061cce5e36da6040754c5a0763010de Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:37:20 +0100 Subject: [PATCH 3/3] set_value: quote the value a control now reads, and check the read-back catch and a text field (review) From the re-check of b884144: the text summary quotes what a reformatting control reads, or says "(empty)", so app text isn't read as part of Anode's note. The quick check now also covers a control that goes away after the value is sent (performed, "could not be read back") and a text field that ignores the value (no drop-down advice). Co-Authored-By: Claude Opus 5.5 --- src/Anode/Cli/DesktopChecks.cs | 35 ++++++++++++++----- src/Anode/Core/Desktop/DesktopPresentation.cs | 2 +- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Anode/Cli/DesktopChecks.cs b/src/Anode/Cli/DesktopChecks.cs index be4ba31..b71482b 100644 --- a/src/Anode/Cli/DesktopChecks.cs +++ b/src/Anode/Cli/DesktopChecks.cs @@ -153,21 +153,23 @@ private sealed class AppBarWindow : HiddenWindow } /// - /// A drop-down list that takes set_value the way it is told to: applies it, ignores it as Chrome's <select> - /// does, applies it a moment later as Chrome's text fields do, or reformats it. + /// A drop-down list, or a text field, that takes set_value the way it is told to: applies it, ignores it as Chrome's + /// <select> does, applies it a moment later as Chrome's text fields do, reformats it, or goes away. /// private sealed class ValueWindow : HiddenWindow, IValueProvider { private volatile string _mode = "apply", _value = "Free"; - public void Use(string mode) { _mode = mode; _value = "Free"; } + private volatile bool _sent, _field; + public void Use(string mode, bool field = false) { _mode = mode; _field = field; _value = "Free"; _sent = false; } public override object? GetPatternProvider(int patternId) => patternId == ValuePattern.Pattern.Id ? this : null; public override object? GetPropertyValue(int propertyId) => - propertyId == AutomationElement.ControlTypeProperty.Id ? ControlType.ComboBox.Id + propertyId == AutomationElement.ControlTypeProperty.Id ? (_field ? ControlType.Edit.Id : ControlType.ComboBox.Id) : propertyId == AutomationElement.NameProperty.Id ? "Plan" : null; public bool IsReadOnly => false; - public string Value => _value; + public string Value => _mode == "vanish" && _sent ? throw new ElementNotAvailableException() : _value; public void SetValue(string value) { + _sent = true; switch (_mode) { case "ignore": break; @@ -244,10 +246,10 @@ public static string IgnoredValues() { var list = Root(AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 0 })); Require(list["actions"]!.AsArray().Any(action => action?.GetValue() == "set_value"), "the list does not offer set_value"); - JsonObject Set(string mode) + JsonObject Set(string mode, JsonObject? control = null) { - window.Use(mode); - return AccessibilityReader.Act(target, list, new JsonObject { ["action"] = "set_value", ["value"] = "Team" }); + window.Use(mode, field: control is not null); + return AccessibilityReader.Act(target, control ?? list, new JsonObject { ["action"] = "set_value", ["value"] = "Team" }); } foreach (string mode in new[] { "apply", "late" }) { @@ -257,7 +259,7 @@ JsonObject Set(string mode) } var reformatted = Set("reformat"); Require(reformatted.Str("performed") == "set_value" && reformatted.Str("value") == "TEAM" - && DesktopPresentation.Summary(reformatted).EndsWith("It now reads: TEAM", StringComparison.Ordinal), + && DesktopPresentation.Summary(reformatted).EndsWith("It now reads: \"TEAM\"", StringComparison.Ordinal), "a value the control reformatted was not reported as it now reads: " + reformatted.ToJsonString()); var watch = Stopwatch.StartNew(); try @@ -273,6 +275,21 @@ JsonObject Set(string mode) } Require(window.Value == "Free" && watch.ElapsedMilliseconds >= AccessibilityReader.SetValueSettleMs - 100, "an ignored value was refused before the control had time to apply it"); + // The value went out before the control went away, so that must not read as "nothing happened". + var unread = Set("vanish"); + Require(unread.Str("performed") == "set_value" && unread.Str("note")!.Contains("could not be read back", StringComparison.Ordinal), + "a control that went away after set_value was reported as " + unread.ToJsonString()); + window.Use("ignore", field: true); + var field = Root(AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 0 })); + try + { + var ignored = Set("ignore", field); + throw new InvalidOperationException("a value the text field ignored was reported as " + ignored.ToJsonString()); + } + catch (InvalidOperationException error) when (error.Message.Contains("still showed its old value", StringComparison.Ordinal)) + { + Require(!error.Message.Contains("expand the list", StringComparison.Ordinal), "a text field was told to expand a list: " + error.Message); + } return "set_value reads the control back: an ignored value (as Chrome's