From f08518fee793ffd183c1d8d406504b4b82a2d9ba Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:24:05 +0100 Subject: [PATCH 1/6] Fix: set_value reports "performed" when the control ignores it (B-026) seat_element set_value on a Chrome `), `seat_element` + `set_value` reported `performed` while the list kept its old choice, so an agent could submit a + form with the wrong option. Anode now reads the control back: if it still shows its old value 1.5 + seconds later, the action fails and says to expand the list and select the option. A control that + reformats the value succeeds and returns what it now reads. - **Packaged apps' Anode folders.** A daemon started by a desktop app that redirects AppData, such as the Claude desktop app, keeps its log and state in that app's `%LOCALAPPDATA%\Packages\\LocalCache\Local\Anode`, which the privacy, install and troubleshooting diff --git a/docs/DESKTOP-TOOLS.md b/docs/DESKTOP-TOOLS.md index 1277a94..f135081 100644 --- a/docs/DESKTOP-TOOLS.md +++ b/docs/DESKTOP-TOOLS.md @@ -63,6 +63,11 @@ by `--`, for example `--direction down --amount large` or `--number 50`. Only use actions listed on that control. Read-only fields do not offer value changes. Password controls omit their values and offer no element actions. +`set_value` reads the control back. If it still shows its old value 1.5 seconds +later, the action fails instead of reporting success: some controls accept the +change and ignore it. Chrome's drop-down lists (`) fails instead of reporting performed; applied, late and reformatted values pass"; + }); } } diff --git a/src/Anode/Cli/SelfTest.cs b/src/Anode/Cli/SelfTest.cs index deedcc2..7ca4ba8 100644 --- a/src/Anode/Cli/SelfTest.cs +++ b/src/Anode/Cli/SelfTest.cs @@ -131,6 +131,7 @@ public static int Run(string[] args) Check(results, "desktop worker deadline", () => DesktopChecks.WorkerTimeout().GetAwaiter().GetResult()); Check(results, "desktop report escaping", DesktopChecks.Presentation); Check(results, "desktop unknown control types", DesktopChecks.UnknownControlTypes); + Check(results, "desktop ignored set_value", DesktopChecks.IgnoredValues); Check(results, "execution output", DevelopmentChecks.Output); Check(results, "execution jobs", () => DevelopmentChecks.Jobs().GetAwaiter().GetResult()); Check(results, "cancelled execution", () => DevelopmentChecks.CancelledStart().GetAwaiter().GetResult()); diff --git a/src/Anode/Core/Desktop/AccessibilityReader.cs b/src/Anode/Core/Desktop/AccessibilityReader.cs index 17aeeae..8e9ac01 100644 --- a/src/Anode/Core/Desktop/AccessibilityReader.cs +++ b/src/Anode/Core/Desktop/AccessibilityReader.cs @@ -192,7 +192,7 @@ T Pattern(AutomationPattern id) where T : class => element.TryGetCurrentPatte { case "invoke": Pattern(InvokePattern.Pattern).Invoke(); break; case "focus": element.SetFocus(); break; - case "set_value": Pattern(ValuePattern.Pattern).SetValue(request.Str("value")!); break; + case "set_value": return SetValue(Pattern(ValuePattern.Pattern), request.Str("value")!); case "toggle": Pattern(TogglePattern.Pattern).Toggle(); break; case "select": Pattern(SelectionItemPattern.Pattern).Select(); break; case "expand": Pattern(ExpandCollapsePattern.Pattern).Expand(); break; @@ -208,7 +208,53 @@ T Pattern(AutomationPattern id) where T : class => element.TryGetCurrentPatte case "set_range": Pattern(RangeValuePattern.Pattern).SetValue(request["number"]!.GetValue()); break; default: throw new ArgumentException("Unknown element action."); } - return new JsonObject { ["performed"] = action, ["note"] = "Observation consumed. Inspect again before the next action." }; + return Performed(action); + } + + private static JsonObject Performed(string action) => + new() { ["performed"] = action, ["note"] = "Observation consumed. Inspect again before the next action." }; + + /// How long set_value watches for the control to take its new value. Chrome shows a change a moment later. + internal const int SetValueSettleMs = 1500; + + /// + /// Sets a value and reads it back. Some controls accept the call and ignore it, as Chrome's drop-down lists + /// (<select>) do, and "performed" would then let an agent submit a form with the old choice. + /// + private static JsonObject SetValue(ValuePattern pattern, string value) + { + string before = pattern.Current.Value ?? ""; + pattern.SetValue(value); + string now; + var watch = Stopwatch.StartNew(); + try + { + now = pattern.Current.Value ?? ""; + while (now == before && now != value && watch.ElapsedMilliseconds < SetValueSettleMs) + { + Thread.Sleep(50); + now = pattern.Current.Value ?? ""; + } + } + catch (ElementNotAvailableException) + { + var gone = Performed("set_value"); + gone["note"] = "The control went away after the change, so its new value wasn't read back. Inspect again before the next action."; + return gone; + } + if (now == value) return Performed("set_value"); + if (now != before) + { + // An app may reformat what it was given, for example a number field that adds its separators. + var changed = Performed("set_value"); + changed["value"] = Clip(now, 500); + changed["note"] = "The control changed the value it was given; value is what it reads now. Observation consumed. Inspect again before the next action."; + return changed; + } + throw new InvalidOperationException( + $"The control accepted set_value but still read \"{Clip(before, 200)}\" after {SetValueSettleMs} ms, so the new value " + + "wasn't applied, or the control doesn't show it. Some controls ignore set_value, as Chrome's drop-down lists do: " + + "expand the list and select the option, or click it. Inspect again before you retry."); } /// The role of a control whose type .NET's UI Automation client can't name. diff --git a/src/Anode/Mcp/Tools.cs b/src/Anode/Mcp/Tools.cs index 6267445..2098805 100644 --- a/src/Anode/Mcp/Tools.cs +++ b/src/Anode/Mcp/Tools.cs @@ -144,6 +144,7 @@ private sealed record Tool(string Name, string Title, string Op, bool StartsDaem "Act on one control from a fresh seat_observe result. Use only an action listed on that control. " + "Supports invoke, focus, set_value, toggle, select, expand, collapse, scroll_into_view, scroll and set_range. " + "Verifies the window process and control identity again before acting. Password controls are refused. " + + "set_value reads the control back and fails if it still shows its old value; Chrome's drop-down lists ignore it, so expand one and select the option. " + "The observation is consumed even if the app times out. Never blindly replay a timed-out action; observe again.", Schema(("snapshotId", "string", "Observation ID returned by seat_observe.", true), ("elementId", "string", "Control ID from that observation, for example e7.", true), From b884144acb1a86828ac94eb372995af2e9e67136 Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:33:40 +0100 Subject: [PATCH 2/6] set_value: keep control text out of the failure, and say when a change may have taken (review) From the second review of B-026's fix: - The failure quoted up to 200 characters of the control's old value, and the pipe server logs every handler error to anode.log, so a field's contents could reach the log. The message no longer quotes the control. - It now says the control may have taken the change and kept its old text (a tag field, a value it rewrites, a change still pending), and gives the "expand the list" advice only for a ComboBox. - A COMException, timeout or InvalidOperationException while reading back, after the value was sent, now answers "performed" with a note instead of a failure that reads as "nothing happened". - The text summary shows what a reformatting control now reads (sanitised), and PROTOCOL.md lists the optional `value`. Co-Authored-By: Claude Opus 5.5 --- docs/DESKTOP-TOOLS.md | 6 +++-- docs/PROTOCOL.md | 2 +- src/Anode/Cli/DesktopChecks.cs | 10 +++++-- src/Anode/Core/Desktop/AccessibilityReader.cs | 26 +++++++++++-------- src/Anode/Core/Desktop/DesktopPresentation.cs | 6 ++++- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/DESKTOP-TOOLS.md b/docs/DESKTOP-TOOLS.md index f135081..8b83862 100644 --- a/docs/DESKTOP-TOOLS.md +++ b/docs/DESKTOP-TOOLS.md @@ -66,8 +66,10 @@ changes. Password controls omit their values and offer no element actions. `set_value` reads the control back. If it still shows its old value 1.5 seconds later, the action fails instead of reporting success: some controls accept the change and ignore it. Chrome's drop-down lists (`) fails instead of reporting performed; applied, late and reformatted values pass"; diff --git a/src/Anode/Core/Desktop/AccessibilityReader.cs b/src/Anode/Core/Desktop/AccessibilityReader.cs index 8e9ac01..256c3df 100644 --- a/src/Anode/Core/Desktop/AccessibilityReader.cs +++ b/src/Anode/Core/Desktop/AccessibilityReader.cs @@ -192,7 +192,7 @@ T Pattern(AutomationPattern id) where T : class => element.TryGetCurrentPatte { case "invoke": Pattern(InvokePattern.Pattern).Invoke(); break; case "focus": element.SetFocus(); break; - case "set_value": return SetValue(Pattern(ValuePattern.Pattern), request.Str("value")!); + case "set_value": return SetValue(Pattern(ValuePattern.Pattern), request.Str("value")!, state.ControlType == ControlType.ComboBox); case "toggle": Pattern(TogglePattern.Pattern).Toggle(); break; case "select": Pattern(SelectionItemPattern.Pattern).Select(); break; case "expand": Pattern(ExpandCollapsePattern.Pattern).Expand(); break; @@ -219,9 +219,10 @@ private static JsonObject Performed(string action) => /// /// Sets a value and reads it back. Some controls accept the call and ignore it, as Chrome's drop-down lists - /// (<select>) do, and "performed" would then let an agent submit a form with the old choice. + /// (<select>) do, and "performed" would then let an agent submit a form with the old choice. The failure never + /// quotes the control's text: the pipe server logs errors, and a field can hold anything. /// - private static JsonObject SetValue(ValuePattern pattern, string value) + private static JsonObject SetValue(ValuePattern pattern, string value, bool list) { string before = pattern.Current.Value ?? ""; pattern.SetValue(value); @@ -236,11 +237,13 @@ private static JsonObject SetValue(ValuePattern pattern, string value) now = pattern.Current.Value ?? ""; } } - catch (ElementNotAvailableException) + catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or TimeoutException + or System.Runtime.InteropServices.COMException) { - var gone = Performed("set_value"); - gone["note"] = "The control went away after the change, so its new value wasn't read back. Inspect again before the next action."; - return gone; + // The value was already sent, so this must not read as "nothing happened". + var unread = Performed("set_value"); + unread["note"] = "The value was sent, but the control could not be read back (it may have gone away). Inspect again before the next action."; + return unread; } if (now == value) return Performed("set_value"); if (now != before) @@ -248,13 +251,14 @@ private static JsonObject SetValue(ValuePattern pattern, string value) // An app may reformat what it was given, for example a number field that adds its separators. var changed = Performed("set_value"); changed["value"] = Clip(now, 500); - changed["note"] = "The control changed the value it was given; value is what it reads now. Observation consumed. Inspect again before the next action."; + changed["note"] = "The control changed the value it was given. Observation consumed. Inspect again before the next action."; return changed; } throw new InvalidOperationException( - $"The control accepted set_value but still read \"{Clip(before, 200)}\" after {SetValueSettleMs} ms, so the new value " - + "wasn't applied, or the control doesn't show it. Some controls ignore set_value, as Chrome's drop-down lists do: " - + "expand the list and select the option, or click it. Inspect again before you retry."); + $"The control accepted set_value but still showed its old value {SetValueSettleMs} ms later. It ignored the change, or " + + "took it and kept its old text (a tag field that adds a tag, a value it rewrites, a change still pending). " + + (list ? "Drop-down lists such as Chrome's ignore set_value: expand the list and select the option, or click it. " : "") + + "Inspect again before you retry."); } /// The role of a control whose type .NET's UI Automation client can't name. diff --git a/src/Anode/Core/Desktop/DesktopPresentation.cs b/src/Anode/Core/Desktop/DesktopPresentation.cs index baa86d6..8397aa5 100644 --- a/src/Anode/Core/Desktop/DesktopPresentation.cs +++ b/src/Anode/Core/Desktop/DesktopPresentation.cs @@ -66,7 +66,11 @@ public static string Summary(JsonObject result) if (result["warnings"] is JsonArray warnings) foreach (var warning in warnings) text.AppendLine(Line(warning?.ToString())); } - else text.Append(result.Str("note") ?? "ok"); + else + { + text.Append(result.Str("note") ?? "ok"); + if (result.Str("value") is { } value) text.Append(" It now reads: " + Line(value)); + } return text.ToString().TrimEnd(); } From 1f7d5a1b9061cce5e36da6040754c5a0763010de Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:37:20 +0100 Subject: [PATCH 3/6] set_value: quote the value a control now reads, and check the read-back catch and a text field (review) From the re-check of b884144: the text summary quotes what a reformatting control reads, or says "(empty)", so app text isn't read as part of Anode's note. The quick check now also covers a control that goes away after the value is sent (performed, "could not be read back") and a text field that ignores the value (no drop-down advice). Co-Authored-By: Claude Opus 5.5 --- src/Anode/Cli/DesktopChecks.cs | 35 ++++++++++++++----- src/Anode/Core/Desktop/DesktopPresentation.cs | 2 +- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Anode/Cli/DesktopChecks.cs b/src/Anode/Cli/DesktopChecks.cs index be4ba31..b71482b 100644 --- a/src/Anode/Cli/DesktopChecks.cs +++ b/src/Anode/Cli/DesktopChecks.cs @@ -153,21 +153,23 @@ private sealed class AppBarWindow : HiddenWindow } /// - /// A drop-down list that takes set_value the way it is told to: applies it, ignores it as Chrome's <select> - /// does, applies it a moment later as Chrome's text fields do, or reformats it. + /// A drop-down list, or a text field, that takes set_value the way it is told to: applies it, ignores it as Chrome's + /// <select> does, applies it a moment later as Chrome's text fields do, reformats it, or goes away. /// private sealed class ValueWindow : HiddenWindow, IValueProvider { private volatile string _mode = "apply", _value = "Free"; - public void Use(string mode) { _mode = mode; _value = "Free"; } + private volatile bool _sent, _field; + public void Use(string mode, bool field = false) { _mode = mode; _field = field; _value = "Free"; _sent = false; } public override object? GetPatternProvider(int patternId) => patternId == ValuePattern.Pattern.Id ? this : null; public override object? GetPropertyValue(int propertyId) => - propertyId == AutomationElement.ControlTypeProperty.Id ? ControlType.ComboBox.Id + propertyId == AutomationElement.ControlTypeProperty.Id ? (_field ? ControlType.Edit.Id : ControlType.ComboBox.Id) : propertyId == AutomationElement.NameProperty.Id ? "Plan" : null; public bool IsReadOnly => false; - public string Value => _value; + public string Value => _mode == "vanish" && _sent ? throw new ElementNotAvailableException() : _value; public void SetValue(string value) { + _sent = true; switch (_mode) { case "ignore": break; @@ -244,10 +246,10 @@ public static string IgnoredValues() { var list = Root(AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 0 })); Require(list["actions"]!.AsArray().Any(action => action?.GetValue() == "set_value"), "the list does not offer set_value"); - JsonObject Set(string mode) + JsonObject Set(string mode, JsonObject? control = null) { - window.Use(mode); - return AccessibilityReader.Act(target, list, new JsonObject { ["action"] = "set_value", ["value"] = "Team" }); + window.Use(mode, field: control is not null); + return AccessibilityReader.Act(target, control ?? list, new JsonObject { ["action"] = "set_value", ["value"] = "Team" }); } foreach (string mode in new[] { "apply", "late" }) { @@ -257,7 +259,7 @@ JsonObject Set(string mode) } var reformatted = Set("reformat"); Require(reformatted.Str("performed") == "set_value" && reformatted.Str("value") == "TEAM" - && DesktopPresentation.Summary(reformatted).EndsWith("It now reads: TEAM", StringComparison.Ordinal), + && DesktopPresentation.Summary(reformatted).EndsWith("It now reads: \"TEAM\"", StringComparison.Ordinal), "a value the control reformatted was not reported as it now reads: " + reformatted.ToJsonString()); var watch = Stopwatch.StartNew(); try @@ -273,6 +275,21 @@ JsonObject Set(string mode) } Require(window.Value == "Free" && watch.ElapsedMilliseconds >= AccessibilityReader.SetValueSettleMs - 100, "an ignored value was refused before the control had time to apply it"); + // The value went out before the control went away, so that must not read as "nothing happened". + var unread = Set("vanish"); + Require(unread.Str("performed") == "set_value" && unread.Str("note")!.Contains("could not be read back", StringComparison.Ordinal), + "a control that went away after set_value was reported as " + unread.ToJsonString()); + window.Use("ignore", field: true); + var field = Root(AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 0 })); + try + { + var ignored = Set("ignore", field); + throw new InvalidOperationException("a value the text field ignored was reported as " + ignored.ToJsonString()); + } + catch (InvalidOperationException error) when (error.Message.Contains("still showed its old value", StringComparison.Ordinal)) + { + Require(!error.Message.Contains("expand the list", StringComparison.Ordinal), "a text field was told to expand a list: " + error.Message); + } return "set_value reads the control back: an ignored value (as Chrome's list is open (B-027) With a ) fails instead of reporting performed; applied, late and reformatted values pass"; }); } + + /// + /// A window whose tree repeats controls, as Chrome's does: C lists A, its own ancestor, as a child (a loop, like + /// Chrome's window below itself while a <select> list is open), and D lists B, which is A's (like Chrome's + /// toolbar appearing twice). + /// + private sealed class LoopWindow : HiddenWindow, IRawElementProviderFragmentRoot + { + public readonly LoopNode A, B, C, D; + public LoopWindow() + { + A = new(this, 1, "A"); B = new(this, 2, "B"); C = new(this, 3, "C"); D = new(this, 4, "D"); + (A.Parent, A.Next, A.FirstChild, A.LastChild) = (this, D, B, C); + (D.Parent, D.Previous, D.FirstChild, D.LastChild) = (this, A, B, B); + (B.Parent, B.Next) = (A, C); + (C.Parent, C.Previous, C.FirstChild, C.LastChild) = (A, B, A, A); + } + public override object? GetPatternProvider(int patternId) => null; + public override object? GetPropertyValue(int propertyId) => null; + public System.Windows.Rect BoundingRectangle => System.Windows.Rect.Empty; + public IRawElementProviderFragmentRoot FragmentRoot => this; + public IRawElementProviderSimple[]? GetEmbeddedFragmentRoots() => null; + public int[]? GetRuntimeId() => null; // the window's own, from its handle + public IRawElementProviderFragment? Navigate(NavigateDirection direction) => + direction == NavigateDirection.FirstChild ? A : direction == NavigateDirection.LastChild ? D : null; + public void SetFocus() { } + public IRawElementProviderFragment? ElementProviderFromPoint(double x, double y) => null; + public IRawElementProviderFragment? GetFocus() => null; + } + + private sealed class LoopNode(LoopWindow window, int id, string name) : IRawElementProviderFragment, IInvokeProvider + { + public IRawElementProviderFragment? Parent, FirstChild, LastChild, Next, Previous; + public int Invoked; + public ProviderOptions ProviderOptions => ProviderOptions.ServerSideProvider; + public object? GetPatternProvider(int patternId) => patternId == InvokePattern.Pattern.Id ? this : null; + public object? GetPropertyValue(int propertyId) => + propertyId == AutomationElement.ControlTypeProperty.Id ? ControlType.Button.Id + : propertyId == AutomationElement.NameProperty.Id ? name + : propertyId == AutomationElement.IsEnabledProperty.Id ? true : null; + public IRawElementProviderSimple? HostRawElementProvider => null; + public System.Windows.Rect BoundingRectangle => System.Windows.Rect.Empty; + public IRawElementProviderFragmentRoot FragmentRoot => window; + public IRawElementProviderSimple[]? GetEmbeddedFragmentRoots() => null; + public int[] GetRuntimeId() => new[] { AutomationInteropProvider.AppendRuntimeId, id }; + public IRawElementProviderFragment? Navigate(NavigateDirection direction) => direction switch + { + NavigateDirection.Parent => Parent, NavigateDirection.FirstChild => FirstChild, NavigateDirection.LastChild => LastChild, + NavigateDirection.NextSibling => Next, NavigateDirection.PreviousSibling => Previous, _ => null + }; + public void SetFocus() { } + public void Invoke() => Interlocked.Increment(ref Invoked); + } + + public static string RepeatedElements() + { + var window = new LoopWindow(); + return Hosted(window, "Anode repeated elements check", target => + { + var observation = AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 6, ["includeOffscreen"] = true }); + var elements = observation["elements"]!.AsArray().OfType().ToList(); + string names = string.Join(",", elements.Skip(1).Select(element => element.Str("name"))); + Require(names == "A,D,B,C" && observation.Bool("truncated") == false, + $"a tree that repeats controls was observed as {names} (truncated: {observation.Bool("truncated")}), not A,D,B,C once each"); + Require(observation["warnings"]!.AsArray().Any(warning => warning?.GetValue().StartsWith("Skipped 4 controls", StringComparison.Ordinal) == true), + "the observation did not say it skipped the 4 repeats: " + observation["warnings"]!.ToJsonString()); + // A control's path counts the skipped repeats, so actions still reach the control that was observed. + var c = elements.Single(element => element.Str("name") == "C"); + AccessibilityReader.Act(target, c, new JsonObject { ["action"] = "invoke" }); + Require(window.C.Invoked == 1 && window.A.Invoked + window.B.Invoked + window.D.Invoked == 0, "an action reached the wrong control"); + return "a control the app reports again, below itself (Chrome with a list is open; walking it + // again would repeat its whole subtree until the budget ran out. + var seen = new HashSet(); + int repeated = 0; bool truncated = false; int visited = 0; while (queue.Count > 0 && visited < maximum && watch.ElapsedMilliseconds < 4000) @@ -38,7 +42,15 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) visited++; try { - var node = Describe(current.Element, current.Path, current.Parent, ref textBudget); + int[] runtimeId = current.Element.GetRuntimeId(); + if (runtimeId.Length > 0 && !seen.Add(string.Join(".", runtimeId))) + { + // A repeat is neither described nor walked, so it doesn't use the element budget. + visited--; + repeated++; + continue; + } + var node = Describe(current.Element, runtimeId, current.Path, current.Parent, ref textBudget); string? parent = current.Parent; if (offscreen || node.Bool("offscreen") != true || current.Path.Length == 0) { @@ -62,6 +74,8 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) } } truncated |= queue.Count > 0; + if (repeated > 0) + warnings.Add($"Skipped {repeated} control{(repeated == 1 ? "" : "s")} the app reported again elsewhere in this window's tree."); return new JsonObject { ["window"] = WindowAccess.Describe(target), ["elements"] = elements, @@ -70,7 +84,7 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) }; } - private static JsonObject Describe(AutomationElement element, int[] path, string? parent, ref int textBudget) + private static JsonObject Describe(AutomationElement element, int[] runtimeId, int[] path, string? parent, ref int textBudget) { var cache = new CacheRequest { TreeScope = TreeScope.Element }; foreach (var property in new[] @@ -102,7 +116,7 @@ private static JsonObject Describe(AutomationElement element, int[] path, string ["actions"] = new JsonArray(actions.Select(value => (JsonNode)value).ToArray()), // Private references are retained by the host and removed from public results. ["path"] = new JsonArray(path.Select(value => (JsonNode)value).ToArray()), - ["runtimeId"] = new JsonArray(element.GetRuntimeId().Select(value => (JsonNode)value).ToArray()), + ["runtimeId"] = new JsonArray(runtimeId.Select(value => (JsonNode)value).ToArray()), ["processId"] = data.ProcessId }; if (!password) From fed1aeafbbd38f44f5712f1914a5ee86a065744a Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:42:58 +0100 Subject: [PATCH 5/6] seat_observe: report skipped repeats in their own field, check them before queueing (review) From the second review of B-027's fix: - The "Skipped N controls" warning made seat_wait's "missing" state, which needs a tree without warnings, time out on any window with a repeat. The count is now `skippedRepeats` and a summary line, not a warning. - A repeat is caught when its parent lists it, so it no longer takes a queue slot from real controls; a control whose description fails leaves room for a later copy; a null runtime ID no longer throws. - The CHANGELOG and DESKTOP-TOOLS.md describe what Anode does, not Chrome's internals, which still need a seat to confirm; PROTOCOL.md lists the field. - The check now has a second provider object with B's runtime ID before a unique control E, acts on E through a path that counts the skipped repeat, uses a budget one above the unique count, and asserts that a wait for a missing control still matches. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++-- docs/DESKTOP-TOOLS.md | 5 +- docs/PROTOCOL.md | 2 +- src/Anode/Cli/DesktopChecks.cs | 39 +++++++++------ src/Anode/Core/Desktop/AccessibilityReader.cs | 50 ++++++++++++------- src/Anode/Core/Desktop/DesktopPresentation.cs | 2 + 6 files changed, 68 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31be4a4..efc74e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,11 @@ form with the wrong option. Anode now reads the control back: if it still shows its old value 1.5 seconds later, the action fails and says to expand the list and select the option. A control that reformats the value succeeds and returns what it now reads. -- **`seat_observe` lists each control once.** While a Chrome drop-down list was open, Chrome reported - the window again below itself, and `seat_observe` walked the copy over and over until its element - budget ran out. A control the app reports again anywhere in the window is now skipped, and a - warning says how many were. +- **`seat_observe` lists each control once.** While a Chrome drop-down list was open, `seat_observe` + returned the window's tree nested inside itself, over and over, until its element budget ran out. + An app can list one control in two places, or even inside itself. The walk now skips a control with + the same UI Automation runtime ID as one it has already listed, without using the budget, and + `skippedRepeats` says how many it skipped. - **Packaged apps' Anode folders.** A daemon started by a desktop app that redirects AppData, such as the Claude desktop app, keeps its log and state in that app's `%LOCALAPPDATA%\Packages\\LocalCache\Local\Anode`, which the privacy, install and troubleshooting diff --git a/docs/DESKTOP-TOOLS.md b/docs/DESKTOP-TOOLS.md index a9d42ca..f7d9645 100644 --- a/docs/DESKTOP-TOOLS.md +++ b/docs/DESKTOP-TOOLS.md @@ -84,8 +84,9 @@ Observations default to 200 controls, depth 8 and 6,000 text characters. MCP acc `maxElements` (1–500), `maxDepth` (0–20), `maxTextChars` (0–20,000), and `includeOffscreen`. CLI uses `--max-elements`, `--max-depth`, `--max-text` and `--offscreen`. The tree reports truncation and provider warnings. A control the app -reports more than once, as Chrome does while a drop-down list is open, appears once, -and a warning says how many repeats were skipped. Accessibility +lists more than once (the same UI Automation runtime ID), even inside itself as +Chrome appeared to while a drop-down list was open, appears once; `skippedRepeats` +says how many copies were skipped. Accessibility work runs in a disposable child-session process with a ten-second deadline, so an unresponsive app provider does not hold the seat indefinitely. diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index 7aaef36..c4c281e 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -176,7 +176,7 @@ is still restoring when the operation's deadline passes, it fails with `errorCod | `op` | Arguments | Result | | --- | --- | --- | | `desktop.windows` | `query`, `pid` | `{windows: [{windowId, pid, process, title, bounds, foreground, minimized, maximized}], summary}` | -| `desktop.observe` | `windowId`, `maxElements`, `maxDepth`, `maxTextChars`, `includeOffscreen`, `includeScreenshot`, `maxWidth` | `{windowId, snapshotId, expiresInSeconds, window, elements, observedAt, truncated, warnings, screenshot?, screenshotError?, summary}` | +| `desktop.observe` | `windowId`, `maxElements`, `maxDepth`, `maxTextChars`, `includeOffscreen`, `includeScreenshot`, `maxWidth` | `{windowId, snapshotId, expiresInSeconds, window, elements, observedAt, truncated, warnings, skippedRepeats?, screenshot?, screenshotError?, summary}` | | `desktop.window` | `windowId`, `action`, optional move geometry `x`, `y`, `width`, `height` | `{requested, note, summary}` | | `desktop.element` | `snapshotId`, `elementId`, `action`, optional `value`, `number`, `direction`, `amount` | `{performed, note, value?, summary}`; `value` is what the control reads after a `set_value` it reformatted | | `desktop.capabilities` | `probeCapture` (default true) | `{version, session, workingDirectory, screen, capture, input, supported, limitations, summary}` | diff --git a/src/Anode/Cli/DesktopChecks.cs b/src/Anode/Cli/DesktopChecks.cs index 56f100f..a7e7df7 100644 --- a/src/Anode/Cli/DesktopChecks.cs +++ b/src/Anode/Cli/DesktopChecks.cs @@ -295,20 +295,23 @@ JsonObject Set(string mode, JsonObject? control = null) } /// - /// A window whose tree repeats controls, as Chrome's does: C lists A, its own ancestor, as a child (a loop, like - /// Chrome's window below itself while a <select> list is open), and D lists B, which is A's (like Chrome's - /// toolbar appearing twice). + /// A window whose tree repeats controls. The window lists A and D; A lists B and C; C lists A, its own parent, so + /// its children are A and D again (a loop, as Chrome's window appeared below itself while a <select> list + /// was open); D lists a second provider with B's runtime ID, then E. /// private sealed class LoopWindow : HiddenWindow, IRawElementProviderFragmentRoot { - public readonly LoopNode A, B, C, D; + public readonly LoopNode A, B, C, D, B2, E; public LoopWindow() { A = new(this, 1, "A"); B = new(this, 2, "B"); C = new(this, 3, "C"); D = new(this, 4, "D"); + B2 = new(this, 2, "B"); E = new(this, 5, "E"); (A.Parent, A.Next, A.FirstChild, A.LastChild) = (this, D, B, C); - (D.Parent, D.Previous, D.FirstChild, D.LastChild) = (this, A, B, B); + (D.Parent, D.Previous, D.FirstChild, D.LastChild) = (this, A, B2, E); (B.Parent, B.Next) = (A, C); (C.Parent, C.Previous, C.FirstChild, C.LastChild) = (A, B, A, A); + (B2.Parent, B2.Next) = (D, E); + (E.Parent, E.Previous) = (D, B2); } public override object? GetPatternProvider(int patternId) => null; public override object? GetPropertyValue(int propertyId) => null; @@ -352,18 +355,24 @@ public static string RepeatedElements() var window = new LoopWindow(); return Hosted(window, "Anode repeated elements check", target => { - var observation = AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 6, ["includeOffscreen"] = true }); + // Six controls, and a budget of seven: repeats must not use it up. + var observation = AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 6, ["maxElements"] = 7, ["includeOffscreen"] = true }); var elements = observation["elements"]!.AsArray().OfType().ToList(); string names = string.Join(",", elements.Skip(1).Select(element => element.Str("name"))); - Require(names == "A,D,B,C" && observation.Bool("truncated") == false, - $"a tree that repeats controls was observed as {names} (truncated: {observation.Bool("truncated")}), not A,D,B,C once each"); - Require(observation["warnings"]!.AsArray().Any(warning => warning?.GetValue().StartsWith("Skipped 4 controls", StringComparison.Ordinal) == true), - "the observation did not say it skipped the 4 repeats: " + observation["warnings"]!.ToJsonString()); - // A control's path counts the skipped repeats, so actions still reach the control that was observed. - var c = elements.Single(element => element.Str("name") == "C"); - AccessibilityReader.Act(target, c, new JsonObject { ["action"] = "invoke" }); - Require(window.C.Invoked == 1 && window.A.Invoked + window.B.Invoked + window.D.Invoked == 0, "an action reached the wrong control"); - return "a control the app reports again, below itself (Chrome with a open) or under a second parent, is observed once, without using the budget; waits and actions still work"; }); } } diff --git a/src/Anode/Core/Desktop/AccessibilityReader.cs b/src/Anode/Core/Desktop/AccessibilityReader.cs index dbb8199..4b8647e 100644 --- a/src/Anode/Core/Desktop/AccessibilityReader.cs +++ b/src/Anode/Core/Desktop/AccessibilityReader.cs @@ -28,29 +28,28 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) var elements = new JsonArray(); var warnings = new JsonArray(); var watch = Stopwatch.StartNew(); - var queue = new Queue<(AutomationElement Element, int[] Path, string? Parent)>(); - queue.Enqueue((root, Array.Empty(), null)); - // Some apps report an element again further down, Chrome even below itself while a list + // was open). Walking it again would repeat its whole subtree until the budget ran out, so a control with a runtime + // ID already queued is skipped. Its index still counts, so the paths seat_element follows stay right. var seen = new HashSet(); int repeated = 0; + var queue = new Queue<(AutomationElement Element, int[]? RuntimeId, int[] Path, string? Parent)>(); + int[]? rootId = RuntimeId(root); + if (rootId is { Length: > 0 }) seen.Add(Key(rootId)); + queue.Enqueue((root, rootId, Array.Empty(), null)); bool truncated = false; int visited = 0; while (queue.Count > 0 && visited < maximum && watch.ElapsedMilliseconds < 4000) { var current = queue.Dequeue(); visited++; + JsonObject? node = null; try { - int[] runtimeId = current.Element.GetRuntimeId(); - if (runtimeId.Length > 0 && !seen.Add(string.Join(".", runtimeId))) - { - // A repeat is neither described nor walked, so it doesn't use the element budget. - visited--; - repeated++; - continue; - } - var node = Describe(current.Element, runtimeId, current.Path, current.Parent, ref textBudget); + // A runtime ID that couldn't be read when queued is read again here, and its failure reported. + int[] runtimeId = current.RuntimeId ?? current.Element.GetRuntimeId() ?? Array.Empty(); + if (current.RuntimeId is null && runtimeId.Length > 0) seen.Add(Key(runtimeId)); + node = Describe(current.Element, runtimeId, current.Path, current.Parent, ref textBudget); string? parent = current.Parent; if (offscreen || node.Bool("offscreen") != true || current.Path.Length == 0) { @@ -63,25 +62,42 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) var child = Walker.GetFirstChild(current.Element); while (child is not null && queue.Count + visited < maximum && watch.ElapsedMilliseconds < 4000) { - queue.Enqueue((child, current.Path.Append(index++).ToArray(), parent)); + int[]? id = RuntimeId(child); + if (id is { Length: > 0 } && !seen.Add(Key(id))) repeated++; + else queue.Enqueue((child, id, current.Path.Append(index).ToArray(), parent)); + index++; child = Walker.GetNextSibling(child); } if (child is not null) truncated = true; } catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or System.Runtime.InteropServices.COMException) { + // A control that couldn't be described leaves room for a later copy of it. + if (node is null && current.RuntimeId is { Length: > 0 } id) seen.Remove(Key(id)); if (warnings.Count < 3) warnings.Add("A control changed or did not expose readable accessibility information."); } } truncated |= queue.Count > 0; - if (repeated > 0) - warnings.Add($"Skipped {repeated} control{(repeated == 1 ? "" : "s")} the app reported again elsewhere in this window's tree."); - return new JsonObject + var result = new JsonObject { ["window"] = WindowAccess.Describe(target), ["elements"] = elements, ["truncated"] = truncated, ["warnings"] = warnings, ["observedAt"] = DateTime.UtcNow.ToString("o"), ["elapsedMs"] = watch.ElapsedMilliseconds }; + // Not a warning: nothing was hidden, and seat_wait's "missing" needs a tree without warnings. + if (repeated > 0) result["skippedRepeats"] = repeated; + return result; + + // Null when the control can't say yet; it is then read again, and its failure reported, when it is described. + static int[]? RuntimeId(AutomationElement element) + { + try { return element.GetRuntimeId(); } + catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or System.Runtime.InteropServices.COMException) + { + return null; + } + } + static string Key(int[] id) => string.Join(".", id); } private static JsonObject Describe(AutomationElement element, int[] runtimeId, int[] path, string? parent, ref int textBudget) diff --git a/src/Anode/Core/Desktop/DesktopPresentation.cs b/src/Anode/Core/Desktop/DesktopPresentation.cs index 94589b2..a160571 100644 --- a/src/Anode/Core/Desktop/DesktopPresentation.cs +++ b/src/Anode/Core/Desktop/DesktopPresentation.cs @@ -62,6 +62,8 @@ public static string Summary(JsonObject result) if (node.Str("selectedText") is { Length: > 0 } selected) text.AppendLine(" selected: " + Line(selected)); } if (result.Bool("truncated") == true) text.AppendLine("Tree truncated; adjust maxElements/maxDepth or inspect a smaller window."); + if (result.Int("skippedRepeats") is int repeats) + text.AppendLine($"Skipped {repeats} control(s) the app listed again elsewhere in this window's tree; each is shown once."); if (result.Str("screenshotError") is { } error) text.AppendLine(Line(error)); if (result["warnings"] is JsonArray warnings) foreach (var warning in warnings) text.AppendLine(Line(warning?.ToString())); From 0acceef36f236d836650f9cad2ea5cdb30f2e41e Mon Sep 17 00:00:00 2001 From: skulitom Date: Thu, 8 Oct 2026 23:47:18 +0100 Subject: [PATCH 6/6] seat_observe: stop a list of children that loops back, and keep re-read IDs unique (review) From the re-check of fed1aea: since a skipped repeat takes no budget, a provider whose next sibling comes back to a control it already gave (X -> X, or X -> Y -> X) kept the walk going until the 4 s limit, then dropped the rest of the tree. A parent's list of children now ends at the first control it gives twice. Also: an ID read only when a control is described is checked against the ones already listed; a failed description frees that ID either way; and Act compares a null runtime ID as empty instead of throwing. The check's E now says it is its own next sibling; without the stop, the check fails with only A and D listed. Co-Authored-By: Claude Opus 5.5 --- src/Anode/Cli/DesktopChecks.cs | 4 ++-- src/Anode/Core/Desktop/AccessibilityReader.cs | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/Anode/Cli/DesktopChecks.cs b/src/Anode/Cli/DesktopChecks.cs index a7e7df7..d89ef78 100644 --- a/src/Anode/Cli/DesktopChecks.cs +++ b/src/Anode/Cli/DesktopChecks.cs @@ -297,7 +297,7 @@ JsonObject Set(string mode, JsonObject? control = null) /// /// A window whose tree repeats controls. The window lists A and D; A lists B and C; C lists A, its own parent, so /// its children are A and D again (a loop, as Chrome's window appeared below itself while a <select> list - /// was open); D lists a second provider with B's runtime ID, then E. + /// was open); D lists a second provider with B's runtime ID, then E, which says E is its own next sibling. /// private sealed class LoopWindow : HiddenWindow, IRawElementProviderFragmentRoot { @@ -311,7 +311,7 @@ public LoopWindow() (B.Parent, B.Next) = (A, C); (C.Parent, C.Previous, C.FirstChild, C.LastChild) = (A, B, A, A); (B2.Parent, B2.Next) = (D, E); - (E.Parent, E.Previous) = (D, B2); + (E.Parent, E.Previous, E.Next) = (D, B2, E); } public override object? GetPatternProvider(int patternId) => null; public override object? GetPropertyValue(int propertyId) => null; diff --git a/src/Anode/Core/Desktop/AccessibilityReader.cs b/src/Anode/Core/Desktop/AccessibilityReader.cs index 4b8647e..835ecf7 100644 --- a/src/Anode/Core/Desktop/AccessibilityReader.cs +++ b/src/Anode/Core/Desktop/AccessibilityReader.cs @@ -44,11 +44,18 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) var current = queue.Dequeue(); visited++; JsonObject? node = null; + string? key = current.RuntimeId is { Length: > 0 } queued ? Key(queued) : null; try { // A runtime ID that couldn't be read when queued is read again here, and its failure reported. int[] runtimeId = current.RuntimeId ?? current.Element.GetRuntimeId() ?? Array.Empty(); - if (current.RuntimeId is null && runtimeId.Length > 0) seen.Add(Key(runtimeId)); + if (current.RuntimeId is null && runtimeId.Length > 0 && !seen.Add(Key(runtimeId))) + { + visited--; + repeated++; + continue; + } + if (runtimeId.Length > 0) key = Key(runtimeId); node = Describe(current.Element, runtimeId, current.Path, current.Parent, ref textBudget); string? parent = current.Parent; if (offscreen || node.Bool("offscreen") != true || current.Path.Length == 0) @@ -59,11 +66,16 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) } if (current.Path.Length >= depthLimit) { truncated = true; continue; } int index = 0; + var siblings = new HashSet(); var child = Walker.GetFirstChild(current.Element); while (child is not null && queue.Count + visited < maximum && watch.ElapsedMilliseconds < 4000) { int[]? id = RuntimeId(child); - if (id is { Length: > 0 } && !seen.Add(Key(id))) repeated++; + string? childKey = id is { Length: > 0 } ? Key(id) : null; + // A list of children that comes back to one it already gave has no more to give; repeats take no + // budget, so only this stops it. + if (childKey is not null && !siblings.Add(childKey)) { child = null; break; } + if (childKey is not null && !seen.Add(childKey)) repeated++; else queue.Enqueue((child, id, current.Path.Append(index).ToArray(), parent)); index++; child = Walker.GetNextSibling(child); @@ -72,8 +84,8 @@ public static JsonObject Observe(WindowTarget target, JsonObject request) } catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or System.Runtime.InteropServices.COMException) { - // A control that couldn't be described leaves room for a later copy of it. - if (node is null && current.RuntimeId is { Length: > 0 } id) seen.Remove(Key(id)); + // A control that couldn't be described lets a copy of it listed later be described instead. + if (node is null && key is not null) seen.Remove(key); if (warnings.Count < 3) warnings.Add("A control changed or did not expose readable accessibility information."); } } @@ -204,7 +216,7 @@ public static JsonObject Act(WindowTarget target, JsonObject reference, JsonObje if (element is null) throw Stale(); } int[] runtimeId = reference["runtimeId"]!.AsArray().Select(value => value!.GetValue()).ToArray(); - if (!element.GetRuntimeId().SequenceEqual(runtimeId)) throw Stale(); + if (!(element.GetRuntimeId() ?? Array.Empty()).SequenceEqual(runtimeId)) throw Stale(); var state = element.Current; VerifyElementSession(state.ProcessId); if (state.ProcessId != reference.Int("processId") || Clip(state.Name, 500) != reference.Str("name")