[compass] Fix: seat_observe repeats a window's tree while a Chrome drop-down list is open

CHANGELOG.md

@@ -7,6 +7,11 @@
   form with the wrong option. Anode now reads the control back: if it still shows its old value 1.5
   seconds later, the action fails and says to expand the list and select the option. A control that
   reformats the value succeeds and returns what it now reads.
+- **`seat_observe` lists each control once.** While a Chrome drop-down list was open, `seat_observe`
+  returned the window's tree nested inside itself, over and over, until its element budget ran out.
+  An app can list one control in two places, or even inside itself. The walk now skips a control with
+  the same UI Automation runtime ID as one it has already listed, without using the budget, and
+  `skippedRepeats` says how many it skipped.
 - **Packaged apps' Anode folders.** A daemon started by a desktop app that redirects AppData, such
   as the Claude desktop app, keeps its log and state in that app's
   `%LOCALAPPDATA%\Packages\<app>\LocalCache\Local\Anode`, which the privacy, install and troubleshooting

docs/DESKTOP-TOOLS.md

@@ -83,7 +83,10 @@ action may already have happened: inspect again instead of replaying it.
 Observations default to 200 controls, depth 8 and 6,000 text characters. MCP accepts
 `maxElements` (1–500), `maxDepth` (0–20), `maxTextChars` (0–20,000), and
 `includeOffscreen`. CLI uses `--max-elements`, `--max-depth`, `--max-text` and
-`--offscreen`. The tree reports truncation and provider warnings. Accessibility
+`--offscreen`. The tree reports truncation and provider warnings. A control the app
+lists more than once (the same UI Automation runtime ID), even inside itself as
+Chrome appeared to while a drop-down list was open, appears once; `skippedRepeats`
+says how many copies were skipped. Accessibility
 work runs in a disposable child-session process with a ten-second deadline, so
 an unresponsive app provider does not hold the seat indefinitely.
 

docs/PROTOCOL.md

@@ -176,7 +176,7 @@ is still restoring when the operation's deadline passes, it fails with `errorCod
 | `op` | Arguments | Result |
 | --- | --- | --- |
 | `desktop.windows` | `query`, `pid` | `{windows: [{windowId, pid, process, title, bounds, foreground, minimized, maximized}], summary}` |
-| `desktop.observe` | `windowId`, `maxElements`, `maxDepth`, `maxTextChars`, `includeOffscreen`, `includeScreenshot`, `maxWidth` | `{windowId, snapshotId, expiresInSeconds, window, elements, observedAt, truncated, warnings, screenshot?, screenshotError?, summary}` |
+| `desktop.observe` | `windowId`, `maxElements`, `maxDepth`, `maxTextChars`, `includeOffscreen`, `includeScreenshot`, `maxWidth` | `{windowId, snapshotId, expiresInSeconds, window, elements, observedAt, truncated, warnings, skippedRepeats?, screenshot?, screenshotError?, summary}` |
 | `desktop.window` | `windowId`, `action`, optional move geometry `x`, `y`, `width`, `height` | `{requested, note, summary}` |
 | `desktop.element` | `snapshotId`, `elementId`, `action`, optional `value`, `number`, `direction`, `amount` | `{performed, note, value?, summary}`; `value` is what the control reads after a `set_value` it reformatted |
 | `desktop.capabilities` | `probeCapture` (default true) | `{version, session, workingDirectory, screen, capture, input, supported, limitations, summary}` |

src/Anode/Cli/DesktopChecks.cs

@@ -293,4 +293,86 @@ JsonObject Set(string mode, JsonObject? control = null)
             return "set_value reads the control back: an ignored value (as Chrome's <select>) fails instead of reporting performed; applied, late and reformatted values pass";
         });
     }
+
+    /// <summary>
+    /// A window whose tree repeats controls. The window lists A and D; A lists B and C; C lists A, its own parent, so
+    /// its children are A and D again (a loop, as Chrome's window appeared below itself while a &lt;select&gt; list
+    /// was open); D lists a second provider with B's runtime ID, then E, which says E is its own next sibling.
+    /// </summary>
+    private sealed class LoopWindow : HiddenWindow, IRawElementProviderFragmentRoot
+    {
+        public readonly LoopNode A, B, C, D, B2, E;
+        public LoopWindow()
+        {
+            A = new(this, 1, "A"); B = new(this, 2, "B"); C = new(this, 3, "C"); D = new(this, 4, "D");
+            B2 = new(this, 2, "B"); E = new(this, 5, "E");
+            (A.Parent, A.Next, A.FirstChild, A.LastChild) = (this, D, B, C);
+            (D.Parent, D.Previous, D.FirstChild, D.LastChild) = (this, A, B2, E);
+            (B.Parent, B.Next) = (A, C);
+            (C.Parent, C.Previous, C.FirstChild, C.LastChild) = (A, B, A, A);
+            (B2.Parent, B2.Next) = (D, E);
+            (E.Parent, E.Previous, E.Next) = (D, B2, E);
+        }
+        public override object? GetPatternProvider(int patternId) => null;
+        public override object? GetPropertyValue(int propertyId) => null;
+        public System.Windows.Rect BoundingRectangle => System.Windows.Rect.Empty;
+        public IRawElementProviderFragmentRoot FragmentRoot => this;
+        public IRawElementProviderSimple[]? GetEmbeddedFragmentRoots() => null;
+        public int[]? GetRuntimeId() => null; // the window's own, from its handle
+        public IRawElementProviderFragment? Navigate(NavigateDirection direction) =>
+            direction == NavigateDirection.FirstChild ? A : direction == NavigateDirection.LastChild ? D : null;
+        public void SetFocus() { }
+        public IRawElementProviderFragment? ElementProviderFromPoint(double x, double y) => null;
+        public IRawElementProviderFragment? GetFocus() => null;
+    }
+
+    private sealed class LoopNode(LoopWindow window, int id, string name) : IRawElementProviderFragment, IInvokeProvider
+    {
+        public IRawElementProviderFragment? Parent, FirstChild, LastChild, Next, Previous;
+        public int Invoked;
+        public ProviderOptions ProviderOptions => ProviderOptions.ServerSideProvider;
+        public object? GetPatternProvider(int patternId) => patternId == InvokePattern.Pattern.Id ? this : null;
+        public object? GetPropertyValue(int propertyId) =>
+            propertyId == AutomationElement.ControlTypeProperty.Id ? ControlType.Button.Id
+            : propertyId == AutomationElement.NameProperty.Id ? name
+            : propertyId == AutomationElement.IsEnabledProperty.Id ? true : null;
+        public IRawElementProviderSimple? HostRawElementProvider => null;
+        public System.Windows.Rect BoundingRectangle => System.Windows.Rect.Empty;
+        public IRawElementProviderFragmentRoot FragmentRoot => window;
+        public IRawElementProviderSimple[]? GetEmbeddedFragmentRoots() => null;
+        public int[] GetRuntimeId() => new[] { AutomationInteropProvider.AppendRuntimeId, id };
+        public IRawElementProviderFragment? Navigate(NavigateDirection direction) => direction switch
+        {
+            NavigateDirection.Parent => Parent, NavigateDirection.FirstChild => FirstChild, NavigateDirection.LastChild => LastChild,
+            NavigateDirection.NextSibling => Next, NavigateDirection.PreviousSibling => Previous, _ => null
+        };
+        public void SetFocus() { }
+        public void Invoke() => Interlocked.Increment(ref Invoked);
+    }
+
+    public static string RepeatedElements()
+    {
+        var window = new LoopWindow();
+        return Hosted(window, "Anode repeated elements check", target =>
+        {
+            // Six controls, and a budget of seven: repeats must not use it up.
+            var observation = AccessibilityReader.Observe(target, new JsonObject { ["maxDepth"] = 6, ["maxElements"] = 7, ["includeOffscreen"] = true });
+            var elements = observation["elements"]!.AsArray().OfType<JsonObject>().ToList();
+            string names = string.Join(",", elements.Skip(1).Select(element => element.Str("name")));
+            Require(names == "A,D,B,C,E" && observation.Bool("truncated") == false,
+                $"a tree that repeats controls was observed as {names} (truncated: {observation.Bool("truncated")}), not A,D,B,C,E once each");
+            Require(observation.Int("skippedRepeats") == 3 && DesktopPresentation.Summary(observation).Contains("Skipped 3 control(s)", StringComparison.Ordinal),
+                "the observation did not report the 3 repeats it skipped: " + observation.ToJsonString());
+            // Nothing was hidden, so a wait for a control to go away still matches.
+            Require(observation["warnings"]!.AsArray().Count == 0
+                && DesktopWait.Matches(observation, new JsonObject { ["name"] = "Spinner", ["state"] = "missing" }),
+                "skipped repeats stopped a wait for a missing control from matching");
+            // E's path counts the repeat before it, so an action still reaches the control that was observed.
+            var e = elements.Single(element => element.Str("name") == "E");
+            AccessibilityReader.Act(target, e, new JsonObject { ["action"] = "invoke" });
+            Require(window.E.Invoked == 1 && new[] { window.A, window.B, window.B2, window.C, window.D }.All(node => node.Invoked == 0),
+                "an action reached the wrong control");
+            return "a control the app lists again, below itself (Chrome with a <select> open) or under a second parent, is observed once, without using the budget; waits and actions still work";
+        });
+    }
 }

src/Anode/Cli/SelfTest.cs

@@ -132,6 +132,7 @@ public static int Run(string[] args)
         Check(results, "desktop report escaping", DesktopChecks.Presentation);
         Check(results, "desktop unknown control types", DesktopChecks.UnknownControlTypes);
         Check(results, "desktop ignored set_value", DesktopChecks.IgnoredValues);
+        Check(results, "desktop repeated elements", DesktopChecks.RepeatedElements);
         Check(results, "execution output", DevelopmentChecks.Output);
         Check(results, "execution jobs", () => DevelopmentChecks.Jobs().GetAwaiter().GetResult());
         Check(results, "cancelled execution", () => DevelopmentChecks.CancelledStart().GetAwaiter().GetResult());

src/Anode/Core/Desktop/AccessibilityReader.cs

@@ -28,17 +28,35 @@ public static JsonObject Observe(WindowTarget target, JsonObject request)
         var elements = new JsonArray();
         var warnings = new JsonArray();
         var watch = Stopwatch.StartNew();
-        var queue = new Queue<(AutomationElement Element, int[] Path, string? Parent)>();
-        queue.Enqueue((root, Array.Empty<int>(), null));
+        // An app can list the same control twice in its tree, even below itself (Chrome appeared to while a <select> list
+        // was open). Walking it again would repeat its whole subtree until the budget ran out, so a control with a runtime
+        // ID already queued is skipped. Its index still counts, so the paths seat_element follows stay right.
+        var seen = new HashSet<string>();
+        int repeated = 0;
+        var queue = new Queue<(AutomationElement Element, int[]? RuntimeId, int[] Path, string? Parent)>();
+        int[]? rootId = RuntimeId(root);
+        if (rootId is { Length: > 0 }) seen.Add(Key(rootId));
+        queue.Enqueue((root, rootId, Array.Empty<int>(), null));
         bool truncated = false;
         int visited = 0;
         while (queue.Count > 0 && visited < maximum && watch.ElapsedMilliseconds < 4000)
         {
             var current = queue.Dequeue();
             visited++;
+            JsonObject? node = null;
+            string? key = current.RuntimeId is { Length: > 0 } queued ? Key(queued) : null;
             try
             {
-                var node = Describe(current.Element, current.Path, current.Parent, ref textBudget);
+                // A runtime ID that couldn't be read when queued is read again here, and its failure reported.
+                int[] runtimeId = current.RuntimeId ?? current.Element.GetRuntimeId() ?? Array.Empty<int>();
+                if (current.RuntimeId is null && runtimeId.Length > 0 && !seen.Add(Key(runtimeId)))
+                {
+                    visited--;
+                    repeated++;
+                    continue;
+                }
+                if (runtimeId.Length > 0) key = Key(runtimeId);
+                node = Describe(current.Element, runtimeId, current.Path, current.Parent, ref textBudget);
                 string? parent = current.Parent;
                 if (offscreen || node.Bool("offscreen") != true || current.Path.Length == 0)
                 {
@@ -48,29 +66,53 @@ public static JsonObject Observe(WindowTarget target, JsonObject request)
                 }
                 if (current.Path.Length >= depthLimit) { truncated = true; continue; }
                 int index = 0;
+                var siblings = new HashSet<string>();
                 var child = Walker.GetFirstChild(current.Element);
                 while (child is not null && queue.Count + visited < maximum && watch.ElapsedMilliseconds < 4000)
                 {
-                    queue.Enqueue((child, current.Path.Append(index++).ToArray(), parent));
+                    int[]? id = RuntimeId(child);
+                    string? childKey = id is { Length: > 0 } ? Key(id) : null;
+                    // A list of children that comes back to one it already gave has no more to give; repeats take no
+                    // budget, so only this stops it.
+                    if (childKey is not null && !siblings.Add(childKey)) { child = null; break; }
+                    if (childKey is not null && !seen.Add(childKey)) repeated++;
+                    else queue.Enqueue((child, id, current.Path.Append(index).ToArray(), parent));
+                    index++;
                     child = Walker.GetNextSibling(child);
                 }
                 if (child is not null) truncated = true;
             }
             catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or System.Runtime.InteropServices.COMException)
             {
+                // A control that couldn't be described lets a copy of it listed later be described instead.
+                if (node is null && key is not null) seen.Remove(key);
                 if (warnings.Count < 3) warnings.Add("A control changed or did not expose readable accessibility information.");
             }
         }
         truncated |= queue.Count > 0;
-        return new JsonObject
+        var result = new JsonObject
         {
             ["window"] = WindowAccess.Describe(target), ["elements"] = elements,
             ["truncated"] = truncated, ["warnings"] = warnings,
             ["observedAt"] = DateTime.UtcNow.ToString("o"), ["elapsedMs"] = watch.ElapsedMilliseconds
         };
+        // Not a warning: nothing was hidden, and seat_wait's "missing" needs a tree without warnings.
+        if (repeated > 0) result["skippedRepeats"] = repeated;
+        return result;
+
+        // Null when the control can't say yet; it is then read again, and its failure reported, when it is described.
+        static int[]? RuntimeId(AutomationElement element)
+        {
+            try { return element.GetRuntimeId(); }
+            catch (Exception ex) when (ex is ElementNotAvailableException or InvalidOperationException or System.Runtime.InteropServices.COMException)
+            {
+                return null;
+            }
+        }
+        static string Key(int[] id) => string.Join(".", id);
     }
 
-    private static JsonObject Describe(AutomationElement element, int[] path, string? parent, ref int textBudget)
+    private static JsonObject Describe(AutomationElement element, int[] runtimeId, int[] path, string? parent, ref int textBudget)
     {
         var cache = new CacheRequest { TreeScope = TreeScope.Element };
         foreach (var property in new[]
@@ -102,7 +144,7 @@ private static JsonObject Describe(AutomationElement element, int[] path, string
             ["actions"] = new JsonArray(actions.Select(value => (JsonNode)value).ToArray()),
             // Private references are retained by the host and removed from public results.
             ["path"] = new JsonArray(path.Select(value => (JsonNode)value).ToArray()),
-            ["runtimeId"] = new JsonArray(element.GetRuntimeId().Select(value => (JsonNode)value).ToArray()),
+            ["runtimeId"] = new JsonArray(runtimeId.Select(value => (JsonNode)value).ToArray()),
             ["processId"] = data.ProcessId
         };
         if (!password)
@@ -174,7 +216,7 @@ public static JsonObject Act(WindowTarget target, JsonObject reference, JsonObje
             if (element is null) throw Stale();
         }
         int[] runtimeId = reference["runtimeId"]!.AsArray().Select(value => value!.GetValue<int>()).ToArray();
-        if (!element.GetRuntimeId().SequenceEqual(runtimeId)) throw Stale();
+        if (!(element.GetRuntimeId() ?? Array.Empty<int>()).SequenceEqual(runtimeId)) throw Stale();
         var state = element.Current;
         VerifyElementSession(state.ProcessId);
         if (state.ProcessId != reference.Int("processId") || Clip(state.Name, 500) != reference.Str("name")

src/Anode/Core/Desktop/DesktopPresentation.cs

@@ -62,6 +62,8 @@ public static string Summary(JsonObject result)
                 if (node.Str("selectedText") is { Length: > 0 } selected) text.AppendLine("    selected: " + Line(selected));
             }
             if (result.Bool("truncated") == true) text.AppendLine("Tree truncated; adjust maxElements/maxDepth or inspect a smaller window.");
+            if (result.Int("skippedRepeats") is int repeats)
+                text.AppendLine($"Skipped {repeats} control(s) the app listed again elsewhere in this window's tree; each is shown once.");
             if (result.Str("screenshotError") is { } error) text.AppendLine(Line(error));
             if (result["warnings"] is JsonArray warnings)
                 foreach (var warning in warnings) text.AppendLine(Line(warning?.ToString()));