From 8dfca319fbe11ae50ba48e682501560f2d17a6ea Mon Sep 17 00:00:00 2001 From: jeffrey Date: Wed, 11 Nov 2026 22:16:52 +0800 Subject: [PATCH 01/30] feat(computer): browse watched sites in a window of the user's everyday Chrome The Computer's browser for sites that watch for automated browsing is now a window of its own in the user's everyday Chrome, with the user's sign-ins, history and device trust, instead of a separate Chrome profile. - launch_app with agent_browser opens one site per call in a new window through Chrome's command line, so no key reaches the user's windows. - A covered window keeps working only with --disable-backgrounding-occluded-windows. A Chrome running without it returns COMPUTER_BROWSER_RESTART_REQUIRED until the Agent passes restart_browser after the user agreed; the restart restores the session. - The user's Chrome takes no keys from the Computer (COMPUTER_KEYS_REFUSED): shortcuts run as menu commands in the window that has the keyboard, and Cua types text as keys that all read as the A key. Only a plain key bound to an element goes. Scripts there are guarded the same way. - The retired separate profile, computer-tools/agent-chrome, is removed when the Daemon starts and that Chrome no longer runs. - humanlike-browse 1.0.5 fills fields with set_value, presses the page's buttons and asks before a restart. Co-Authored-By: Claude Code --- apps/computer/local-daemon/README.md | 30 ++- .../linger/localtools/agent_browser.go | 238 +++++++++++++++++- .../linger/localtools/agent_chrome.go | 30 ++- .../internal/linger/localtools/cua.go | 16 +- .../internal/linger/localtools/cua_script.go | 10 +- .../localtools/cua_script_guard_test.go | 42 +++- .../linger/localtools/cua_script_test.go | 20 +- .../linger/localtools/localtools_test.go | 104 +++++++- .../internal/linger/localtools/manager.go | 34 ++- skills/builtin/catalog.yaml | 5 +- .../builtin/humanlike-browse/1.0.5/SKILL.md | 80 ++++++ 11 files changed, 554 insertions(+), 55 deletions(-) create mode 100644 skills/builtin/humanlike-browse/1.0.5/SKILL.md diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 609f66e3..5e3fd806 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -323,13 +323,29 @@ profile directory, stored under `computer-tools/browser-profiles/` in the state directory. Cua can observe the native Chrome window that Playwright connects to. On macOS, `launch_app` with `agent_browser: true` opens the Computer's own -browser for a site that watches for automated browsers. The Agent drives it -only with the computer tools, never through Playwright or a debugging port. -Chrome, Edge and Firefox each run as a separate instance with their own -profile under `computer-tools/agent-` in the state directory, so the -user signs in there once. Safari cannot run a second profile; the Computer -uses its own window in the user's Safari. Without a `bundle_id`, the Daemon -picks the first installed of Chrome, Edge, Firefox and Safari. +browser window for a site that watches for automated browsers. The Agent drives +it only with the computer tools, never through Playwright or a debugging port. +In Chrome it is a window of the Computer's own in the user's everyday Chrome, +with the user's sign-ins, history and device trust. The window opens through +Chrome's command line, one site per call, so no key reaches the user's other +windows. A covered window keeps working only when Chrome runs with +`--disable-backgrounding-occluded-windows`: a Chrome that is not running +starts with it, and one running without it returns +`COMPUTER_BROWSER_RESTART_REQUIRED` until the Agent passes `restart_browser` +after the user agreed. The restart quits Chrome as its menu does and reopens it +with `--restore-last-session`. The user's Chrome takes no keys from the +Computer (`COMPUTER_KEYS_REFUSED`): a shortcut runs as a menu command in the +Chrome window that has the keyboard, and Cua types text as keys that all read +as the A key. Only a plain key bound to an element goes, and Cua sends it only +when no other Chrome window could take it. The Agent fills fields with +`set_value`, presses the page's buttons and goes back through its links. +The separate Chrome profile earlier versions used, `computer-tools/agent-chrome`, +is removed when the Daemon starts and that Chrome no longer runs. +Edge and Firefox each run as a separate instance with their own profile under +`computer-tools/agent-` in the state directory, so the user signs in +there once. Safari cannot run a second profile; the Computer uses its own +window in the user's Safari. Without a `bundle_id`, the Daemon picks the first +installed of Chrome, Edge, Firefox and Safari. It opens a site only at its home page and refuses any other web address with `COMPUTER_HOME_PAGE_ONLY`: the Agent reaches a search, listing or item through the site's own search box, menus and links, as a person does. diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index fa0d2cdd..fb19efe7 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -10,7 +10,9 @@ import ( "os/exec" "path/filepath" "slices" + "strconv" "strings" + "syscall" "time" "github.com/simplexai-labs/linger/apps/computer/local-daemon/internal/platform/privatefs" @@ -44,6 +46,11 @@ type agentBrowser struct { // Safari does: it has no separate profile, so the Computer works in a // window of its own there, with the user's sign-ins. Shared bool + // Everyday marks the user's everyday Chrome: the Computer opens each + // site in a window of its own there, by command line, and the running + // instance must carry Settings, so a covered window keeps working. One + // started without them restarts only after the user agreed. + Everyday bool } // chromiumArgs start a Chromium browser as the Computer's own. Without @@ -85,9 +92,17 @@ user_pref("browser.translations.automaticallyPopup", false); user_pref("media.autoplay.default", 5); ` +// everydayChromeArgs are what the user's everyday Chrome needs for the +// Computer's window in it: see chromiumArgs for why a covered window stops +// working without it. Sites see no other difference. +var everydayChromeArgs = []string{"--disable-backgrounding-occluded-windows"} + var agentBrowsers = []agentBrowser{ - {Name: "Chrome", Bundle: "com.google.Chrome", App: "Google Chrome.app", Directory: "agent-chrome", Args: chromium, - Settings: append(slices.Clone(chromiumArgs), "-AppleLanguages"), AddressRole: "AXTextField", AddressLabel: "Address and search bar"}, + // Chrome is the user's everyday Chrome, with their sign-ins, history and + // device trust: sites that watch for automated browsing trust that + // browser most, and the user signs in only once. + {Name: "Chrome", Bundle: "com.google.Chrome", App: "Google Chrome.app", Settings: everydayChromeArgs, + AddressRole: "AXTextField", AddressLabel: "Address and search bar", Shared: true, Everyday: true}, {Name: "Edge", Bundle: "com.microsoft.edgemac", App: "Microsoft Edge.app", Directory: "agent-edge", Args: chromium, Settings: append(slices.Clone(chromiumArgs), "-AppleLanguages"), AddressRole: "AXTextField", AddressLabel: "Address and search bar"}, // Firefox keeps its window working while covered without a flag. Its @@ -179,8 +194,8 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls return nil, err } m.mu.Lock() - own := m.agentSafariWindow - if m.agentSafariPID != pid { + own := m.sharedBrowserWindow + if m.sharedBrowserPID != pid { own = 0 } m.mu.Unlock() @@ -224,7 +239,7 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls c.agentChromeTabsUntil(ctx, pid, window.WindowID, agentSafariWindowWait, func(n int) bool { return n > 0 }) } m.mu.Lock() - m.agentSafariPID, m.agentSafariWindow = pid, window.WindowID + m.sharedBrowserBundle, m.sharedBrowserPID, m.sharedBrowserWindow = b.Bundle, pid, window.WindowID m.mu.Unlock() for _, url := range urls { if err := c.agentChromeOpenTab(ctx, b, pid, window.WindowID, offSpace(window), url); err != nil { @@ -273,3 +288,216 @@ func (c *cuaSession) waitSafariWindows(ctx context.Context, b agentBrowser, pid } } } + +// everydayChromePID finds the user's everyday Chrome: the main process of +// Google Chrome on its default profile. Chrome instances on a profile of their +// own, such as a test browser, name --user-data-dir and are not it. It returns +// the pid and command line, or 0 when that Chrome is not running. +var everydayChromePID = func(ctx context.Context) (int, string, error) { + output, err := exec.CommandContext(ctx, "/bin/ps", "-axww", "-o", "pid=,args=").Output() + if err != nil { + return 0, "", err + } + for _, line := range strings.Split(string(output), "\n") { + pidText, args, _ := strings.Cut(strings.TrimSpace(line), " ") + if !strings.HasPrefix(args, everydayChromeExecutable) || strings.Contains(args, "--type=") || strings.Contains(args, "--user-data-dir=") { + continue + } + if pid, err := strconv.Atoi(pidText); err == nil { + return pid, args, nil + } + } + return 0, "", nil +} + +const everydayChromeExecutable = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" + +// stopEverydayChrome asks the user's Chrome to quit, as quitting from its +// menu does, and waits until it is gone. Chrome saves its session first, so +// the restart brings back the user's windows and tabs. +var stopEverydayChrome = func(ctx context.Context, pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return err + } + if err := process.Signal(syscall.SIGTERM); err != nil { + return err + } + for deadline := time.Now().Add(everydayChromeQuitWait); ; { + if running, _, err := everydayChromePID(ctx); err != nil || running == 0 { + return err + } + if time.Now().After(deadline) { + return errors.New("COMPUTER_BROWSER_RESTART_FAILED: the user's Chrome did not quit, so it was not restarted; ask the user to quit Chrome and try again") + } + if err := sleepContext(ctx, 200*time.Millisecond); err != nil { + return err + } + } +} + +// everydayChromeQuitWait bounds how long the user's Chrome may take to quit. +var everydayChromeQuitWait = 20 * time.Second + +// sharedBrowserRunning returns the pid of the user's own browser named by +// bundle, or 0. +func sharedBrowserRunning(ctx context.Context, bundle string) (int, error) { + if bundle == "com.google.Chrome" { + pid, _, err := everydayChromePID(ctx) + return pid, err + } + return lsappPID(ctx, bundle) +} + +// everydayChromeRestart asks for the user's consent before the Computer +// restarts their Chrome. +const everydayChromeRestart = "COMPUTER_BROWSER_RESTART_REQUIRED: the user's Chrome runs without the setting that keeps the Computer's window working while other windows cover it, and Chrome takes it only when it starts; nothing was opened. Ask the user with a question card whether the Computer may restart Chrome now: Chrome quits and reopens with their windows and tabs, but text typed into a page and not yet sent may be lost. If the user agrees, call launch_app again with restart_browser: true; if not, end the task" + +// launchEverydayChrome opens url in a new window of the user's everyday +// Chrome, the Computer's own window there, which the task then works in. The +// window opens through Chrome's command line, as a link from another app +// does, so no key reaches the user's other Chrome windows. A Chrome that is +// not running starts with the setting the window needs; one running without +// it restarts only when restart says the user agreed. +func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, urls []string, restart bool) (json.RawMessage, error) { + if len(urls) != 1 { + return nil, fmt.Errorf("launch_app opens one site at a time in the user's Chrome; call it once for each site: %w", errNotSent) + } + pid, command, err := everydayChromePID(ctx) + if err != nil { + return nil, err + } + restarted := false + if pid > 0 && !b.startedWith(command) { + if !restart { + return nil, fmt.Errorf("%s: %w", everydayChromeRestart, errNotSent) + } + if err := stopEverydayChrome(ctx, pid); err != nil { + return nil, err + } + pid, restarted = 0, true + } + if pid == 0 { + // Chrome starts with the setting first, bringing back the user's + // windows after a restart; the Computer's window opens after them, + // so it is the one window that was not there before. + args := append([]string{"-g", "-b", b.Bundle, "--args"}, b.Settings...) + if restarted { + args = append(args, "--restore-last-session") + } + if output, err := agentChromeOpen(ctx, args); err != nil { + return nil, fmt.Errorf("launch_app: %s: %w", strings.TrimSpace(string(output)), err) + } + if pid, err = c.waitEverydayChrome(ctx); err != nil { + return nil, err + } + } + windows, err := c.appWindows(ctx, pid) + if err != nil { + return nil, err + } + before := map[float64]bool{} + for _, w := range windows { + before[w.WindowID] = true + } + if output, err := agentChromeOpen(ctx, []string{"-n", "-g", "-b", b.Bundle, "--args", "--new-window", urls[0]}); err != nil { + return nil, fmt.Errorf("launch_app: %s: %w", strings.TrimSpace(string(output)), err) + } + var window listedWindow + tabs := 0 + for deadline := time.Now().Add(agentSafariWindowWait); window.WindowID == 0; { + windows, err := c.appWindows(ctx, pid) + if err != nil { + return nil, err + } + for _, w := range windows { + if !before[w.WindowID] { + if n := c.agentChromeTabs(ctx, pid, w.WindowID); n > 0 { + window, tabs = w, n + break + } + } + } + if window.WindowID != 0 { + break + } + if time.Now().After(deadline) { + return nil, errors.New("launch_app: the user's Chrome did not open a window for the Computer") + } + if err := sleepContext(ctx, 200*time.Millisecond); err != nil { + return nil, err + } + } + m := c.op.manager + m.mu.Lock() + m.sharedBrowserBundle, m.sharedBrowserPID, m.sharedBrowserWindow = b.Bundle, pid, window.WindowID + m.mu.Unlock() + c.op.mu.Lock() + c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = tabs, pid, window.WindowID + c.op.mu.Unlock() + note := "This is the Computer's own window in the user's everyday Chrome, with the user's sign-ins; work only in this window, never in the user's other Chrome windows. It works while other windows cover it, but only on the desktop (Space) the user is on. The user's Chrome takes no keys from the Computer, because a key could reach the window the user is typing in: fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links. Close a tab with the Close button of its tab; closing the window's last tab closes the window. If a site asks to sign in, ask the user to sign in once in this window." + if restarted { + note = "Restarted the user's Chrome with their windows and tabs. " + note + } + text := fmt.Sprintf("Opened %s in a new window of the user's Chrome (pid %d), window %q [window_id: %v]. The window has %d tab(s).\n%s", urls[0], pid, window.Title, window.WindowID, tabs, note) + return json.Marshal(map[string]any{"content": []any{map[string]string{"type": "text", "text": text}}, + "structuredContent": map[string]any{"browser": b.Name, "pid": pid, "windows": []map[string]any{{"window_id": window.WindowID, "title": window.Title, "on_screen": window.IsOnScreen}}, "tabs": tabs, "note": note}}) +} + +// waitEverydayChrome waits until the user's Chrome that just started runs +// and shows its windows, and returns its pid. +func (c *cuaSession) waitEverydayChrome(ctx context.Context) (int, error) { + for deadline := time.Now().Add(agentSafariWindowWait); ; { + pid, _, err := everydayChromePID(ctx) + if err != nil { + return 0, err + } + if pid > 0 { + if windows, err := c.appWindows(ctx, pid); err != nil { + return 0, err + } else if len(windows) > 0 { + // Restored windows appear one after another; a short settle + // keeps a late one from reading as the Computer's. + return pid, sleepContext(ctx, everydayChromeSettle) + } + } + if time.Now().After(deadline) { + return 0, errors.New("launch_app: the user's Chrome did not start") + } + if err := sleepContext(ctx, 200*time.Millisecond); err != nil { + return 0, err + } + } +} + +// everydayChromeSettle is how long a just-started Chrome gets to reopen the +// user's windows. +var everydayChromeSettle = 2 * time.Second + +// everydayChrome reports whether pid is the user's everyday Chrome that the +// Computer opened its window in. +func (c *cuaSession) everydayChrome(pid any) bool { + number, ok := pid.(float64) + m := c.op.manager + m.mu.Lock() + defer m.mu.Unlock() + return ok && m.sharedBrowserPID > 0 && int(number) == m.sharedBrowserPID && m.sharedBrowserBundle == "com.google.Chrome" +} + +// everydayChromeKeysRefused tells the Agent how to work without keys in the +// user's everyday Chrome. +const everydayChromeKeysRefused = "COMPUTER_KEYS_REFUSED: the user's everyday Chrome takes no keys from the Computer: a key or shortcut goes to the Chrome window that has the keyboard, which may be the one the user is typing in; nothing was sent. Fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links" + +// everydayChromeKeys refuses keys to the user's everyday Chrome. Only one +// plain key bound to an element, such as Return in a search box, may go: +// Cua delivers it only when no other Chrome window could receive it. +// Shortcuts are menu commands, which Chrome runs in the window that has the +// keyboard, and typed text arrives as keys that all read as the A key. +func everydayChromeKeys(name string, arguments map[string]any) error { + modifiers, _ := arguments["modifiers"].([]any) + _, bound := arguments["element_token"].(string) + if name == "hotkey" || name == "type_text" || name == "press_key" && (!bound || len(modifiers) > 0) { + return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) + } + return nil +} diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index d244b4b5..feff01de 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -58,7 +58,7 @@ var agentChromePID = func(ctx context.Context, directory string) (int, string, e // brings its first window forward once, where the user is. Later URLs open // in a new tab through the window's accessibility tree and one Return key to // that browser alone, so the user's apps, focus and Space stay as they are. -func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls []string) (json.RawMessage, error) { +func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls []string, restart bool) (json.RawMessage, error) { var sites []string for _, address := range urls { if site := siteOf(address); site != "" { @@ -78,6 +78,9 @@ func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls if _, err := c.paceAgentChrome(ctx, 0); err != nil { return nil, err } + if b.Everyday { + return c.launchEverydayChrome(ctx, b, urls, restart) + } if b.Shared { return c.launchAgentSafari(ctx, b, urls) } @@ -413,8 +416,9 @@ var agentChromeRecheck = 30 * time.Second // because the Computer's own browser gets a new pid when it restarts. A failed // lookup refuses the input instead of treating the app as another one. // -// In the user's Safari only the window the Computer opened is its own; an -// input that names no window goes to that window. +// In the user's own browser, such as their everyday Chrome or Safari, only +// the window the Computer opened is its own; an input that names no window +// goes to that window. func (c *cuaSession) isAgentChrome(ctx context.Context, pid, window any) (bool, error) { number, ok := pid.(float64) if !ok || runtime.GOOS != "darwin" { @@ -423,11 +427,11 @@ func (c *cuaSession) isAgentChrome(ctx context.Context, pid, window any) (bool, m := c.op.manager m.mu.Lock() known, checked := m.agentChromePID, m.agentChromeChecked - safari, safariWindow := m.agentSafariPID, m.agentSafariWindow + shared, sharedWindow := m.sharedBrowserPID, m.sharedBrowserWindow m.mu.Unlock() - if safari > 0 && safari == int(number) { + if shared > 0 && shared == int(number) { id, named := window.(float64) - return !named || id == safariWindow, nil + return !named || id == sharedWindow, nil } if known > 0 && known == int(number) && time.Since(checked) < agentChromeRecheck { return true, nil @@ -483,11 +487,11 @@ func (c *cuaSession) agentChromeRunning(ctx context.Context) (int, error) { } m := c.op.manager m.mu.Lock() - safari := m.agentSafariPID + shared, bundle := m.sharedBrowserPID, m.sharedBrowserBundle m.mu.Unlock() - if own == 0 && safari > 0 { - if running, _ := lsappPID(ctx, "com.apple.Safari"); running == safari { - return safari, nil + if own == 0 && shared > 0 { + if running, _ := sharedBrowserRunning(ctx, bundle); running == shared { + return shared, nil } } m.mu.Lock() @@ -546,10 +550,10 @@ func (c *cuaSession) agentChromeKeys(ctx context.Context, arguments map[string]a } m := c.op.manager m.mu.Lock() - safari, safariWindow := m.agentSafariPID, m.agentSafariWindow + shared, sharedWindow := m.sharedBrowserPID, m.sharedBrowserWindow m.mu.Unlock() - if pid, ok := arguments["pid"].(float64); ok && safari > 0 && int(pid) == safari { - arguments["window_id"] = safariWindow + if pid, ok := arguments["pid"].(float64); ok && shared > 0 && int(pid) == shared { + arguments["window_id"] = sharedWindow return nil } windows, err := c.appWindows(ctx, arguments["pid"]) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 60d71c98..2533e485 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -160,7 +160,8 @@ func (m *Manager) openCua(ctx context.Context, op *operation) (*cuaSession, []to if item.Name == "launch_app" && runtime.GOOS == "darwin" { item.Description = "Open a macOS app in the background through Launch Services, as `open -g` does, and return its pid and windows. Optional urls are handed to the app; for Finder, pass a folder to open it in a window." item.InputSchema = schema(map[string]any{"bundle_id": map[string]string{"type": "string", "description": "Preferred, for example com.apple.TextEdit."}, "name": map[string]string{"type": "string", "description": "App name, when the bundle id is unknown."}, "urls": map[string]any{"type": "array", "items": map[string]string{"type": "string"}, "description": "Files or folders to open in the app."}, - "agent_browser": map[string]string{"type": "boolean", "description": "Open the URLs in the Computer's own browser, which keeps working while other windows cover it on the user's desktop and has its own sign-ins, instead of the user's everyday browser. Only a site's home page opens there, such as https://www.example.com/. With no bundle_id it is the first installed of Chrome, Edge, Firefox and Safari; a bundle_id of com.google.Chrome, com.microsoft.edgemac, org.mozilla.firefox or com.apple.Safari picks one. Safari has no separate profile, so there it is a window of its own in the user's Safari, with the user's sign-ins."}}) + "agent_browser": map[string]string{"type": "boolean", "description": "Open the URLs in the Computer's own browser window, which keeps working while other windows cover it on the user's desktop. Only a site's home page opens there, such as https://www.example.com/. With no bundle_id it is the first installed of Chrome, Edge, Firefox and Safari; a bundle_id of com.google.Chrome, com.microsoft.edgemac, org.mozilla.firefox or com.apple.Safari picks one. In Chrome and Safari it is a window of the Computer's own in the user's everyday browser, with the user's sign-ins; in Chrome it opens one site per call. Edge and Firefox run as separate instances with their own sign-ins."}, + "restart_browser": map[string]string{"type": "boolean", "description": "With agent_browser in Chrome: restart the user's Chrome so the Computer's window keeps working while covered. Pass it only after the user agreed to a COMPUTER_BROWSER_RESTART_REQUIRED question."}}) } if item.Name == "get_desktop_state" { item.Description += " Read-only whole-screen view; act through the window tools with a pid and window_id. A system privacy prompt about CuaDriver belongs to the user: never answer it, and keep working through the window tools." @@ -223,7 +224,7 @@ const MacToolGuide = `Computer tools on this Mac (MCP server "computer"): - report_blocker({kind, url}) as soon as a page in the Computer's own browser shows a check only the user may pass (captcha, verification, login_required, rate_limited) or one item that cannot be viewed (content_unavailable). Never try to pass the check. The Computer's own browser takes input at a person's pace, within a budget per task and per day, and a script there sends at most 20 inputs: return what you have and look at the window before the next script. A tab that a link in the script opens is the task's and is in front afterwards. The browser tools stay off every site a task opened in that browser. Plan as many steps as you can in one run_actions call, as in one script; its read shows their effect, so do not read again before the next input unless you need another window. Keys are often shorter than clicks: in Finder, cmd+shift+g then a path and Return opens a folder, cmd+down opens the selected item, and in the Trash cmd+delete puts the selected item back. -In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser instead: it keeps working while covered and has its own sign-ins, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input: press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. +In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. In the Computer's window of the user's everyday Chrome no key or shortcut is sent: fill fields with set_value, press the page's buttons, and go back through its links. In Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. A menu item in the read, such as File > Put Back, can be clicked directly; do not click a menu bar title first, because an open menu holds the app until it closes. A right_click menu does not show in the read, so select the row with click and use the menu bar item or its keyboard shortcut. Before the first shell screenshot, or when one fails or comes back blank, run ` + "`teamhub computer permissions status`" + `; if Screen Recording is missing for Linger, run ` + "`teamhub computer permissions request --permissions screen_recording`" + ` and end your turn — the card wakes you. Never operate System Settings or privacy prompts yourself. ` + browserToolGuide @@ -438,8 +439,14 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage return nil, err } } + everyday := paced && c.everydayChrome(target["pid"]) + if everyday && name != "run_script" { + if err := everydayChromeKeys(name, arguments); err != nil { + return nil, err + } + } if name == "run_script" { - if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs); err != nil { + if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, everyday); err != nil { return nil, err } } @@ -1951,7 +1958,8 @@ func (c *cuaSession) launchApp(ctx context.Context, arguments map[string]any) (j if err != nil { return nil, err } - return c.launchAgentChrome(ctx, b, addresses) + restart, _ := arguments["restart_browser"].(bool) + return c.launchAgentChrome(ctx, b, addresses, restart) } app, flag := bundle, "-b" if app == "" { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index ebb032fd..e32a1ac1 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -69,7 +69,7 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // Computer's own browser: the guard waits a person's pause before each input, // throws once the script tries to send more than inputLimit inputs, and // refuses launch_app, whose site checks run only in the Daemon's own call. -func guardedScript(args, body string, deadline int64, inputLimit int) string { +func guardedScript(args, body string, deadline int64, inputLimit int, noKeys bool) string { pattern := make([]string, 0, len(scriptForbidden)) for _, word := range scriptForbidden { pattern = append(pattern, regexp.QuoteMeta(word)) @@ -80,7 +80,8 @@ const keys = Object.keys, names = Object.getOwnPropertyNames, proto = Object.get const apply = Reflect.apply, stringify = JSON.stringify, parse = JSON.parse, test = RegExp.prototype.test, then = Promise.prototype.then; const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis.cua; const bad = /`+strings.Join(pattern, "|")+`/; -const limit = `+strconv.Itoa(inputLimit)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; +const limit = `+strconv.Itoa(inputLimit)+`, noKeys = `+strconv.FormatBool(noKeys)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; +const keyed = (k, c) => k === "hotkey" || k === "typeText" || k === "pressKey" && (c.length < 2 || c[1] === null || c[1] === undefined || typeof c[0] !== "string" || c[0].indexOf("+") >= 0) || k === "call" && (c[0] === "hotkey" || c[0] === "type_text" || c[0] === "press_key" && (c[1] === null || typeof c[1] !== "object" || typeof c[1].element_token !== "string" || c[1].modifiers !== undefined)); const inputMethods = {__proto__: null, click: true, doubleClick: true, rightClick: true, drag: true, scroll: true, typeText: true, pressKey: true, hotkey: true, setValue: true}; const inputOps = {__proto__: null, click: true, double_click: true, right_click: true, drag: true, scroll: true, type_text: true, press_key: true, hotkey: true, set_value: true}; const input = (k, c) => inputMethods[k] === true || k === "call" && typeof c[0] === "string" && inputOps[c[0]] === true; @@ -97,6 +98,7 @@ if (limit > 0 && k === "typeText") { const given = c.length > 2 && c[2] !== null if (limit > 0 && k === "sleep" && o === real && typeof c[0] === "number" && c[0] > `+strconv.Itoa(scriptSleepLimit)+`) { const t = {length: 1}; define(t, 0, {value: `+strconv.Itoa(scriptSleepLimit)+`}); c = t; } const run = () => { const r = apply(v, o, c); return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [out]) : out(r); }; if (limit > 0 && (k === "launchApp" || k === "call" && c[0] === "launch_app")) throw new Error("COMPUTER_SCRIPT_REFUSED: a script in the Computer's own browser opens no app or URL, so the site checks of launch_app hold; call launch_app as its own tool"); +if (noKeys && keyed(k, c)) throw new Error(`+strconv.Quote(everydayChromeKeysRefused)+`); if (limit <= 0 || !input(k, c)) return run(); if (++sent > limit) throw new Error("COMPUTER_SCRIPT_INPUT_LIMIT: a script sends at most " + limit + " inputs to the Computer's own browser; return what you have, look at the window, then continue in a new script"); return apply(then, apply(rest, real, [`+strconv.Itoa(scriptInputPauseMin)+` + floor(random() * `+strconv.Itoa(scriptInputPauseSpan)+`)]), [run]); }; }}); } } @@ -135,7 +137,7 @@ func forbiddenKey(value any, word string) bool { // into what remains of the call after waiting for the desktop lane. In a // reviewed Run, whose review scope is not empty, a script file runs only as // the judge allowed it in that scope: the body it read and the args it saw. -func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope string, inputLimit int) error { +func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope string, inputLimit int, noKeys bool) error { for key := range arguments { switch key { case "script", "script_file", "args", "timeout_ms", "max_calls": @@ -194,7 +196,7 @@ func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope arguments["timeout_ms"] = timeout delete(arguments, "script_file") delete(arguments, "args") - arguments["script"] = guardedScript(string(encoded), script, time.Now().Add(time.Duration(timeout)*time.Millisecond).UnixMilli(), inputLimit) + arguments["script"] = guardedScript(string(encoded), script, time.Now().Add(time.Duration(timeout)*time.Millisecond).UnixMilli(), inputLimit, noKeys) return nil } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 7376be87..352d385f 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -55,7 +55,7 @@ func TestScriptGuardRefusesForbiddenInputInAJSEngine(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{"n":1}`, tc.body, 42, 0)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{"n":1}`, tc.body, 42, 0, false)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -84,7 +84,7 @@ func TestScriptGuardPacesInputsForTheComputersChrome(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 2)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 2, false)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -112,7 +112,7 @@ func TestScriptGuardSpeedsTypingAndCapsWaits(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 20)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 20, false)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -145,7 +145,7 @@ func TestScriptGuardRefusesLaunchAppForTheComputersChrome(t *testing.T) { want string }{{2, "threw:COMPUTER_SCRIPT_REFUSED"}, {0, "call:launch_app"}} { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, tc.limit)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, tc.limit, false)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -157,3 +157,37 @@ func TestScriptGuardRefusesLaunchAppForTheComputersChrome(t *testing.T) { } } } + +// In the user's everyday Chrome a script sends no shortcut, typed text or +// unbound key; a plain key bound to an element and other input still go. +func TestScriptGuardRefusesKeysForTheEverydayChrome(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node is not installed") + } + dir := t.TempDir() + harness := filepath.Join(dir, "harness.mjs") + if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ body, want string }{ + {`return await cua.call("hotkey", {pid: 1, keys: ["cmd", "["]});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1}); return app.typeText("kettle", {role: "AXTextField"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, key: "return"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2", modifiers: ["cmd"]});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2"});`, "call:press_key"}, + {`return await cua.call("click", {pid: 1, element_token: "s1:2"});`, "call:click"}, + } { + script := filepath.Join(dir, "script.js") + if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, 20, true)), 0o600); err != nil { + t.Fatal(err) + } + out, err := exec.Command(node, harness, script).CombinedOutput() + if err != nil { + t.Fatalf("node: %v\n%s", err, out) + } + if got := string(out); !strings.Contains(got, tc.want) { + t.Fatalf("%s: got %s, want %q", tc.body, got, tc.want) + } + } +} diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go index 893d8919..ab2cdb26 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go @@ -16,22 +16,22 @@ import ( func TestScriptArgumentsBoundTheScript(t *testing.T) { arguments := map[string]any{"script": "return 1;"} - if err := scriptArguments(context.Background(), arguments, "", 0); err != nil || arguments["timeout_ms"] != float64(scriptTimeoutMs) { + if err := scriptArguments(context.Background(), arguments, "", 0, false); err != nil || arguments["timeout_ms"] != float64(scriptTimeoutMs) { t.Fatalf("default: %v %v", err, arguments) } arguments = map[string]any{"script": "return 1;", "timeout_ms": float64(500000)} - if err := scriptArguments(context.Background(), arguments, "", 0); err != nil || arguments["timeout_ms"] != float64(scriptMaxTimeoutMs) { + if err := scriptArguments(context.Background(), arguments, "", 0, false); err != nil || arguments["timeout_ms"] != float64(scriptMaxTimeoutMs) { t.Fatalf("cap: %v %v", err, arguments) } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() arguments = map[string]any{"script": "return 1;", "timeout_ms": float64(90000)} - if err := scriptArguments(ctx, arguments, "", 0); err != nil || arguments["timeout_ms"].(float64) > 45000 { + if err := scriptArguments(ctx, arguments, "", 0, false); err != nil || arguments["timeout_ms"].(float64) > 45000 { t.Fatalf("deadline: %v %v", err, arguments) } late, stop := context.WithTimeout(context.Background(), 10*time.Second) defer stop() - if err := scriptArguments(late, map[string]any{"script": "return 1;"}, "", 0); !errors.Is(err, errNotSent) { + if err := scriptArguments(late, map[string]any{"script": "return 1;"}, "", 0, false); !errors.Is(err, errNotSent) { t.Fatalf("late: %v", err) } for _, refused := range []map[string]any{ @@ -40,7 +40,7 @@ func TestScriptArgumentsBoundTheScript(t *testing.T) { {"script": `await cua.call("click", {pid: 1, x: 2, y: 3, delivery_mode: "foreground"})`}, {"script": `await app.getState({screenshot_out_file: "/tmp/a.png"})`}, } { - if err := scriptArguments(context.Background(), refused, "", 0); !errors.Is(err, errNotSent) { + if err := scriptArguments(context.Background(), refused, "", 0, false); !errors.Is(err, errNotSent) { t.Fatalf("%v: got %v", refused, err) } } @@ -56,7 +56,7 @@ func TestScriptArgumentsReadASkillScriptWithArgs(t *testing.T) { t.Fatal(err) } arguments := map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}} - if err := scriptArguments(context.Background(), arguments, "", 0); err != nil { + if err := scriptArguments(context.Background(), arguments, "", 0, false); err != nil { t.Fatal(err) } script, _ := arguments["script"].(string) @@ -70,19 +70,19 @@ func TestScriptArgumentsReadASkillScriptWithArgs(t *testing.T) { {"script": `return cua.call("click", {delivery_mode: "foreground"});`}, {"script_file": file, "args": map[string]any{"click": []any{map[string]any{"delivery_mode": "foreground"}}}}, } { - if err := scriptArguments(context.Background(), refused, "", 0); !errors.Is(err, errNotSent) { + if err := scriptArguments(context.Background(), refused, "", 0, false); !errors.Is(err, errNotSent) { t.Fatalf("%v: got %v", refused, err) } } // args carry page and user text, which may name a forbidden option. - if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"text": "in the foreground"}}, "", 0); err != nil { + if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"text": "in the foreground"}}, "", 0, false); err != nil { t.Fatalf("user text in args: %v", err) } // A reviewed Run runs a script file only as a review allowed it. - if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}}, permission.ScriptScope("agent-1", "context-1"), 0); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_SCRIPT_REFUSED") { + if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}}, permission.ScriptScope("agent-1", "context-1"), 0, false); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_SCRIPT_REFUSED") { t.Fatalf("unreviewed script file in a reviewed Run: %v", err) } - if err := scriptArguments(context.Background(), map[string]any{"script": "return 1;"}, permission.ScriptScope("agent-1", "context-1"), 0); err != nil { + if err := scriptArguments(context.Background(), map[string]any{"script": "return 1;"}, permission.ScriptScope("agent-1", "context-1"), 0, false); err != nil { t.Fatalf("inline script in a reviewed Run: %v", err) } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 51089d6d..f54e4c6b 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -205,6 +205,13 @@ func runToolFixture(mode string) { } case "list_windows": windows := []any{map[string]any{"pid": 42, "window_id": 9, "title": "b", "z_index": 3, "is_on_screen": true, "bounds": map[string]any{"width": 800, "height": 600}}} + if mode == "tabs-everyday" { + // Each start or command line of the user's Chrome + // shows one more window beside the user's own. + for i := range strings.Count(string(data), `"name":"open"`) { + windows = append(windows, map[string]any{"pid": 42, "window_id": 11 + i, "title": "Home", "z_index": 2, "is_on_screen": true, "bounds": map[string]any{"width": 900, "height": 700}}) + } + } if mode == "tabs-ghost" { // A starting Chrome lists a larger surface with no // accessibility window beside its browser window. @@ -1853,6 +1860,101 @@ func TestLocalToolCuaDeltaCountFailureReadsFullWindowOnlyOnce(t *testing.T) { // A launch opens a fresh window when the listed one has left the // accessibility tree. +func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("the Computer's window in the user's Chrome is macOS only") + } + inputSettle = 0 + t.Cleanup(func() { inputSettle = 300 * time.Millisecond }) + pause := agentChromePause + agentChromePause = func() time.Duration { return 0 } + t.Cleanup(func() { agentChromePause = pause }) + marker := filepath.Join(t.TempDir(), "calls.jsonl") + c := cuaFixtureSession(t, "tabs-everyday", marker) + command := everydayChromeExecutable + pid := everydayChromePID + everydayChromePID = func(context.Context) (int, string, error) { return 42, command, nil } + t.Cleanup(func() { everydayChromePID = pid }) + stopped := 0 + stop := stopEverydayChrome + stopEverydayChrome = func(context.Context, int) error { + stopped++ + command = everydayChromeExecutable + " --disable-backgrounding-occluded-windows" + return nil + } + t.Cleanup(func() { stopEverydayChrome = stop }) + var opened [][]string + open := agentChromeOpen + agentChromeOpen = func(_ context.Context, args []string) ([]byte, error) { + opened = append(opened, args) + f, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, err + } + defer f.Close() + _, err = f.WriteString(`{"name":"open"}` + "\n") + return nil, err + } + t.Cleanup(func() { agentChromeOpen = open }) + settle := everydayChromeSettle + everydayChromeSettle = 0 + t.Cleanup(func() { everydayChromeSettle = settle }) + b := agentBrowsers[0] + // A Chrome started without the setting restarts only with consent. + if _, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}, false); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_BROWSER_RESTART_REQUIRED") || stopped != 0 || len(opened) != 0 { + t.Fatalf("err=%v stopped=%d opened=%v, want a refused launch", err, stopped, opened) + } + result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}, true) + if err != nil { + t.Fatal(err) + } + if stopped != 1 || len(opened) != 2 || !slices.Contains(opened[0], "--restore-last-session") || !slices.Contains(opened[0], "--disable-backgrounding-occluded-windows") || !slices.Equal(opened[1][len(opened[1])-2:], []string{"--new-window", "https://example.com/"}) { + t.Fatalf("stopped=%d opened=%v, want a restart, then the site in a new window", stopped, opened) + } + if !strings.Contains(string(result), `"window_id":12`) || !strings.Contains(string(result), "takes no keys") { + t.Fatalf("result=%s, want the new window 12 after the restored one, and the note", result) + } + // The new window is the Computer's: input to it is paced and held, and + // the user's own window 9 is not the Computer's. + if own, err := c.isAgentChrome(t.Context(), float64(42), float64(12)); err != nil || !own { + t.Fatalf("own=%v err=%v, want window 12 as the Computer's", own, err) + } + for _, user := range []float64{9, 11} { + if own, _ := c.isAgentChrome(t.Context(), float64(42), user); own { + t.Fatalf("the user's own Chrome window %v read as the Computer's", user) + } + } + for _, call := range []string{ + `{"pid":42,"window_id":12,"keys":["cmd","["]}`, + } { + if _, err := c.call(t.Context(), "hotkey", json.RawMessage(call)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { + t.Fatalf("hotkey err=%v, want refused", err) + } + } + if _, err := c.call(t.Context(), "type_text", json.RawMessage(`{"pid":42,"window_id":12,"text":"kettle","element_token":"s1:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { + t.Fatalf("type_text err=%v, want refused", err) + } + if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":12,"key":"return"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { + t.Fatalf("unbound press_key err=%v, want refused", err) + } + if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":12,"key":"return","element_token":"s1:1"}`)); err != nil { + t.Fatalf("Return bound to a field: %v", err) + } + if _, err := c.call(t.Context(), "set_value", json.RawMessage(`{"pid":42,"window_id":12,"value":"kettle","element_token":"s1:1"}`)); err != nil { + t.Fatalf("set_value: %v", err) + } + // A script there may not send keys either. + if _, err := c.call(t.Context(), "run_script", json.RawMessage(`{"script":"return 1"}`)); err != nil { + t.Fatal(err) + } + if calls := fixtureCalls(t, marker); !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "noKeys = true") { + t.Fatalf("last call=%v, want a script guarded against keys", calls[len(calls)-1]) + } + if err := c.drain(); err != nil { + t.Fatal(err) + } +} + func TestLocalToolCuaMacAgentChromeOpensAFreshWindowForAnUnreadableOne(t *testing.T) { if runtime.GOOS != "darwin" { t.Skip("the Computer's own Chrome is macOS only") @@ -1882,7 +1984,7 @@ func TestLocalToolCuaMacAgentChromeOpensAFreshWindowForAnUnreadableOne(t *testin } t.Cleanup(func() { agentChromeOpen = open }) b := agentBrowser{Name: "Chrome", Bundle: "com.google.Chrome", Directory: "agent-chrome-test", Args: func(string) []string { return nil }, AddressRole: "AXTextField", AddressLabel: "Address and search bar"} - result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}) + result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}, false) if err != nil { t.Fatal(err) } diff --git a/apps/computer/local-daemon/internal/linger/localtools/manager.go b/apps/computer/local-daemon/internal/linger/localtools/manager.go index f5c07930..201295ca 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/manager.go +++ b/apps/computer/local-daemon/internal/linger/localtools/manager.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "maps" "os" "os/exec" @@ -90,11 +91,13 @@ type Manager struct { agentChromeOwner *operation agentChromePID int agentChromeChecked time.Time - // agentSafariPID and agentSafariWindow are the user's Safari and the - // window in it that the Computer opened as its own; see - // launchAgentSafari. - agentSafariPID int - agentSafariWindow float64 + // sharedBrowserBundle, sharedBrowserPID and sharedBrowserWindow are the + // user's own browser the Computer works in, such as their everyday Chrome + // or Safari, and the window in it that the Computer opened as its own; + // see launchEverydayChrome and launchAgentSafari. + sharedBrowserBundle string + sharedBrowserPID int + sharedBrowserWindow float64 // paceMu guards the persisted pace record of the Computer's own browser // and paceLast, when that Chrome last received input; see // paceAgentChrome. @@ -121,10 +124,29 @@ func New(options Options) (*Manager, error) { if err := m.openBrowserJournals(); err != nil { return nil, err } + retireAgentChromeProfile(options.StateDirectory) } return m, nil } +// retireAgentChromeProfile removes the separate Chrome profile the Computer +// used before it worked in a window of the user's everyday Chrome. A profile +// whose Chrome still runs stays until a later start finds it closed. +func retireAgentChromeProfile(state string) { + directory := filepath.Join(state, "computer-tools", "agent-chrome") + if _, err := os.Stat(directory); err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if pid, _, err := agentChromePID(ctx, directory); err != nil || pid > 0 { + return + } + if err := os.RemoveAll(directory); err != nil { + slog.Warn("agent_browser.retire_profile_failed", "error", err) + } +} + type operation struct { manager *Manager spec provider.OperationSpec @@ -692,7 +714,7 @@ func describeBrowserMode(tools []tool, userChrome, chromeRequired, connectable b // siteSkillNote leaves the choice of tools on a site to a Skill written for // it: sites that block automated browsers are used through the computer // tools, and this browser leaves automation traces in their pages. -const siteSkillNote = " When a Skill covers the site, follow its choice of tools, even when it asks for the computer tools instead of these. A social network, a marketplace or another site that watches for automated browsers is used through the computer tools in the Computer's own browser, where the user signs in once, even when the user says they are signed in: never connect their everyday browser for such a site." +const siteSkillNote = " When a Skill covers the site, follow its choice of tools, even when it asks for the computer tools instead of these. A social network, a marketplace or another site that watches for automated browsers is used through the computer tools, in the Computer's own window of the user's everyday browser, never through these browser tools: they drive the page in a way such sites detect." func filterBrowserTools(catalog []tool) []tool { var result []tool diff --git a/skills/builtin/catalog.yaml b/skills/builtin/catalog.yaml index dbacfb15..d1a4f3ef 100644 --- a/skills/builtin/catalog.yaml +++ b/skills/builtin/catalog.yaml @@ -219,11 +219,14 @@ skills: version: 1.0.0 package_path: builtin/personal-profile/1.0.0 - skill_id: simplexai_humanlike_browse - bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_4 + bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_5 default_agent_types: - anotherme - general manifests: + - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_5 + version: 1.0.5 + package_path: builtin/humanlike-browse/1.0.5 - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_4 version: 1.0.4 package_path: builtin/humanlike-browse/1.0.4 diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md new file mode 100644 index 00000000..8c0b3f7a --- /dev/null +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -0,0 +1,80 @@ +--- +name: humanlike-browse +description: Read, survey, post to or manage a site that watches for automated browsing, such as a social network, a marketplace or any site where the user is signed in, through the computer tools in the Computer's own window of the user's browser. Use whenever a task reads posts, comments, profiles or search results on such a site, or writes there for the user. +--- + +# Sites that watch for automated browsing + +## Choose the route + +- A public page that needs no sign-in and no clicks: read it with the web search or fetch tools. +- Anything else on a social network, a marketplace or a site where the user is signed in: only the computer tools, through `launch_app({agent_browser: true, urls: [url]})`. In Chrome or Safari this opens a window of the Computer's own in the user's everyday browser, with the user's sign-ins; Edge and Firefox run as separate instances. Pass `bundle_id` only when the user asks for one browser. The browser tools drive a page in a way these sites detect, so never use them on these sites; the Computer keeps them off every site the task opened this way. +- If `launch_app` has no `agent_browser`, this needs a Mac Computer: tell the user and stop. +- If a call says a computer tool does not exist, take the tool's exact name from your tool list and call it again before telling the user the tools are unavailable. +- If `launch_app` answers `COMPUTER_BROWSER_RESTART_REQUIRED`, ask the user with a question card whether the Computer may restart Chrome, saying that their windows and tabs come back. Only if they agree, call `launch_app` again with `restart_browser: true`; otherwise end the task. +- If the site asks to sign in, ask the user to sign in once in that browser window and stop; the sign-in stays for later tasks. + +## Move like a person + +These sites watch how a visitor moves, and the user may watch the window. Every page change looks like a person's. + +- Open each site once with `launch_app`, at its home page, such as `https://www.reddit.com/`; in Chrome, one site per call. A page the user names or describes, such as a forum, a category, a search or their order list, is reached from the home page through the site's own controls. Never call `launch_app` again for another page of a site already open. +- After that, reach each page the way a person does: type into the site's own search box, use its menus, filters, sort controls and page links, and click the item you want. +- Never type or paste an address into the browser's address bar, and never build a search, filter or page address yourself, in a tool call or in a script. +- If the page the user wants cannot be reached from the home page, such as a private share link, tell the user and stop. +- Go back through the page's own links, such as the site's logo or its breadcrumb. If the item opened in a new tab, close that tab with its Close button. In Edge or Firefox the `cmd+[` hotkey to the window also goes back. +- To see more items in a list that grows as it scrolls, scroll the list with `scroll`. Use the site's next-page link for a paged list. + +## Keep it quick + +The user watches the window, and each turn of yours costs seconds. The Computer already spaces each input like a quick person. Do not add waits of your own. + +- Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. +- Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. +- Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Edge and Firefox. +- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; Return bound to the search box goes only while no other Chrome window could take it. +- After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. +- Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. + +A visit to a result, in one script: + +```js +const app = await cua.getApp({pid, windowId}); +await app.setValue({role: "AXTextField", name: "Search"}, "camping kettle"); +await app.click({role: "AXButton", name: "Search"}); +await app.waitFor({role: "AXLink", name: "Next", timeout_ms: 8000}); +const links = await app.query({role: "AXLink"}); +const first = links.find((l) => /kettle/i.test(l.label)); +await app.click({role: "AXLink", name: first.label}); +await app.waitFor({text: "out of 5 stars", timeout_ms: 8000}); +const page = await app.getState({query: "out of 5 stars"}); +await app.click({role: "AXLink", name: "Back to results"}); +return {title: first.label, rating: page.tree_markdown}; +``` + +## Read + +- Read with `get_window_state`. It returns the whole page the browser has loaded, including the part below the window, so text already on the page needs no scrolling. Pass `query` to narrow a long page. If a read says it was cut, read again narrower instead of guessing. +- Text inside pictures: `recognize_text`; in a picture viewer, `pages` with `next_key`. +- A state the tree does not show, such as liked or followed: look with `include_screenshot: true`. +- To repeat a flow over items, use `run_script`. A script there sends at most 20 inputs: return what it extracted, look at the window, then continue. Many sites open a result in a new tab; the script then reads that tab, so read the tab in front before the next step. +- Stop scrolling when two reads add no new items. Read only what the task needs: one search per keyword, and never reopen an item already read. +- Act only in the tabs this task opened. A click that opens a new tab is the task's tab. Close each item's tab with its tab's Close button after reading it, and the task's last tab when done. If a tab the task did not open is in front, stop and tell the user. + +## Write + +Write only when the user asked for that action in this task: publish, comment, like, follow or delete. Text to review or a request to draft is not permission. + +1. Check on the page that the signed-in account is the one the user meant. If not, stop and tell the user. +2. Make sure exactly one item matches the target. If several do, ask. +3. Send once, then read the page again and confirm the result shows, such as the comment under that account. +4. If the result is unclear, read the page. Never send again blind. + +If the user takes over the window, read the page and the account again before the next input. + +## Checks and limits + +- When a page shows a captcha, a code or QR check, a sign-in wall or a too-many-requests message, call `report_blocker` with that kind and the page's URL at once. Then tell the user in the Room what the page asks for and that they can deal with it in that browser window, report what was done, and end the task. +- When only one item cannot be viewed, call `report_blocker` with `content_unavailable` and its URL, and skip it. +- Never pass a check yourself, switch accounts or retry around a limit. +- When the Computer refuses input with `COMPUTER_INPUT_BUDGET`, `COMPUTER_SITE_COOLDOWN`, `COMPUTER_SITE_BLOCKED` or `COMPUTER_TAB_CHANGED`, report what was done and end the task. From 9e59cde570c8cee98170084b4f6f257de9d8395f Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 00:22:36 +0800 Subject: [PATCH 02/30] fix(computer): open the Computer's Chrome window in the profile Chrome last used A user with several Chrome profiles may get a profile picker instead of a window when Chrome is started without a profile. Co-Authored-By: Claude Code --- .../linger/localtools/agent_browser.go | 28 ++++++++++++++++++- .../linger/localtools/localtools_test.go | 5 +++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index fb19efe7..37bc9ec0 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -400,7 +400,11 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u for _, w := range windows { before[w.WindowID] = true } - if output, err := agentChromeOpen(ctx, []string{"-n", "-g", "-b", b.Bundle, "--args", "--new-window", urls[0]}); err != nil { + // Naming the profile Chrome last used keeps a profile picker, which + // Chrome may show for a user with several profiles, from taking the + // command instead of a window. + open := []string{"-n", "-g", "-b", b.Bundle, "--args", "--profile-directory=" + everydayChromeProfile(), "--new-window", urls[0]} + if output, err := agentChromeOpen(ctx, open); err != nil { return nil, fmt.Errorf("launch_app: %s: %w", strings.TrimSpace(string(output)), err) } var window listedWindow @@ -501,3 +505,25 @@ func everydayChromeKeys(name string, arguments map[string]any) error { } return nil } + +// everydayChromeData is the user's Chrome data folder, whose Local State +// names the profile Chrome last used. +var everydayChromeData = func() string { + home, _ := os.UserHomeDir() + return filepath.Join(home, "Library", "Application Support", "Google", "Chrome") +} + +// everydayChromeProfile returns the folder name of the profile the user's +// Chrome last used, or Default when Local State does not say. +func everydayChromeProfile() string { + var state struct { + Profile struct { + LastUsed string `json:"last_used"` + } `json:"profile"` + } + data, err := os.ReadFile(filepath.Join(everydayChromeData(), "Local State")) + if err == nil && json.Unmarshal(data, &state) == nil && chromeProfilePattern.MatchString(state.Profile.LastUsed) { + return state.Profile.LastUsed + } + return "Default" +} diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index f54e4c6b..85c38687 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1896,6 +1896,9 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t return nil, err } t.Cleanup(func() { agentChromeOpen = open }) + data := everydayChromeData + everydayChromeData = func() string { return t.TempDir() } + t.Cleanup(func() { everydayChromeData = data }) settle := everydayChromeSettle everydayChromeSettle = 0 t.Cleanup(func() { everydayChromeSettle = settle }) @@ -1908,7 +1911,7 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t if err != nil { t.Fatal(err) } - if stopped != 1 || len(opened) != 2 || !slices.Contains(opened[0], "--restore-last-session") || !slices.Contains(opened[0], "--disable-backgrounding-occluded-windows") || !slices.Equal(opened[1][len(opened[1])-2:], []string{"--new-window", "https://example.com/"}) { + if stopped != 1 || len(opened) != 2 || !slices.Contains(opened[0], "--restore-last-session") || !slices.Contains(opened[0], "--disable-backgrounding-occluded-windows") || !slices.Equal(opened[1][len(opened[1])-3:], []string{"--profile-directory=Default", "--new-window", "https://example.com/"}) { t.Fatalf("stopped=%d opened=%v, want a restart, then the site in a new window", stopped, opened) } if !strings.Contains(string(result), `"window_id":12`) || !strings.Contains(string(result), "takes no keys") { From 67f9070be61ccd54b889e9a700a308efb1c35fa3 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 10:35:20 +0800 Subject: [PATCH 03/30] fix(computer): leave quitting the user's Chrome to the user and refuse its keys in every window The Daemon no longer quits the user's everyday Chrome on a model-supplied flag: it asks the Agent to have the user quit Chrome, and starts it with the occlusion flag and the user's windows once it is closed. Keys are refused by the target pid, so no window of the user's Chrome takes one, in tool calls, paged reads and scripts. The Go and script key rules now agree, the Computer's window is the new one-tab window that shows the site just opened, and Chrome is found wherever it is installed. Co-Authored-By: Claude Code --- apps/computer/local-daemon/README.md | 20 +- .../linger/localtools/agent_browser.go | 190 +++++++++++------- .../linger/localtools/agent_browser_test.go | 41 ++++ .../linger/localtools/agent_chrome.go | 4 +- .../internal/linger/localtools/cua.go | 40 +++- .../internal/linger/localtools/cua_script.go | 2 +- .../localtools/cua_script_guard_test.go | 7 +- .../linger/localtools/localtools_test.go | 103 +++++----- .../internal/linger/localtools/manager.go | 3 + .../builtin/humanlike-browse/1.0.5/SKILL.md | 2 +- 10 files changed, 266 insertions(+), 146 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 5e3fd806..db5ac5d5 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -331,13 +331,19 @@ Chrome's command line, one site per call, so no key reaches the user's other windows. A covered window keeps working only when Chrome runs with `--disable-backgrounding-occluded-windows`: a Chrome that is not running starts with it, and one running without it returns -`COMPUTER_BROWSER_RESTART_REQUIRED` until the Agent passes `restart_browser` -after the user agreed. The restart quits Chrome as its menu does and reopens it -with `--restore-last-session`. The user's Chrome takes no keys from the -Computer (`COMPUTER_KEYS_REFUSED`): a shortcut runs as a menu command in the -Chrome window that has the keyboard, and Cua types text as keys that all read -as the A key. Only a plain key bound to an element goes, and Cua sends it only -when no other Chrome window could take it. The Agent fills fields with +`COMPUTER_BROWSER_RESTART_REQUIRED`. The Daemon never quits the user's Chrome, +because a quit loses text typed into a page and not yet sent: the Agent asks +the user to quit Chrome, and the next `launch_app` starts it with the flag and +`--restore-last-session`. The Computer's window is the new window with one tab +whose address bar shows the site just opened, so a window the user opens or +Chrome restores meanwhile is never taken for it. The Daemon finds the running +Chrome by its executable wherever the app is installed. The user's Chrome +takes no keys from the Computer (`COMPUTER_KEYS_REFUSED`), in any of its +windows, in tool calls, `run_actions` steps, paged `recognize_text` reads and +scripts that target it: a shortcut runs as a menu command in the Chrome window +that has the keyboard, and Cua types text as keys that all read as the A key. +Only a plain key bound to an element of the Computer's window goes, and Cua +sends it only when no other Chrome window could take it. The Agent fills fields with `set_value`, presses the page's buttons and goes back through its links. The separate Chrome profile earlier versions used, `computer-tools/agent-chrome`, is removed when the Daemon starts and that Chrome no longer runs. diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index 37bc9ec0..a217c74f 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -9,10 +9,10 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" "strconv" "strings" - "syscall" "time" "github.com/simplexai-labs/linger/apps/computer/local-daemon/internal/platform/privatefs" @@ -290,9 +290,10 @@ func (c *cuaSession) waitSafariWindows(ctx context.Context, b agentBrowser, pid } // everydayChromePID finds the user's everyday Chrome: the main process of -// Google Chrome on its default profile. Chrome instances on a profile of their -// own, such as a test browser, name --user-data-dir and are not it. It returns -// the pid and command line, or 0 when that Chrome is not running. +// Google Chrome on its default profile, wherever the app is installed. Chrome +// instances on a profile of their own, such as a test browser, name +// --user-data-dir and are not it. It returns the pid and command line, or 0 +// when that Chrome is not running. var everydayChromePID = func(ctx context.Context) (int, string, error) { output, err := exec.CommandContext(ctx, "/bin/ps", "-axww", "-o", "pid=,args=").Output() if err != nil { @@ -300,7 +301,7 @@ var everydayChromePID = func(ctx context.Context) (int, string, error) { } for _, line := range strings.Split(string(output), "\n") { pidText, args, _ := strings.Cut(strings.TrimSpace(line), " ") - if !strings.HasPrefix(args, everydayChromeExecutable) || strings.Contains(args, "--type=") || strings.Contains(args, "--user-data-dir=") { + if !everydayChromeCommand(args) { continue } if pid, err := strconv.Atoi(pidText); err == nil { @@ -310,34 +311,19 @@ var everydayChromePID = func(ctx context.Context) (int, string, error) { return 0, "", nil } -const everydayChromeExecutable = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" - -// stopEverydayChrome asks the user's Chrome to quit, as quitting from its -// menu does, and waits until it is gone. Chrome saves its session first, so -// the restart brings back the user's windows and tabs. -var stopEverydayChrome = func(ctx context.Context, pid int) error { - process, err := os.FindProcess(pid) - if err != nil { - return err - } - if err := process.Signal(syscall.SIGTERM); err != nil { - return err - } - for deadline := time.Now().Add(everydayChromeQuitWait); ; { - if running, _, err := everydayChromePID(ctx); err != nil || running == 0 { - return err - } - if time.Now().After(deadline) { - return errors.New("COMPUTER_BROWSER_RESTART_FAILED: the user's Chrome did not quit, so it was not restarted; ask the user to quit Chrome and try again") - } - if err := sleepContext(ctx, 200*time.Millisecond); err != nil { - return err - } - } +// everydayChromeCommand reports whether args starts Google Chrome's own +// executable on the default profile. The executable path has spaces, so it +// ends where the first argument starts. +func everydayChromeCommand(args string) bool { + index := strings.Index(args, everydayChromeExecutable) + if !strings.HasPrefix(args, "/") || index < 0 { + return false + } + rest := args[index+len(everydayChromeExecutable):] + return (rest == "" || rest[0] == ' ') && !strings.Contains(args, "--type=") && !strings.Contains(args, "--user-data-dir=") } -// everydayChromeQuitWait bounds how long the user's Chrome may take to quit. -var everydayChromeQuitWait = 20 * time.Second +const everydayChromeExecutable = "/Google Chrome.app/Contents/MacOS/Google Chrome" // sharedBrowserRunning returns the pid of the user's own browser named by // bundle, or 0. @@ -349,40 +335,45 @@ func sharedBrowserRunning(ctx context.Context, bundle string) (int, error) { return lsappPID(ctx, bundle) } -// everydayChromeRestart asks for the user's consent before the Computer -// restarts their Chrome. -const everydayChromeRestart = "COMPUTER_BROWSER_RESTART_REQUIRED: the user's Chrome runs without the setting that keeps the Computer's window working while other windows cover it, and Chrome takes it only when it starts; nothing was opened. Ask the user with a question card whether the Computer may restart Chrome now: Chrome quits and reopens with their windows and tabs, but text typed into a page and not yet sent may be lost. If the user agrees, call launch_app again with restart_browser: true; if not, end the task" +// everydayChromeQuit asks the user to quit their Chrome. Only the user +// quits it: a quit loses text typed into a page and not yet sent, and no +// tool call can prove the user agreed to that. +const everydayChromeQuit = "COMPUTER_BROWSER_RESTART_REQUIRED: the user's Chrome runs without the setting that keeps the Computer's window working while other windows cover it, and Chrome takes it only when it starts; nothing was opened. Ask the user with a question card to quit Chrome from its menu (Google Chrome > Quit Google Chrome) and leave it closed, then end your turn. When they answer that Chrome is closed, call launch_app again: the Computer starts Chrome with that setting and brings back their windows and tabs. If they decline, end the task" // launchEverydayChrome opens url in a new window of the user's everyday // Chrome, the Computer's own window there, which the task then works in. The // window opens through Chrome's command line, as a link from another app // does, so no key reaches the user's other Chrome windows. A Chrome that is // not running starts with the setting the window needs; one running without -// it restarts only when restart says the user agreed. -func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, urls []string, restart bool) (json.RawMessage, error) { +// it is never quit by the Computer: the user quits it. +func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, urls []string) (json.RawMessage, error) { if len(urls) != 1 { return nil, fmt.Errorf("launch_app opens one site at a time in the user's Chrome; call it once for each site: %w", errNotSent) } + site := siteOf(urls[0]) + if site == "" { + return nil, fmt.Errorf("launch_app opens a web address in the user's Chrome: %w", errNotSent) + } + m := c.op.manager pid, command, err := everydayChromePID(ctx) if err != nil { return nil, err } - restarted := false if pid > 0 && !b.startedWith(command) { - if !restart { - return nil, fmt.Errorf("%s: %w", everydayChromeRestart, errNotSent) - } - if err := stopEverydayChrome(ctx, pid); err != nil { - return nil, err - } - pid, restarted = 0, true + m.mu.Lock() + m.everydayChromeQuitAsked = true + m.mu.Unlock() + return nil, fmt.Errorf("%s: %w", everydayChromeQuit, errNotSent) } + restored := false if pid == 0 { - // Chrome starts with the setting first, bringing back the user's - // windows after a restart; the Computer's window opens after them, - // so it is the one window that was not there before. + // After the user quit Chrome for the Computer, it reopens their + // windows and tabs, as the question to quit promised. + m.mu.Lock() + restored, m.everydayChromeQuitAsked = m.everydayChromeQuitAsked, false + m.mu.Unlock() args := append([]string{"-g", "-b", b.Bundle, "--args"}, b.Settings...) - if restarted { + if restored { args = append(args, "--restore-last-session") } if output, err := agentChromeOpen(ctx, args); err != nil { @@ -407,6 +398,9 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u if output, err := agentChromeOpen(ctx, open); err != nil { return nil, fmt.Errorf("launch_app: %s: %w", strings.TrimSpace(string(output)), err) } + // The Computer's window is a new window with one tab whose address bar + // shows the site just opened. A window the user opens meanwhile, or one + // Chrome restores late, fails that test, so it is never taken for it. var window listedWindow tabs := 0 for deadline := time.Now().Add(agentSafariWindowWait); window.WindowID == 0; { @@ -415,11 +409,12 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u return nil, err } for _, w := range windows { - if !before[w.WindowID] { - if n := c.agentChromeTabs(ctx, pid, w.WindowID); n > 0 { - window, tabs = w, n - break - } + if before[w.WindowID] || siteOf(c.windowAddress(ctx, b, pid, w.WindowID)) != site { + continue + } + if n := c.agentChromeTabs(ctx, pid, w.WindowID); n == 1 { + window, tabs = w, n + break } } if window.WindowID != 0 { @@ -432,7 +427,6 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u return nil, err } } - m := c.op.manager m.mu.Lock() m.sharedBrowserBundle, m.sharedBrowserPID, m.sharedBrowserWindow = b.Bundle, pid, window.WindowID m.mu.Unlock() @@ -440,16 +434,48 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = tabs, pid, window.WindowID c.op.mu.Unlock() note := "This is the Computer's own window in the user's everyday Chrome, with the user's sign-ins; work only in this window, never in the user's other Chrome windows. It works while other windows cover it, but only on the desktop (Space) the user is on. The user's Chrome takes no keys from the Computer, because a key could reach the window the user is typing in: fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links. Close a tab with the Close button of its tab; closing the window's last tab closes the window. If a site asks to sign in, ask the user to sign in once in this window." - if restarted { - note = "Restarted the user's Chrome with their windows and tabs. " + note + if restored { + note = "Started the user's Chrome again with their windows and tabs. " + note } text := fmt.Sprintf("Opened %s in a new window of the user's Chrome (pid %d), window %q [window_id: %v]. The window has %d tab(s).\n%s", urls[0], pid, window.Title, window.WindowID, tabs, note) return json.Marshal(map[string]any{"content": []any{map[string]string{"type": "text", "text": text}}, "structuredContent": map[string]any{"browser": b.Name, "pid": pid, "windows": []map[string]any{{"window_id": window.WindowID, "title": window.Title, "on_screen": window.IsOnScreen}}, "tabs": tabs, "note": note}}) } +// windowAddress returns the web address the address bar of window shows, +// with https:// added when the bar leaves out the scheme, or "" when the +// bar cannot be read. +func (c *cuaSession) windowAddress(ctx context.Context, b agentBrowser, pid int, window float64) string { + read, err := c.client.request(ctx, "tools/call", map[string]any{"name": "get_window_state", "arguments": map[string]any{ + "pid": pid, "window_id": window, "tree_format": "elements", "include_screenshot": false, "max_elements": 3000, "timeout_ms": 4000, "session": c.label}}) + if err != nil { + return "" + } + var state struct { + Structured struct { + Elements []struct { + Role string `json:"role"` + Label string `json:"label"` + Value any `json:"value"` + } `json:"elements"` + } `json:"structuredContent"` + } + _ = json.Unmarshal(read, &state) + for _, element := range state.Structured.Elements { + value, ok := element.Value.(string) + if !ok || element.Role != b.AddressRole || element.Label != b.AddressLabel { + continue + } + if !strings.Contains(value, "://") { + value = "https://" + value + } + return value + } + return "" +} + // waitEverydayChrome waits until the user's Chrome that just started runs -// and shows its windows, and returns its pid. +// and shows a window, and returns its pid. func (c *cuaSession) waitEverydayChrome(ctx context.Context) (int, error) { for deadline := time.Now().Add(agentSafariWindowWait); ; { pid, _, err := everydayChromePID(ctx) @@ -460,9 +486,7 @@ func (c *cuaSession) waitEverydayChrome(ctx context.Context) (int, error) { if windows, err := c.appWindows(ctx, pid); err != nil { return 0, err } else if len(windows) > 0 { - // Restored windows appear one after another; a short settle - // keeps a late one from reading as the Computer's. - return pid, sleepContext(ctx, everydayChromeSettle) + return pid, nil } } if time.Now().After(deadline) { @@ -474,18 +498,28 @@ func (c *cuaSession) waitEverydayChrome(ctx context.Context) (int, error) { } } -// everydayChromeSettle is how long a just-started Chrome gets to reopen the -// user's windows. -var everydayChromeSettle = 2 * time.Second - -// everydayChrome reports whether pid is the user's everyday Chrome that the -// Computer opened its window in. -func (c *cuaSession) everydayChrome(pid any) bool { +// userChrome reports whether pid is the user's everyday Chrome, whichever of +// its windows an input names: a key reaches the Chrome window that has the +// keyboard, not the one named. The pid the Computer opened its window in is +// known; any other pid is looked up, so the rule holds before the Computer +// opened a window too. +func (c *cuaSession) userChrome(ctx context.Context, pid any) (bool, error) { number, ok := pid.(float64) + if !ok || runtime.GOOS != "darwin" { + return false, nil + } m := c.op.manager m.mu.Lock() - defer m.mu.Unlock() - return ok && m.sharedBrowserPID > 0 && int(number) == m.sharedBrowserPID && m.sharedBrowserBundle == "com.google.Chrome" + shared := m.sharedBrowserBundle == "com.google.Chrome" && m.sharedBrowserPID > 0 && int(number) == m.sharedBrowserPID + m.mu.Unlock() + if shared { + return true, nil + } + running, _, err := everydayChromePID(ctx) + if err != nil { + return false, fmt.Errorf("COMPUTER_INPUT_NOT_SENT: could not tell whether the input goes to the user's Chrome: %w: %w", err, errNotSent) + } + return running > 0 && running == int(number), nil } // everydayChromeKeysRefused tells the Agent how to work without keys in the @@ -496,14 +530,20 @@ const everydayChromeKeysRefused = "COMPUTER_KEYS_REFUSED: the user's everyday Ch // plain key bound to an element, such as Return in a search box, may go: // Cua delivers it only when no other Chrome window could receive it. // Shortcuts are menu commands, which Chrome runs in the window that has the -// keyboard, and typed text arrives as keys that all read as the A key. -func everydayChromeKeys(name string, arguments map[string]any) error { - modifiers, _ := arguments["modifiers"].([]any) +// keyboard, and typed text arrives as keys that all read as the A key. own +// says the key names the Computer's window; one of the user's windows takes +// none. The script guard (keyed in guardedScript) applies the same rule. +func everydayChromeKeys(name string, arguments map[string]any, own bool) error { + if name != "hotkey" && name != "type_text" && name != "press_key" { + return nil + } _, bound := arguments["element_token"].(string) - if name == "hotkey" || name == "type_text" || name == "press_key" && (!bound || len(modifiers) > 0) { - return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) + _, modified := arguments["modifiers"] + key, plain := arguments["key"].(string) + if own && name == "press_key" && bound && !modified && plain && !strings.Contains(key, "+") { + return nil } - return nil + return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) } // everydayChromeData is the user's Chrome data folder, whose Local State diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go index fccf9d30..abcbb537 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go @@ -46,3 +46,44 @@ func TestPickAgentBrowserTakesTheFirstInstalledOrTheNamedOne(t *testing.T) { t.Fatal("agentBrowserApp misread an app") } } + +func TestEverydayChromeCommandFindsChromeInAnyFolder(t *testing.T) { + for args, want := range map[string]bool{ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome": true, + "/Users/a/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --disable-backgrounding-occluded-windows": true, + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test": false, + "/Applications/Google Chrome.app/Contents/Frameworks/Google Chrome Framework.framework/Helpers/Google Chrome Helper.app/Contents/MacOS/Google Chrome Helper --type=renderer": false, + "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary": false, + "tail -f /Applications/Google Chrome.app/Contents/MacOS/Google Chrome": false, + } { + if got := everydayChromeCommand(args); got != want { + t.Errorf("%q: %v, want %v", args, got, want) + } + } +} + +func TestEverydayChromeKeysAllowOnlyOnePlainBoundKey(t *testing.T) { + for _, tc := range []struct { + name string + arguments map[string]any + allowed bool + }{ + {"press_key", map[string]any{"key": "return", "element_token": "s1:1"}, true}, + {"set_value", map[string]any{"value": "kettle", "element_token": "s1:1"}, true}, + {"press_key", map[string]any{"key": "return"}, false}, + {"press_key", map[string]any{"key": "return", "element_token": "s1:1", "modifiers": []any{}}, false}, + {"press_key", map[string]any{"key": "return", "element_token": "s1:1", "modifiers": "cmd"}, false}, + {"press_key", map[string]any{"key": "cmd+r", "element_token": "s1:1"}, false}, + {"press_key", map[string]any{"key": 13.0, "element_token": "s1:1"}, false}, + {"hotkey", map[string]any{"keys": []any{"cmd", "["}}, false}, + {"type_text", map[string]any{"text": "kettle", "element_token": "s1:1"}, false}, + } { + if err := everydayChromeKeys(tc.name, tc.arguments, true); (err == nil) != tc.allowed { + t.Errorf("%s %v: err=%v, want allowed=%v", tc.name, tc.arguments, err, tc.allowed) + } + } + // One of the user's windows takes no key at all. + if err := everydayChromeKeys("press_key", map[string]any{"key": "return", "element_token": "s1:1"}, false); err == nil { + t.Error("a bound Return went to one of the user's windows") + } +} diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index feff01de..9ffb6d61 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -58,7 +58,7 @@ var agentChromePID = func(ctx context.Context, directory string) (int, string, e // brings its first window forward once, where the user is. Later URLs open // in a new tab through the window's accessibility tree and one Return key to // that browser alone, so the user's apps, focus and Space stay as they are. -func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls []string, restart bool) (json.RawMessage, error) { +func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls []string) (json.RawMessage, error) { var sites []string for _, address := range urls { if site := siteOf(address); site != "" { @@ -79,7 +79,7 @@ func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls return nil, err } if b.Everyday { - return c.launchEverydayChrome(ctx, b, urls, restart) + return c.launchEverydayChrome(ctx, b, urls) } if b.Shared { return c.launchAgentSafari(ctx, b, urls) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 2533e485..324df202 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -160,8 +160,7 @@ func (m *Manager) openCua(ctx context.Context, op *operation) (*cuaSession, []to if item.Name == "launch_app" && runtime.GOOS == "darwin" { item.Description = "Open a macOS app in the background through Launch Services, as `open -g` does, and return its pid and windows. Optional urls are handed to the app; for Finder, pass a folder to open it in a window." item.InputSchema = schema(map[string]any{"bundle_id": map[string]string{"type": "string", "description": "Preferred, for example com.apple.TextEdit."}, "name": map[string]string{"type": "string", "description": "App name, when the bundle id is unknown."}, "urls": map[string]any{"type": "array", "items": map[string]string{"type": "string"}, "description": "Files or folders to open in the app."}, - "agent_browser": map[string]string{"type": "boolean", "description": "Open the URLs in the Computer's own browser window, which keeps working while other windows cover it on the user's desktop. Only a site's home page opens there, such as https://www.example.com/. With no bundle_id it is the first installed of Chrome, Edge, Firefox and Safari; a bundle_id of com.google.Chrome, com.microsoft.edgemac, org.mozilla.firefox or com.apple.Safari picks one. In Chrome and Safari it is a window of the Computer's own in the user's everyday browser, with the user's sign-ins; in Chrome it opens one site per call. Edge and Firefox run as separate instances with their own sign-ins."}, - "restart_browser": map[string]string{"type": "boolean", "description": "With agent_browser in Chrome: restart the user's Chrome so the Computer's window keeps working while covered. Pass it only after the user agreed to a COMPUTER_BROWSER_RESTART_REQUIRED question."}}) + "agent_browser": map[string]string{"type": "boolean", "description": "Open the URLs in the Computer's own browser window, which keeps working while other windows cover it on the user's desktop. Only a site's home page opens there, such as https://www.example.com/. With no bundle_id it is the first installed of Chrome, Edge, Firefox and Safari; a bundle_id of com.google.Chrome, com.microsoft.edgemac, org.mozilla.firefox or com.apple.Safari picks one. In Chrome and Safari it is a window of the Computer's own in the user's everyday browser, with the user's sign-ins; in Chrome it opens one site per call. Edge and Firefox run as separate instances with their own sign-ins."}}) } if item.Name == "get_desktop_state" { item.Description += " Read-only whole-screen view; act through the window tools with a pid and window_id. A system privacy prompt about CuaDriver belongs to the user: never answer it, and keep working through the window tools." @@ -396,6 +395,26 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage } } } + // A key to the user's Chrome reaches the window that has the keyboard, + // whichever window it names, so it is refused before anything is taken. + if name == "hotkey" || name == "type_text" || name == "press_key" || paged { + user, err := c.userChrome(ctx, arguments["pid"]) + if err != nil { + return nil, err + } + if user && paged { + return nil, fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) + } + if user { + own, err := c.isAgentChrome(ctx, arguments["pid"], arguments["window_id"]) + if err != nil { + return nil, err + } + if err := everydayChromeKeys(name, arguments, own); err != nil { + return nil, err + } + } + } agentChrome, err := c.claimAgentChromeInput(ctx, name, arguments, paged) if err != nil { return nil, err @@ -439,14 +458,14 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage return nil, err } } - everyday := paced && c.everydayChrome(target["pid"]) - if everyday && name != "run_script" { - if err := everydayChromeKeys(name, arguments); err != nil { - return nil, err - } - } if name == "run_script" { - if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, everyday); err != nil { + noKeys := false + if scriptTarget != nil { + if noKeys, err = c.userChrome(ctx, scriptTarget["pid"]); err != nil { + return nil, err + } + } + if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, noKeys); err != nil { return nil, err } } @@ -1958,8 +1977,7 @@ func (c *cuaSession) launchApp(ctx context.Context, arguments map[string]any) (j if err != nil { return nil, err } - restart, _ := arguments["restart_browser"].(bool) - return c.launchAgentChrome(ctx, b, addresses, restart) + return c.launchAgentChrome(ctx, b, addresses) } app, flag := bundle, "-b" if app == "" { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index e32a1ac1..ab020ee9 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -81,7 +81,7 @@ const apply = Reflect.apply, stringify = JSON.stringify, parse = JSON.parse, tes const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis.cua; const bad = /`+strings.Join(pattern, "|")+`/; const limit = `+strconv.Itoa(inputLimit)+`, noKeys = `+strconv.FormatBool(noKeys)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; -const keyed = (k, c) => k === "hotkey" || k === "typeText" || k === "pressKey" && (c.length < 2 || c[1] === null || c[1] === undefined || typeof c[0] !== "string" || c[0].indexOf("+") >= 0) || k === "call" && (c[0] === "hotkey" || c[0] === "type_text" || c[0] === "press_key" && (c[1] === null || typeof c[1] !== "object" || typeof c[1].element_token !== "string" || c[1].modifiers !== undefined)); +const keyed = (k, c) => k === "hotkey" || k === "typeText" || k === "pressKey" && (c.length !== 2 || c[1] === null || c[1] === undefined || typeof c[0] !== "string" || c[0].indexOf("+") >= 0) || k === "call" && (c[0] === "hotkey" || c[0] === "type_text" || c[0] === "press_key" && (c[1] === null || typeof c[1] !== "object" || typeof c[1].element_token !== "string" || c[1].modifiers !== undefined || typeof c[1].key !== "string" || c[1].key.indexOf("+") >= 0)); const inputMethods = {__proto__: null, click: true, doubleClick: true, rightClick: true, drag: true, scroll: true, typeText: true, pressKey: true, hotkey: true, setValue: true}; const inputOps = {__proto__: null, click: true, double_click: true, right_click: true, drag: true, scroll: true, type_text: true, press_key: true, hotkey: true, set_value: true}; const input = (k, c) => inputMethods[k] === true || k === "call" && typeof c[0] === "string" && inputOps[c[0]] === true; diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 352d385f..25552cb1 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -14,7 +14,7 @@ import ( // the same prelude's semantics. const guardHarness = `import fs from "fs"; const log = []; -class App { constructor(n) { this.n = n; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } } +class App { constructor(n) { this.n = n; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } } const fake = { call(t, a) { log.push("call:" + t + ":" + JSON.stringify(a)); return Promise.resolve({ok: true}); }, getApp() { return Promise.resolve(new App("a")); }, listApps() { return Promise.resolve([new App("b")]); }, @@ -175,6 +175,11 @@ func TestScriptGuardRefusesKeysForTheEverydayChrome(t *testing.T) { {`const app = await cua.getApp({pid: 1}); return app.typeText("kettle", {role: "AXTextField"});`, "threw:COMPUTER_KEYS_REFUSED"}, {`return await cua.call("press_key", {pid: 1, key: "return"});`, "threw:COMPUTER_KEYS_REFUSED"}, {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2", modifiers: ["cmd"]});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2", modifiers: []});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, key: "cmd+r", element_token: "s1:2"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1}); return app.pressKey("cmd+r", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1}); return app.pressKey("return");`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1}); return app.pressKey("return", "s1:2");`, "pressed"}, {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2"});`, "call:press_key"}, {`return await cua.call("click", {pid: 1, element_token: "s1:2"});`, "call:click"}, } { diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 85c38687..b2c51898 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -205,11 +205,12 @@ func runToolFixture(mode string) { } case "list_windows": windows := []any{map[string]any{"pid": 42, "window_id": 9, "title": "b", "z_index": 3, "is_on_screen": true, "bounds": map[string]any{"width": 800, "height": 600}}} - if mode == "tabs-everyday" { - // Each start or command line of the user's Chrome - // shows one more window beside the user's own. - for i := range strings.Count(string(data), `"name":"open"`) { - windows = append(windows, map[string]any{"pid": 42, "window_id": 11 + i, "title": "Home", "z_index": 2, "is_on_screen": true, "bounds": map[string]any{"width": 900, "height": 700}}) + if mode == "tabs-everyday" && strings.Count(string(data), `"name":"open"`) >= 2 { + // After the start and the command line, a window of + // the user's that Chrome restored late lists before + // the Computer's window. + for _, id := range []int{11, 12} { + windows = append(windows, map[string]any{"pid": 42, "window_id": id, "title": "Home", "z_index": 2, "is_on_screen": true, "bounds": map[string]any{"width": 900, "height": 700}}) } } if mode == "tabs-ghost" { @@ -227,8 +228,15 @@ func runToolFixture(mode string) { for i := range tabs { fmt.Fprintf(&tree, "- [%d] AXRadioButton \"Tab %d\"\n- [%d] AXButton \"Close\" [actions=[press]]\n", 2*i, i, 2*i+1) } + // The address bar shows its site without the scheme, as + // Chrome's does: window 12 shows the site just opened. + window, _ := request.Params.Arguments["window_id"].(float64) + address := map[float64]string{12: "example.com"}[window] + if address == "" { + address = "news.example.org/today" + } result = map[string]any{"content": []any{}, "structuredContent": map[string]any{"tree_markdown": tree.String(), "elements": []any{ - map[string]any{"element_token": "s1:1", "role": "AXTextField", "label": "Address and search bar"}, + map[string]any{"element_token": "s1:1", "role": "AXTextField", "label": "Address and search bar", "value": address}, }}} } break @@ -1858,8 +1866,9 @@ func TestLocalToolCuaDeltaCountFailureReadsFullWindowOnlyOnce(t *testing.T) { } } -// A launch opens a fresh window when the listed one has left the -// accessibility tree. +// The user quits their Chrome themselves; the Computer then starts it with +// the setting and opens its own window, found by the site it shows, and the +// user's Chrome takes no keys in any of its windows. func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *testing.T) { if runtime.GOOS != "darwin" { t.Skip("the Computer's window in the user's Chrome is macOS only") @@ -1871,22 +1880,15 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t t.Cleanup(func() { agentChromePause = pause }) marker := filepath.Join(t.TempDir(), "calls.jsonl") c := cuaFixtureSession(t, "tabs-everyday", marker) - command := everydayChromeExecutable + running, command := 42, "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" pid := everydayChromePID - everydayChromePID = func(context.Context) (int, string, error) { return 42, command, nil } + everydayChromePID = func(context.Context) (int, string, error) { return running, command, nil } t.Cleanup(func() { everydayChromePID = pid }) - stopped := 0 - stop := stopEverydayChrome - stopEverydayChrome = func(context.Context, int) error { - stopped++ - command = everydayChromeExecutable + " --disable-backgrounding-occluded-windows" - return nil - } - t.Cleanup(func() { stopEverydayChrome = stop }) var opened [][]string open := agentChromeOpen agentChromeOpen = func(_ context.Context, args []string) ([]byte, error) { opened = append(opened, args) + running, command = 42, "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --disable-backgrounding-occluded-windows" f, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { return nil, err @@ -1899,26 +1901,25 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t data := everydayChromeData everydayChromeData = func() string { return t.TempDir() } t.Cleanup(func() { everydayChromeData = data }) - settle := everydayChromeSettle - everydayChromeSettle = 0 - t.Cleanup(func() { everydayChromeSettle = settle }) b := agentBrowsers[0] - // A Chrome started without the setting restarts only with consent. - if _, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}, false); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_BROWSER_RESTART_REQUIRED") || stopped != 0 || len(opened) != 0 { - t.Fatalf("err=%v stopped=%d opened=%v, want a refused launch", err, stopped, opened) - } - result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}, true) + // A Chrome started without the setting is never quit by the Computer. + if _, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_BROWSER_RESTART_REQUIRED") || len(opened) != 0 { + t.Fatalf("err=%v opened=%v, want a refused launch", err, opened) + } + // The user quits it; the Computer starts it with the setting and their + // windows, then opens the site in a new window. + running = 0 + result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}) if err != nil { t.Fatal(err) } - if stopped != 1 || len(opened) != 2 || !slices.Contains(opened[0], "--restore-last-session") || !slices.Contains(opened[0], "--disable-backgrounding-occluded-windows") || !slices.Equal(opened[1][len(opened[1])-3:], []string{"--profile-directory=Default", "--new-window", "https://example.com/"}) { - t.Fatalf("stopped=%d opened=%v, want a restart, then the site in a new window", stopped, opened) + if len(opened) != 2 || !slices.Contains(opened[0], "--restore-last-session") || !slices.Contains(opened[0], "--disable-backgrounding-occluded-windows") || !slices.Equal(opened[1][len(opened[1])-3:], []string{"--profile-directory=Default", "--new-window", "https://example.com/"}) { + t.Fatalf("opened=%v, want a start with the user's windows, then the site in a new window", opened) } + // Window 11, restored late, lists first, but it shows another site. if !strings.Contains(string(result), `"window_id":12`) || !strings.Contains(string(result), "takes no keys") { - t.Fatalf("result=%s, want the new window 12 after the restored one, and the note", result) + t.Fatalf("result=%s, want the new window 12 that shows the site, and the note", result) } - // The new window is the Computer's: input to it is paced and held, and - // the user's own window 9 is not the Computer's. if own, err := c.isAgentChrome(t.Context(), float64(42), float64(12)); err != nil || !own { t.Fatalf("own=%v err=%v, want window 12 as the Computer's", own, err) } @@ -1927,37 +1928,43 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t t.Fatalf("the user's own Chrome window %v read as the Computer's", user) } } - for _, call := range []string{ - `{"pid":42,"window_id":12,"keys":["cmd","["]}`, + // No key reaches the user's Chrome, in the Computer's window or theirs. + for _, call := range []struct{ name, args string }{ + {"hotkey", `{"pid":42,"window_id":12,"keys":["cmd","["]}`}, + {"hotkey", `{"pid":42,"window_id":9,"keys":["cmd","w"]}`}, + {"type_text", `{"pid":42,"window_id":12,"text":"kettle","element_token":"s1:1"}`}, + {"press_key", `{"pid":42,"window_id":12,"key":"return"}`}, + {"press_key", `{"pid":42,"window_id":12,"key":"return","element_token":"s1:1","modifiers":[]}`}, + {"press_key", `{"pid":42,"window_id":12,"key":"cmd+r","element_token":"s1:1"}`}, + {"press_key", `{"pid":42,"window_id":9,"key":"return","element_token":"s1:1"}`}, + {"recognize_text", `{"pid":42,"window_id":12,"pages":2,"next_key":"pagedown"}`}, } { - if _, err := c.call(t.Context(), "hotkey", json.RawMessage(call)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { - t.Fatalf("hotkey err=%v, want refused", err) + if _, err := c.call(t.Context(), call.name, json.RawMessage(call.args)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { + t.Fatalf("%s %s: err=%v, want refused", call.name, call.args, err) } } - if _, err := c.call(t.Context(), "type_text", json.RawMessage(`{"pid":42,"window_id":12,"text":"kettle","element_token":"s1:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { - t.Fatalf("type_text err=%v, want refused", err) - } - if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":12,"key":"return"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { - t.Fatalf("unbound press_key err=%v, want refused", err) - } if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":12,"key":"return","element_token":"s1:1"}`)); err != nil { t.Fatalf("Return bound to a field: %v", err) } if _, err := c.call(t.Context(), "set_value", json.RawMessage(`{"pid":42,"window_id":12,"value":"kettle","element_token":"s1:1"}`)); err != nil { t.Fatalf("set_value: %v", err) } - // A script there may not send keys either. - if _, err := c.call(t.Context(), "run_script", json.RawMessage(`{"script":"return 1"}`)); err != nil { - t.Fatal(err) - } - if calls := fixtureCalls(t, marker); !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "noKeys = true") { - t.Fatalf("last call=%v, want a script guarded against keys", calls[len(calls)-1]) + // A script there, or in one of the user's windows, may not send keys. + for _, script := range []string{`{"script":"return 1"}`, `{"script":"return 1","args":{"pid":42,"window_id":9}}`} { + if _, err := c.call(t.Context(), "run_script", json.RawMessage(script)); err != nil { + t.Fatal(err) + } + if calls := fixtureCalls(t, marker); !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "noKeys = true") { + t.Fatalf("%s: last call=%v, want a script guarded against keys", script, calls[len(calls)-1]) + } } if err := c.drain(); err != nil { t.Fatal(err) } } +// A launch opens a fresh window when the listed one has left the +// accessibility tree. func TestLocalToolCuaMacAgentChromeOpensAFreshWindowForAnUnreadableOne(t *testing.T) { if runtime.GOOS != "darwin" { t.Skip("the Computer's own Chrome is macOS only") @@ -1987,7 +1994,7 @@ func TestLocalToolCuaMacAgentChromeOpensAFreshWindowForAnUnreadableOne(t *testin } t.Cleanup(func() { agentChromeOpen = open }) b := agentBrowser{Name: "Chrome", Bundle: "com.google.Chrome", Directory: "agent-chrome-test", Args: func(string) []string { return nil }, AddressRole: "AXTextField", AddressLabel: "Address and search bar"} - result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}, false) + result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}) if err != nil { t.Fatal(err) } diff --git a/apps/computer/local-daemon/internal/linger/localtools/manager.go b/apps/computer/local-daemon/internal/linger/localtools/manager.go index 201295ca..ed46edf5 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/manager.go +++ b/apps/computer/local-daemon/internal/linger/localtools/manager.go @@ -98,6 +98,9 @@ type Manager struct { sharedBrowserBundle string sharedBrowserPID int sharedBrowserWindow float64 + // everydayChromeQuitAsked records that the Computer asked the user to quit + // their Chrome, so the next start of it brings back their windows. + everydayChromeQuitAsked bool // paceMu guards the persisted pace record of the Computer's own browser // and paceLast, when that Chrome last received input; see // paceAgentChrome. diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md index 8c0b3f7a..2d80eed4 100644 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -11,7 +11,7 @@ description: Read, survey, post to or manage a site that watches for automated b - Anything else on a social network, a marketplace or a site where the user is signed in: only the computer tools, through `launch_app({agent_browser: true, urls: [url]})`. In Chrome or Safari this opens a window of the Computer's own in the user's everyday browser, with the user's sign-ins; Edge and Firefox run as separate instances. Pass `bundle_id` only when the user asks for one browser. The browser tools drive a page in a way these sites detect, so never use them on these sites; the Computer keeps them off every site the task opened this way. - If `launch_app` has no `agent_browser`, this needs a Mac Computer: tell the user and stop. - If a call says a computer tool does not exist, take the tool's exact name from your tool list and call it again before telling the user the tools are unavailable. -- If `launch_app` answers `COMPUTER_BROWSER_RESTART_REQUIRED`, ask the user with a question card whether the Computer may restart Chrome, saying that their windows and tabs come back. Only if they agree, call `launch_app` again with `restart_browser: true`; otherwise end the task. +- If `launch_app` answers `COMPUTER_BROWSER_RESTART_REQUIRED`, ask the user with a question card to quit Chrome from its menu and leave it closed, saying that the Computer starts it again with their windows and tabs; then end your turn. When they answer that Chrome is closed, call `launch_app` again; if they decline, end the task. - If the site asks to sign in, ask the user to sign in once in that browser window and stop; the sign-in stays for later tasks. ## Move like a person From 645c2679129974ac96b5e65e847a62cd47420b71 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 11:08:38 +0800 Subject: [PATCH 04/30] fix(computer): find the Computer's Chrome window and refuse keys in any language and from any script Tabs and the address bar are found by role and place, not by names that follow the user's language. Chrome starts as its own instance and the window list must settle first; the Computer's window is the new one-tab window of the site, or the only new window, and two of the site are refused. Keys are placed by the window their token was read from, a key that names no app is refused while the user's Chrome runs, and every script checks keys against that Chrome's pid. Shared browser windows are kept per browser, the quit question survives a Daemon restart, and the old agent Chrome is quit before its profile is removed. Co-Authored-By: Claude Code --- apps/computer/local-daemon/README.md | 33 ++- .../linger/localtools/agent_browser.go | 255 +++++++++++++----- .../linger/localtools/agent_chrome.go | 49 ++-- .../internal/linger/localtools/cua.go | 29 +- .../internal/linger/localtools/cua_script.go | 20 +- .../localtools/cua_script_guard_test.go | 42 +-- .../linger/localtools/cua_script_test.go | 20 +- .../internal/linger/localtools/evidence.go | 13 + .../linger/localtools/localtools_test.go | 84 +++++- .../internal/linger/localtools/manager.go | 32 +-- .../builtin/humanlike-browse/1.0.5/SKILL.md | 2 +- 11 files changed, 408 insertions(+), 171 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index db5ac5d5..d335a62a 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -334,19 +334,28 @@ starts with it, and one running without it returns `COMPUTER_BROWSER_RESTART_REQUIRED`. The Daemon never quits the user's Chrome, because a quit loses text typed into a page and not yet sent: the Agent asks the user to quit Chrome, and the next `launch_app` starts it with the flag and -`--restore-last-session`. The Computer's window is the new window with one tab -whose address bar shows the site just opened, so a window the user opens or -Chrome restores meanwhile is never taken for it. The Daemon finds the running -Chrome by its executable wherever the app is installed. The user's Chrome -takes no keys from the Computer (`COMPUTER_KEYS_REFUSED`), in any of its -windows, in tool calls, `run_actions` steps, paged `recognize_text` reads and -scripts that target it: a shortcut runs as a menu command in the Chrome window -that has the keyboard, and Cua types text as keys that all read as the A key. -Only a plain key bound to an element of the Computer's window goes, and Cua -sends it only when no other Chrome window could take it. The Agent fills fields with +`--restore-last-session`. The Daemon records that question in +`computer-tools/everyday-chrome-quit` in the state directory, so a Daemon +restart keeps it, and removes the record once Chrome started. Before the +Computer opens its window, it waits until the list of Chrome windows stops +changing, so every restored window is there. The Computer's window is the new +window with one tab: the one whose address bar shows the site just opened, or +the only new window when the home page moved to another site. Two new windows +of the site are refused with `COMPUTER_WINDOW_AMBIGUOUS`. The address bar and +the tabs are found by role and place in the window, not by their names, which +follow the user's language. The Daemon finds the running Chrome by its +executable wherever the app is installed. The user's Chrome takes no keys from +the Computer (`COMPUTER_KEYS_REFUSED`), in any of its windows, in tool calls, +`run_actions` steps, paged `recognize_text` reads and every script: a shortcut +runs as a menu command in the Chrome window that has the keyboard, and Cua +types text as keys that all read as the A key. A key goes to the window its +`element_token` was read from, and a key that names no app is refused while +that Chrome runs. Only a plain key bound to an element of the Computer's +window goes, and Cua sends it only when no other Chrome window could take it. +The Agent fills fields with `set_value`, presses the page's buttons and goes back through its links. -The separate Chrome profile earlier versions used, `computer-tools/agent-chrome`, -is removed when the Daemon starts and that Chrome no longer runs. +The Daemon removes the separate Chrome profile `computer-tools/agent-chrome` +when it starts, after quitting the Chrome that still runs on it. Edge and Firefox each run as a separate instance with their own profile under `computer-tools/agent-` in the state directory, so the user signs in there once. Safari cannot run a second profile; the Computer uses its own diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index a217c74f..76326860 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -48,8 +48,8 @@ type agentBrowser struct { Shared bool // Everyday marks the user's everyday Chrome: the Computer opens each // site in a window of its own there, by command line, and the running - // instance must carry Settings, so a covered window keeps working. One - // started without them restarts only after the user agreed. + // instance must carry Settings, so a covered window keeps working. The + // user quits one started without them; the Computer never does. Everyday bool } @@ -194,8 +194,8 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls return nil, err } m.mu.Lock() - own := m.sharedBrowserWindow - if m.sharedBrowserPID != pid { + own := m.sharedWindows[b.Bundle].Window + if m.sharedWindows[b.Bundle].PID != pid { own = 0 } m.mu.Unlock() @@ -238,9 +238,7 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls // window's accessibility tree resolves; opening a tab waits for it. c.agentChromeTabsUntil(ctx, pid, window.WindowID, agentSafariWindowWait, func(n int) bool { return n > 0 }) } - m.mu.Lock() - m.sharedBrowserBundle, m.sharedBrowserPID, m.sharedBrowserWindow = b.Bundle, pid, window.WindowID - m.mu.Unlock() + m.setSharedWindow(b.Bundle, pid, window.WindowID) for _, url := range urls { if err := c.agentChromeOpenTab(ctx, b, pid, window.WindowID, offSpace(window), url); err != nil { return nil, err @@ -316,7 +314,9 @@ var everydayChromePID = func(ctx context.Context) (int, string, error) { // ends where the first argument starts. func everydayChromeCommand(args string) bool { index := strings.Index(args, everydayChromeExecutable) - if !strings.HasPrefix(args, "/") || index < 0 { + // A command that only names Chrome's path as an argument has an option + // or another path before it. + if !strings.HasPrefix(args, "/") || index < 0 || strings.Contains(args[:index], " -") || strings.Contains(args[:index], " /") { return false } rest := args[index+len(everydayChromeExecutable):] @@ -360,19 +360,19 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u return nil, err } if pid > 0 && !b.startedWith(command) { - m.mu.Lock() - m.everydayChromeQuitAsked = true - m.mu.Unlock() + if err := m.everydayChromeQuitAsked(true); err != nil { + return nil, err + } return nil, fmt.Errorf("%s: %w", everydayChromeQuit, errNotSent) } restored := false if pid == 0 { // After the user quit Chrome for the Computer, it reopens their - // windows and tabs, as the question to quit promised. - m.mu.Lock() - restored, m.everydayChromeQuitAsked = m.everydayChromeQuitAsked, false - m.mu.Unlock() - args := append([]string{"-g", "-b", b.Bundle, "--args"}, b.Settings...) + // windows and tabs, as the question to quit promised. A second + // instance is started (-n) so a Chrome on another profile, such as + // a test browser, cannot take the command and drop the settings. + restored = m.quitAsked() + args := append([]string{"-n", "-g", "-b", b.Bundle, "--args"}, b.Settings...) if restored { args = append(args, "--restore-last-session") } @@ -382,6 +382,9 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u if pid, err = c.waitEverydayChrome(ctx); err != nil { return nil, err } + if err := m.everydayChromeQuitAsked(false); err != nil { + return nil, err + } } windows, err := c.appWindows(ctx, pid) if err != nil { @@ -398,53 +401,77 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u if output, err := agentChromeOpen(ctx, open); err != nil { return nil, fmt.Errorf("launch_app: %s: %w", strings.TrimSpace(string(output)), err) } - // The Computer's window is a new window with one tab whose address bar - // shows the site just opened. A window the user opens meanwhile, or one - // Chrome restores late, fails that test, so it is never taken for it. - var window listedWindow - tabs := 0 - for deadline := time.Now().Add(agentSafariWindowWait); window.WindowID == 0; { + window, tabs, err := c.everydayChromeWindow(ctx, b, pid, before, site) + if err != nil { + return nil, err + } + m.setSharedWindow(b.Bundle, pid, window.WindowID) + c.op.mu.Lock() + c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = tabs, pid, window.WindowID + c.op.mu.Unlock() + note := "This is the Computer's own window in the user's everyday Chrome, with the user's sign-ins; work only in this window, never in the user's other Chrome windows. It works while other windows cover it, but only on the desktop (Space) the user is on. The user's Chrome takes no keys from the Computer, because a key could reach the window the user is typing in: fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links. Close a tab with the Close button of its tab; closing the window's last tab closes the window. If a site asks to sign in, ask the user to sign in once in this window." + if restored { + note = "Started the user's Chrome again with their windows and tabs. " + note + } + text := fmt.Sprintf("Opened %s in a new window of the user's Chrome (pid %d), window %q [window_id: %v]. The window has %d tab(s).\n%s", urls[0], pid, window.Title, window.WindowID, tabs, note) + return json.Marshal(map[string]any{"content": []any{map[string]string{"type": "text", "text": text}}, + "structuredContent": map[string]any{"browser": b.Name, "pid": pid, "windows": []map[string]any{{"window_id": window.WindowID, "title": window.Title, "on_screen": window.IsOnScreen}}, "tabs": tabs, "note": note}}) +} + +// everydayChromeWindow finds the window the command line just opened: a +// window not in before, with one tab. When several such windows show, such +// as one the user opened meanwhile, the one whose address bar shows site is +// it; a lone new window counts once it has stood for everydayChromeSettle, +// so a home page that moves to another site, such as a sign-in page, still +// finds its window. Windows that stay ambiguous are refused. +func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, pid int, before map[float64]bool, site string) (listedWindow, int, error) { + var seen time.Time + for deadline := time.Now().Add(agentSafariWindowWait); ; { windows, err := c.appWindows(ctx, pid) if err != nil { - return nil, err + return listedWindow{}, 0, err } + var fresh, matching []listedWindow for _, w := range windows { - if before[w.WindowID] || siteOf(c.windowAddress(ctx, b, pid, w.WindowID)) != site { + if before[w.WindowID] || c.agentChromeTabs(ctx, pid, w.WindowID) != 1 { continue } - if n := c.agentChromeTabs(ctx, pid, w.WindowID); n == 1 { - window, tabs = w, n - break + fresh = append(fresh, w) + if siteOf(c.windowAddress(ctx, b, pid, w.WindowID)) == site { + matching = append(matching, w) } } - if window.WindowID != 0 { - break + if len(fresh) > 0 && seen.IsZero() { + seen = time.Now() + } + settled := !seen.IsZero() && time.Since(seen) >= everydayChromeSettle + switch { + case len(matching) == 1: + return matching[0], 1, nil + case len(matching) == 0 && len(fresh) == 1 && settled: + return fresh[0], 1, nil + case len(fresh) > 1 && settled && len(matching) != 1, len(fresh) > 1 && time.Now().After(deadline): + return listedWindow{}, 0, fmt.Errorf("COMPUTER_WINDOW_AMBIGUOUS: %d new one-tab windows of the user's Chrome could be the one the Computer opened, so it works in none of them; ask the user to close the windows they do not need, then call launch_app again", len(fresh)) } if time.Now().After(deadline) { - return nil, errors.New("launch_app: the user's Chrome did not open a window for the Computer") + return listedWindow{}, 0, errors.New("launch_app: the user's Chrome did not open a window for the Computer") } if err := sleepContext(ctx, 200*time.Millisecond); err != nil { - return nil, err + return listedWindow{}, 0, err } } - m.mu.Lock() - m.sharedBrowserBundle, m.sharedBrowserPID, m.sharedBrowserWindow = b.Bundle, pid, window.WindowID - m.mu.Unlock() - c.op.mu.Lock() - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = tabs, pid, window.WindowID - c.op.mu.Unlock() - note := "This is the Computer's own window in the user's everyday Chrome, with the user's sign-ins; work only in this window, never in the user's other Chrome windows. It works while other windows cover it, but only on the desktop (Space) the user is on. The user's Chrome takes no keys from the Computer, because a key could reach the window the user is typing in: fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links. Close a tab with the Close button of its tab; closing the window's last tab closes the window. If a site asks to sign in, ask the user to sign in once in this window." - if restored { - note = "Started the user's Chrome again with their windows and tabs. " + note - } - text := fmt.Sprintf("Opened %s in a new window of the user's Chrome (pid %d), window %q [window_id: %v]. The window has %d tab(s).\n%s", urls[0], pid, window.Title, window.WindowID, tabs, note) - return json.Marshal(map[string]any{"content": []any{map[string]string{"type": "text", "text": text}}, - "structuredContent": map[string]any{"browser": b.Name, "pid": pid, "windows": []map[string]any{{"window_id": window.WindowID, "title": window.Title, "on_screen": window.IsOnScreen}}, "tabs": tabs, "note": note}}) } +// everydayChromeSettle is how long a window list of the user's Chrome must +// stay the same before the Computer trusts it: Chrome reopens restored +// windows one after another. +var everydayChromeSettle = 1500 * time.Millisecond + // windowAddress returns the web address the address bar of window shows, // with https:// added when the bar leaves out the scheme, or "" when the -// bar cannot be read. +// bar cannot be read. The address bar is the window's one text field of +// role b.AddressRole outside the page; its name follows the user's +// language, so it is not read. func (c *cuaSession) windowAddress(ctx context.Context, b agentBrowser, pid int, window float64) string { read, err := c.client.request(ctx, "tools/call", map[string]any{"name": "get_window_state", "arguments": map[string]any{ "pid": pid, "window_id": window, "tree_format": "elements", "include_screenshot": false, "max_elements": 3000, "timeout_ms": 4000, "session": c.label}}) @@ -455,15 +482,15 @@ func (c *cuaSession) windowAddress(ctx context.Context, b agentBrowser, pid int, Structured struct { Elements []struct { Role string `json:"role"` - Label string `json:"label"` Value any `json:"value"` + InWeb bool `json:"in_web_content"` } `json:"elements"` } `json:"structuredContent"` } _ = json.Unmarshal(read, &state) for _, element := range state.Structured.Elements { value, ok := element.Value.(string) - if !ok || element.Role != b.AddressRole || element.Label != b.AddressLabel { + if !ok || element.Role != b.AddressRole || element.InWeb { continue } if !strings.Contains(value, "://") { @@ -475,17 +502,27 @@ func (c *cuaSession) windowAddress(ctx context.Context, b agentBrowser, pid int, } // waitEverydayChrome waits until the user's Chrome that just started runs -// and shows a window, and returns its pid. +// and its window list has stood for everydayChromeSettle, so every window it +// restores is there before the Computer opens its own, and returns its pid. func (c *cuaSession) waitEverydayChrome(ctx context.Context) (int, error) { + last, since := "", time.Now() for deadline := time.Now().Add(agentSafariWindowWait); ; { pid, _, err := everydayChromePID(ctx) if err != nil { return 0, err } if pid > 0 { - if windows, err := c.appWindows(ctx, pid); err != nil { + windows, err := c.appWindows(ctx, pid) + if err != nil { return 0, err - } else if len(windows) > 0 { + } + ids := make([]string, 0, len(windows)) + for _, w := range windows { + ids = append(ids, fmt.Sprint(w.WindowID)) + } + if list := strings.Join(ids, ","); list != last { + last, since = list, time.Now() + } else if len(windows) > 0 && time.Since(since) >= everydayChromeSettle { return pid, nil } } @@ -498,28 +535,114 @@ func (c *cuaSession) waitEverydayChrome(ctx context.Context) (int, error) { } } -// userChrome reports whether pid is the user's everyday Chrome, whichever of -// its windows an input names: a key reaches the Chrome window that has the -// keyboard, not the one named. The pid the Computer opened its window in is -// known; any other pid is looked up, so the rule holds before the Computer -// opened a window too. -func (c *cuaSession) userChrome(ctx context.Context, pid any) (bool, error) { - number, ok := pid.(float64) - if !ok || runtime.GOOS != "darwin" { - return false, nil +// everydayChromeQuitAsked records, in the state directory so a Daemon +// restart keeps it, whether the Computer asked the user to quit their Chrome +// and has not started it again since. +func (m *Manager) everydayChromeQuitAsked(asked bool) error { + path := filepath.Join(m.options.StateDirectory, "computer-tools", "everyday-chrome-quit") + if !asked { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + if err := privatefs.EnsureDir(filepath.Dir(path)); err != nil { + return err + } + return os.WriteFile(path, nil, 0o600) +} + +// quitAsked reports whether the Computer asked the user to quit their +// Chrome; see everydayChromeQuitAsked. +func (m *Manager) quitAsked() bool { + _, err := os.Stat(filepath.Join(m.options.StateDirectory, "computer-tools", "everyday-chrome-quit")) + return err == nil +} + +// sharedWindow is the window the Computer opened as its own in one of the +// user's own browsers, and that browser's pid. +type sharedWindow struct { + PID int + Window float64 +} + +// setSharedWindow records the Computer's window in the user's browser +// bundle. Callers do not hold m.mu. +func (m *Manager) setSharedWindow(bundle string, pid int, window float64) { + m.mu.Lock() + defer m.mu.Unlock() + if m.sharedWindows == nil { + m.sharedWindows = map[string]sharedWindow{} + } + m.sharedWindows[bundle] = sharedWindow{PID: pid, Window: window} +} + +// sharedWindowOf returns the Computer's window in the user's browser that +// runs as pid. Callers do not hold m.mu. +func (m *Manager) sharedWindowOf(pid int) (sharedWindow, bool) { + m.mu.Lock() + defer m.mu.Unlock() + for _, shared := range m.sharedWindows { + if pid > 0 && shared.PID == pid { + return shared, true + } + } + return sharedWindow{}, false +} + +// userChromeWindow returns the pid of the user's everyday Chrome, or 0 when +// it does not run, and the window the Computer opened as its own there, or +// 0. Chrome is looked up on each call, so the key rule holds before the +// Computer opened a window and after Chrome restarts. +func (c *cuaSession) userChromeWindow(ctx context.Context) (sharedWindow, error) { + if runtime.GOOS != "darwin" { + return sharedWindow{}, nil + } + pid, _, err := everydayChromePID(ctx) + if err != nil { + return sharedWindow{}, fmt.Errorf("COMPUTER_INPUT_NOT_SENT: could not tell whether the user's Chrome runs: %w: %w", err, errNotSent) } m := c.op.manager m.mu.Lock() - shared := m.sharedBrowserBundle == "com.google.Chrome" && m.sharedBrowserPID > 0 && int(number) == m.sharedBrowserPID + own := m.sharedWindows["com.google.Chrome"] m.mu.Unlock() - if shared { - return true, nil + if pid == 0 || own.PID != pid { + own.Window = 0 } - running, _, err := everydayChromePID(ctx) - if err != nil { - return false, fmt.Errorf("COMPUTER_INPUT_NOT_SENT: could not tell whether the input goes to the user's Chrome: %w: %w", err, errNotSent) + own.PID = pid + return own, nil +} + +// userChromeKeys refuses a key tool, or a paged read, that reaches the user's +// everyday Chrome. A key goes to the Chrome window that has the keyboard, so +// the window is the one the element_token was read from, not the one named, +// and a key that names no app is refused while that Chrome runs. +func (c *cuaSession) userChromeKeys(ctx context.Context, name string, arguments map[string]any, paged bool) error { + chrome, err := c.userChromeWindow(ctx) + if err != nil || chrome.PID == 0 { + return err + } + pid, window := arguments["pid"], arguments["window_id"] + if token, ok := arguments["element_token"].(string); ok { + // A token the Computer did not see read cannot be placed in a + // window, so it is not taken for one in the Computer's window. + window = nil + if key, found := c.op.reads.window(token); found { + pid, window = float64(key.pid), float64(key.window) + } + } + number, ok := pid.(float64) + if !ok { + return fmt.Errorf("COMPUTER_TARGET_REQUIRED: while the user's Chrome runs, a key names its app by pid, so it cannot reach Chrome unseen: %w", errNotSent) + } + if int(number) != chrome.PID { + return nil + } + if paged { + return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) } - return running > 0 && running == int(number), nil + id, _ := window.(float64) + return everydayChromeKeys(name, arguments, chrome.Window != 0 && id == chrome.Window) } // everydayChromeKeysRefused tells the Agent how to work without keys in the diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 9ffb6d61..575ba505 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "os" "os/exec" "path/filepath" @@ -427,11 +428,10 @@ func (c *cuaSession) isAgentChrome(ctx context.Context, pid, window any) (bool, m := c.op.manager m.mu.Lock() known, checked := m.agentChromePID, m.agentChromeChecked - shared, sharedWindow := m.sharedBrowserPID, m.sharedBrowserWindow m.mu.Unlock() - if shared > 0 && shared == int(number) { + if shared, ok := m.sharedWindowOf(int(number)); ok { id, named := window.(float64) - return !named || id == sharedWindow, nil + return !named || id == shared.Window, nil } if known > 0 && known == int(number) && time.Since(checked) < agentChromeRecheck { return true, nil @@ -487,11 +487,15 @@ func (c *cuaSession) agentChromeRunning(ctx context.Context) (int, error) { } m := c.op.manager m.mu.Lock() - shared, bundle := m.sharedBrowserPID, m.sharedBrowserBundle + shared := maps.Clone(m.sharedWindows) m.mu.Unlock() - if own == 0 && shared > 0 { - if running, _ := sharedBrowserRunning(ctx, bundle); running == shared { - return shared, nil + if own == 0 { + // The user's own browsers are checked in a fixed order, so one + // call names the same browser each time. + for _, bundle := range slices.Sorted(maps.Keys(shared)) { + if running, _ := sharedBrowserRunning(ctx, bundle); running > 0 && running == shared[bundle].PID { + return running, nil + } } } m.mu.Lock() @@ -548,13 +552,11 @@ func (c *cuaSession) agentChromeKeys(ctx context.Context, arguments map[string]a if _, ok := arguments["window_id"].(float64); ok { return nil } - m := c.op.manager - m.mu.Lock() - shared, sharedWindow := m.sharedBrowserPID, m.sharedBrowserWindow - m.mu.Unlock() - if pid, ok := arguments["pid"].(float64); ok && shared > 0 && int(pid) == shared { - arguments["window_id"] = sharedWindow - return nil + if pid, ok := arguments["pid"].(float64); ok { + if shared, found := c.op.manager.sharedWindowOf(int(pid)); found { + arguments["window_id"] = shared.Window + return nil + } } windows, err := c.appWindows(ctx, arguments["pid"]) if err != nil { @@ -604,13 +606,24 @@ func (c *cuaSession) agentChromeTabs(ctx context.Context, pid int, window float6 } // countTabs counts the tab rows of a browser window's tree: a radio button -// whose next row, or a row nested under it, is a Close or Close tab button. +// outside the page that holds a button, its Close button. func countTabs(tree string) int { rows := strings.Split(tree, "\n") indent := func(row string) int { return len(row) - len(strings.TrimLeft(row, " ")) } count := 0 + // web is the indent of the page being skipped, or -1: a radio button + // in a page is not a tab. + web := -1 for i, row := range rows { - if !strings.Contains(row, "] AXRadioButton \"") { + if web >= 0 && indent(row) > web { + continue + } + web = -1 + if strings.Contains(row, "] AXWebArea") || strings.HasPrefix(strings.TrimLeft(row, " -"), "AXWebArea") { + web = indent(row) + continue + } + if !strings.Contains(row, "] AXRadioButton") { continue } if strings.Contains(row, "id=TabBarTab") { @@ -618,11 +631,13 @@ func countTabs(tree string) int { count++ continue } + // A tab holds its Close button, whose name follows the user's + // language, so any button inside it counts. for j, next := range rows[i+1:] { if j > 0 && indent(next) <= indent(row) { break } - if trimmed := strings.TrimSpace(next); strings.HasSuffix(trimmed, `AXButton "Close" [actions=[press]]`) || strings.HasSuffix(trimmed, `AXButton "Close tab" [actions=[press]]`) { + if trimmed := strings.TrimSpace(next); strings.Contains(trimmed, "] AXButton") && strings.HasSuffix(trimmed, "[actions=[press]]") { count++ break } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 324df202..86fb750f 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -223,7 +223,7 @@ const MacToolGuide = `Computer tools on this Mac (MCP server "computer"): - report_blocker({kind, url}) as soon as a page in the Computer's own browser shows a check only the user may pass (captcha, verification, login_required, rate_limited) or one item that cannot be viewed (content_unavailable). Never try to pass the check. The Computer's own browser takes input at a person's pace, within a budget per task and per day, and a script there sends at most 20 inputs: return what you have and look at the window before the next script. A tab that a link in the script opens is the task's and is in front afterwards. The browser tools stay off every site a task opened in that browser. Plan as many steps as you can in one run_actions call, as in one script; its read shows their effect, so do not read again before the next input unless you need another window. Keys are often shorter than clicks: in Finder, cmd+shift+g then a path and Return opens a folder, cmd+down opens the selected item, and in the Trash cmd+delete puts the selected item back. -In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. In the Computer's window of the user's everyday Chrome no key or shortcut is sent: fill fields with set_value, press the page's buttons, and go back through its links. In Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. +In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. The user's everyday Chrome takes no shortcut, typed text or unbound key in any window; only one plain key bound to an element of the Computer's window, such as Return in its search box, goes. Fill fields with set_value, press the page's buttons, and go back through its links. In Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. A menu item in the read, such as File > Put Back, can be clicked directly; do not click a menu bar title first, because an open menu holds the app until it closes. A right_click menu does not show in the read, so select the row with click and use the menu bar item or its keyboard shortcut. Before the first shell screenshot, or when one fails or comes back blank, run ` + "`teamhub computer permissions status`" + `; if Screen Recording is missing for Linger, run ` + "`teamhub computer permissions request --permissions screen_recording`" + ` and end your turn — the card wakes you. Never operate System Settings or privacy prompts yourself. ` + browserToolGuide @@ -398,22 +398,9 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage // A key to the user's Chrome reaches the window that has the keyboard, // whichever window it names, so it is refused before anything is taken. if name == "hotkey" || name == "type_text" || name == "press_key" || paged { - user, err := c.userChrome(ctx, arguments["pid"]) - if err != nil { + if err := c.userChromeKeys(ctx, name, arguments, paged); err != nil { return nil, err } - if user && paged { - return nil, fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) - } - if user { - own, err := c.isAgentChrome(ctx, arguments["pid"], arguments["window_id"]) - if err != nil { - return nil, err - } - if err := everydayChromeKeys(name, arguments, own); err != nil { - return nil, err - } - } } agentChrome, err := c.claimAgentChromeInput(ctx, name, arguments, paged) if err != nil { @@ -459,13 +446,13 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage } } if name == "run_script" { - noKeys := false - if scriptTarget != nil { - if noKeys, err = c.userChrome(ctx, scriptTarget["pid"]); err != nil { - return nil, err - } + // A script's body may reach any app, so its guard checks every key + // against the user's Chrome, whatever window args names. + chrome, err := c.userChromeWindow(ctx) + if err != nil { + return nil, err } - if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, noKeys); err != nil { + if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, chrome); err != nil { return nil, err } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index ab020ee9..d36768b4 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -69,7 +69,11 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // Computer's own browser: the guard waits a person's pause before each input, // throws once the script tries to send more than inputLimit inputs, and // refuses launch_app, whose site checks run only in the Daemon's own call. -func guardedScript(args, body string, deadline int64, inputLimit int, noKeys bool) string { +// While the user's everyday Chrome runs as chrome.PID, the guard refuses +// every key to it, and a key that names no app, by the rule of +// everydayChromeKeys: only one plain press_key bound to an element token in +// the Computer's window chrome.Window goes. +func guardedScript(args, body string, deadline int64, inputLimit int, chrome sharedWindow) string { pattern := make([]string, 0, len(scriptForbidden)) for _, word := range scriptForbidden { pattern = append(pattern, regexp.QuoteMeta(word)) @@ -80,8 +84,12 @@ const keys = Object.keys, names = Object.getOwnPropertyNames, proto = Object.get const apply = Reflect.apply, stringify = JSON.stringify, parse = JSON.parse, test = RegExp.prototype.test, then = Promise.prototype.then; const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis.cua; const bad = /`+strings.Join(pattern, "|")+`/; -const limit = `+strconv.Itoa(inputLimit)+`, noKeys = `+strconv.FormatBool(noKeys)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; -const keyed = (k, c) => k === "hotkey" || k === "typeText" || k === "pressKey" && (c.length !== 2 || c[1] === null || c[1] === undefined || typeof c[0] !== "string" || c[0].indexOf("+") >= 0) || k === "call" && (c[0] === "hotkey" || c[0] === "type_text" || c[0] === "press_key" && (c[1] === null || typeof c[1] !== "object" || typeof c[1].element_token !== "string" || c[1].modifiers !== undefined || typeof c[1].key !== "string" || c[1].key.indexOf("+") >= 0)); +const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome.PID)+`, own = `+strconv.FormatFloat(chrome.Window, 'f', -1, 64)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; +const plainKey = (key, token, win) => typeof key === "string" && key.indexOf("+") < 0 && typeof token === "string" && own !== 0 && win === own; +const keyed = (k, c, o) => { if (chrome === 0) return false; +if (k === "call") { if (c[0] !== "hotkey" && c[0] !== "type_text" && c[0] !== "press_key") return false; const a = c[1] !== null && typeof c[1] === "object" ? c[1] : {}; +if (typeof a.pid !== "number") return true; if (a.pid !== chrome) return false; return !(c[0] === "press_key" && a.modifiers === undefined && plainKey(a.key, a.element_token, a.window_id)); } +if (k !== "hotkey" && k !== "typeText" && k !== "pressKey") return false; if (o.pid !== chrome) return false; return !(k === "pressKey" && c.length === 2 && plainKey(c[0], c[1], o.windowId)); }; const inputMethods = {__proto__: null, click: true, doubleClick: true, rightClick: true, drag: true, scroll: true, typeText: true, pressKey: true, hotkey: true, setValue: true}; const inputOps = {__proto__: null, click: true, double_click: true, right_click: true, drag: true, scroll: true, type_text: true, press_key: true, hotkey: true, set_value: true}; const input = (k, c) => inputMethods[k] === true || k === "call" && typeof c[0] === "string" && inputOps[c[0]] === true; @@ -98,7 +106,7 @@ if (limit > 0 && k === "typeText") { const given = c.length > 2 && c[2] !== null if (limit > 0 && k === "sleep" && o === real && typeof c[0] === "number" && c[0] > `+strconv.Itoa(scriptSleepLimit)+`) { const t = {length: 1}; define(t, 0, {value: `+strconv.Itoa(scriptSleepLimit)+`}); c = t; } const run = () => { const r = apply(v, o, c); return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [out]) : out(r); }; if (limit > 0 && (k === "launchApp" || k === "call" && c[0] === "launch_app")) throw new Error("COMPUTER_SCRIPT_REFUSED: a script in the Computer's own browser opens no app or URL, so the site checks of launch_app hold; call launch_app as its own tool"); -if (noKeys && keyed(k, c)) throw new Error(`+strconv.Quote(everydayChromeKeysRefused)+`); +if (keyed(k, c, o)) throw new Error(`+strconv.Quote(everydayChromeKeysRefused)+`); if (limit <= 0 || !input(k, c)) return run(); if (++sent > limit) throw new Error("COMPUTER_SCRIPT_INPUT_LIMIT: a script sends at most " + limit + " inputs to the Computer's own browser; return what you have, look at the window, then continue in a new script"); return apply(then, apply(rest, real, [`+strconv.Itoa(scriptInputPauseMin)+` + floor(random() * `+strconv.Itoa(scriptInputPauseSpan)+`)]), [run]); }; }}); } } @@ -137,7 +145,7 @@ func forbiddenKey(value any, word string) bool { // into what remains of the call after waiting for the desktop lane. In a // reviewed Run, whose review scope is not empty, a script file runs only as // the judge allowed it in that scope: the body it read and the args it saw. -func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope string, inputLimit int, noKeys bool) error { +func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope string, inputLimit int, chrome sharedWindow) error { for key := range arguments { switch key { case "script", "script_file", "args", "timeout_ms", "max_calls": @@ -196,7 +204,7 @@ func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope arguments["timeout_ms"] = timeout delete(arguments, "script_file") delete(arguments, "args") - arguments["script"] = guardedScript(string(encoded), script, time.Now().Add(time.Duration(timeout)*time.Millisecond).UnixMilli(), inputLimit, noKeys) + arguments["script"] = guardedScript(string(encoded), script, time.Now().Add(time.Duration(timeout)*time.Millisecond).UnixMilli(), inputLimit, chrome) return nil } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 25552cb1..361045f9 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -14,9 +14,9 @@ import ( // the same prelude's semantics. const guardHarness = `import fs from "fs"; const log = []; -class App { constructor(n) { this.n = n; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } } +class App { constructor(n, pid, windowId) { this.n = n; this.pid = pid; this.windowId = windowId; } hotkey(keys) { log.push("hotkey:" + keys.join("+")); return "hotkeyed"; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } } const fake = { call(t, a) { log.push("call:" + t + ":" + JSON.stringify(a)); return Promise.resolve({ok: true}); }, - getApp() { return Promise.resolve(new App("a")); }, + getApp(target) { return Promise.resolve(new App("a", target && target.pid, target && target.windowId)); }, listApps() { return Promise.resolve([new App("b")]); }, box() { return Promise.resolve({inner: {app: new App("c")}}); }, sleep(ms) { log.push("sleep:" + (ms >= 100 && ms < 250 ? "paced" : ms)); return Promise.resolve(); } }; @@ -55,7 +55,7 @@ func TestScriptGuardRefusesForbiddenInputInAJSEngine(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{"n":1}`, tc.body, 42, 0, false)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{"n":1}`, tc.body, 42, 0, sharedWindow{})), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -84,7 +84,7 @@ func TestScriptGuardPacesInputsForTheComputersChrome(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 2, false)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 2, sharedWindow{})), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -112,7 +112,7 @@ func TestScriptGuardSpeedsTypingAndCapsWaits(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 20, false)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 20, sharedWindow{})), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -145,7 +145,7 @@ func TestScriptGuardRefusesLaunchAppForTheComputersChrome(t *testing.T) { want string }{{2, "threw:COMPUTER_SCRIPT_REFUSED"}, {0, "call:launch_app"}} { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, tc.limit, false)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, tc.limit, sharedWindow{})), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -170,21 +170,29 @@ func TestScriptGuardRefusesKeysForTheEverydayChrome(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } + // The user's Chrome runs as pid 1, and the Computer's window there is 12. for _, tc := range []struct{ body, want string }{ - {`return await cua.call("hotkey", {pid: 1, keys: ["cmd", "["]});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1}); return app.typeText("kettle", {role: "AXTextField"});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, key: "return"});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2", modifiers: ["cmd"]});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2", modifiers: []});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, key: "cmd+r", element_token: "s1:2"});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1}); return app.pressKey("cmd+r", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1}); return app.pressKey("return");`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1}); return app.pressKey("return", "s1:2");`, "pressed"}, - {`return await cua.call("press_key", {pid: 1, key: "return", element_token: "s1:2"});`, "call:press_key"}, + {`return await cua.call("hotkey", {pid: 1, window_id: 12, keys: ["cmd", "["]});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("hotkey", {keys: ["cmd", "w"]});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("hotkey", {pid: 2, keys: ["cmd", "s"]});`, "call:hotkey"}, + {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.typeText("kettle", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1, windowId: 9}); return app.hotkey(["cmd", "w"]);`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.hotkey(["cmd", "s"]);`, "hotkeyed"}, + {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return", element_token: "s1:2", modifiers: ["cmd"]});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return", element_token: "s1:2", modifiers: []});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "cmd+r", element_token: "s1:2"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, window_id: 9, key: "return", element_token: "s1:2"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return", element_token: "s1:2"});`, "call:press_key"}, + {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("cmd+r", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("return");`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("return", [3, 4]);`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1, windowId: 9}); return app.pressKey("return", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("return", "s1:2");`, "pressed"}, {`return await cua.call("click", {pid: 1, element_token: "s1:2"});`, "call:click"}, } { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, 20, true)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, 20, sharedWindow{PID: 1, Window: 12})), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go index ab2cdb26..68c9ff1b 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go @@ -16,22 +16,22 @@ import ( func TestScriptArgumentsBoundTheScript(t *testing.T) { arguments := map[string]any{"script": "return 1;"} - if err := scriptArguments(context.Background(), arguments, "", 0, false); err != nil || arguments["timeout_ms"] != float64(scriptTimeoutMs) { + if err := scriptArguments(context.Background(), arguments, "", 0, sharedWindow{}); err != nil || arguments["timeout_ms"] != float64(scriptTimeoutMs) { t.Fatalf("default: %v %v", err, arguments) } arguments = map[string]any{"script": "return 1;", "timeout_ms": float64(500000)} - if err := scriptArguments(context.Background(), arguments, "", 0, false); err != nil || arguments["timeout_ms"] != float64(scriptMaxTimeoutMs) { + if err := scriptArguments(context.Background(), arguments, "", 0, sharedWindow{}); err != nil || arguments["timeout_ms"] != float64(scriptMaxTimeoutMs) { t.Fatalf("cap: %v %v", err, arguments) } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() arguments = map[string]any{"script": "return 1;", "timeout_ms": float64(90000)} - if err := scriptArguments(ctx, arguments, "", 0, false); err != nil || arguments["timeout_ms"].(float64) > 45000 { + if err := scriptArguments(ctx, arguments, "", 0, sharedWindow{}); err != nil || arguments["timeout_ms"].(float64) > 45000 { t.Fatalf("deadline: %v %v", err, arguments) } late, stop := context.WithTimeout(context.Background(), 10*time.Second) defer stop() - if err := scriptArguments(late, map[string]any{"script": "return 1;"}, "", 0, false); !errors.Is(err, errNotSent) { + if err := scriptArguments(late, map[string]any{"script": "return 1;"}, "", 0, sharedWindow{}); !errors.Is(err, errNotSent) { t.Fatalf("late: %v", err) } for _, refused := range []map[string]any{ @@ -40,7 +40,7 @@ func TestScriptArgumentsBoundTheScript(t *testing.T) { {"script": `await cua.call("click", {pid: 1, x: 2, y: 3, delivery_mode: "foreground"})`}, {"script": `await app.getState({screenshot_out_file: "/tmp/a.png"})`}, } { - if err := scriptArguments(context.Background(), refused, "", 0, false); !errors.Is(err, errNotSent) { + if err := scriptArguments(context.Background(), refused, "", 0, sharedWindow{}); !errors.Is(err, errNotSent) { t.Fatalf("%v: got %v", refused, err) } } @@ -56,7 +56,7 @@ func TestScriptArgumentsReadASkillScriptWithArgs(t *testing.T) { t.Fatal(err) } arguments := map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}} - if err := scriptArguments(context.Background(), arguments, "", 0, false); err != nil { + if err := scriptArguments(context.Background(), arguments, "", 0, sharedWindow{}); err != nil { t.Fatal(err) } script, _ := arguments["script"].(string) @@ -70,19 +70,19 @@ func TestScriptArgumentsReadASkillScriptWithArgs(t *testing.T) { {"script": `return cua.call("click", {delivery_mode: "foreground"});`}, {"script_file": file, "args": map[string]any{"click": []any{map[string]any{"delivery_mode": "foreground"}}}}, } { - if err := scriptArguments(context.Background(), refused, "", 0, false); !errors.Is(err, errNotSent) { + if err := scriptArguments(context.Background(), refused, "", 0, sharedWindow{}); !errors.Is(err, errNotSent) { t.Fatalf("%v: got %v", refused, err) } } // args carry page and user text, which may name a forbidden option. - if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"text": "in the foreground"}}, "", 0, false); err != nil { + if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"text": "in the foreground"}}, "", 0, sharedWindow{}); err != nil { t.Fatalf("user text in args: %v", err) } // A reviewed Run runs a script file only as a review allowed it. - if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}}, permission.ScriptScope("agent-1", "context-1"), 0, false); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_SCRIPT_REFUSED") { + if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}}, permission.ScriptScope("agent-1", "context-1"), 0, sharedWindow{}); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_SCRIPT_REFUSED") { t.Fatalf("unreviewed script file in a reviewed Run: %v", err) } - if err := scriptArguments(context.Background(), map[string]any{"script": "return 1;"}, permission.ScriptScope("agent-1", "context-1"), 0, false); err != nil { + if err := scriptArguments(context.Background(), map[string]any{"script": "return 1;"}, permission.ScriptScope("agent-1", "context-1"), 0, sharedWindow{}); err != nil { t.Fatalf("inline script in a reviewed Run: %v", err) } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/evidence.go b/apps/computer/local-daemon/internal/linger/localtools/evidence.go index c12fbfa0..dd60c760 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/evidence.go +++ b/apps/computer/local-daemon/internal/linger/localtools/evidence.go @@ -641,6 +641,19 @@ func applyDiff(previous map[int]string, text string) map[int]string { return rows } +// window returns the window a token was read from, even when a later read +// of that window replaced its snapshot. +func (w *windowReads) window(token string) (windowKey, bool) { + snapshot, _, ok := strings.Cut(token, ":") + if !ok { + return windowKey{}, false + } + w.mu.Lock() + defer w.mu.Unlock() + key, found := w.snapshots[snapshot] + return key, found +} + // row returns the window and row text a token names, when the token is from // that window's latest read. func (w *windowReads) row(token string) (windowKey, string, bool) { diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index b2c51898..6b87ebd9 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -205,7 +205,7 @@ func runToolFixture(mode string) { } case "list_windows": windows := []any{map[string]any{"pid": 42, "window_id": 9, "title": "b", "z_index": 3, "is_on_screen": true, "bounds": map[string]any{"width": 800, "height": 600}}} - if mode == "tabs-everyday" && strings.Count(string(data), `"name":"open"`) >= 2 { + if strings.HasPrefix(mode, "tabs-everyday") && strings.Count(string(data), `"name":"open"`) >= 2 { // After the start and the command line, a window of // the user's that Chrome restored late lists before // the Computer's window. @@ -232,6 +232,9 @@ func runToolFixture(mode string) { // Chrome's does: window 12 shows the site just opened. window, _ := request.Params.Arguments["window_id"].(float64) address := map[float64]string{12: "example.com"}[window] + if mode == "tabs-everyday-twin" && window == 11 { + address = "example.com/feed" + } if address == "" { address = "news.example.org/today" } @@ -1901,6 +1904,9 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t data := everydayChromeData everydayChromeData = func() string { return t.TempDir() } t.Cleanup(func() { everydayChromeData = data }) + settle := everydayChromeSettle + everydayChromeSettle = 0 + t.Cleanup(func() { everydayChromeSettle = settle }) b := agentBrowsers[0] // A Chrome started without the setting is never quit by the Computer. if _, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_BROWSER_RESTART_REQUIRED") || len(opened) != 0 { @@ -1909,11 +1915,17 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t // The user quits it; the Computer starts it with the setting and their // windows, then opens the site in a new window. running = 0 + if !c.op.manager.quitAsked() { + t.Fatal("the question to quit Chrome was not recorded") + } result, err := c.launchAgentChrome(t.Context(), b, []string{"https://example.com/"}) if err != nil { t.Fatal(err) } - if len(opened) != 2 || !slices.Contains(opened[0], "--restore-last-session") || !slices.Contains(opened[0], "--disable-backgrounding-occluded-windows") || !slices.Equal(opened[1][len(opened[1])-3:], []string{"--profile-directory=Default", "--new-window", "https://example.com/"}) { + if c.op.manager.quitAsked() { + t.Fatal("the question to quit Chrome outlived the start") + } + if len(opened) != 2 || opened[0][0] != "-n" || !slices.Contains(opened[0], "--restore-last-session") || !slices.Contains(opened[0], "--disable-backgrounding-occluded-windows") || !slices.Equal(opened[1][len(opened[1])-3:], []string{"--profile-directory=Default", "--new-window", "https://example.com/"}) { t.Fatalf("opened=%v, want a start with the user's windows, then the site in a new window", opened) } // Window 11, restored late, lists first, but it shows another site. @@ -1938,23 +1950,45 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t {"press_key", `{"pid":42,"window_id":12,"key":"cmd+r","element_token":"s1:1"}`}, {"press_key", `{"pid":42,"window_id":9,"key":"return","element_token":"s1:1"}`}, {"recognize_text", `{"pid":42,"window_id":12,"pages":2,"next_key":"pagedown"}`}, + // A token read from the user's window 9 places the key there, + // whatever window_id names; one never read places it nowhere. + {"press_key", `{"pid":42,"window_id":12,"key":"return","element_token":"s9:1"}`}, + {"press_key", `{"pid":42,"key":"return","element_token":"s7:1"}`}, } { + c.op.reads.mu.Lock() + if c.op.reads.snapshots == nil { + c.op.reads.windows, c.op.reads.snapshots = map[windowKey]*windowRows{}, map[string]windowKey{} + } + c.op.reads.snapshots["s9"] = windowKey{pid: 42, window: 9} + c.op.reads.mu.Unlock() if _, err := c.call(t.Context(), call.name, json.RawMessage(call.args)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { t.Fatalf("%s %s: err=%v, want refused", call.name, call.args, err) } } + // A key that names no app could reach the user's Chrome by its token. + if _, err := c.call(t.Context(), "type_text", json.RawMessage(`{"text":"kettle","element_token":"s9:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_KEYS_REFUSED") { + t.Fatalf("type_text by token alone: err=%v, want refused", err) + } + if _, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"keys":["cmd","w"],"element_token":"s7:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_TARGET_REQUIRED") { + t.Fatalf("hotkey with no pid: err=%v, want refused", err) + } + // A token read from the Computer's window 12 takes one plain key. + c.op.reads.mu.Lock() + c.op.reads.snapshots["s1"] = windowKey{pid: 42, window: 12} + c.op.reads.mu.Unlock() if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":12,"key":"return","element_token":"s1:1"}`)); err != nil { t.Fatalf("Return bound to a field: %v", err) } if _, err := c.call(t.Context(), "set_value", json.RawMessage(`{"pid":42,"window_id":12,"value":"kettle","element_token":"s1:1"}`)); err != nil { t.Fatalf("set_value: %v", err) } - // A script there, or in one of the user's windows, may not send keys. - for _, script := range []string{`{"script":"return 1"}`, `{"script":"return 1","args":{"pid":42,"window_id":9}}`} { + // Any script, whatever it targets, is guarded against keys to the + // user's Chrome, with the Computer's window 12 as the one exception. + for _, script := range []string{`{"script":"return 1"}`, `{"script":"return 1","args":{"pid":42,"window_id":9}}`, `{"script":"return 1","args":{"pid":7}}`} { if _, err := c.call(t.Context(), "run_script", json.RawMessage(script)); err != nil { t.Fatal(err) } - if calls := fixtureCalls(t, marker); !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "noKeys = true") { + if calls := fixtureCalls(t, marker); !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "chrome = 42, own = 12") { t.Fatalf("%s: last call=%v, want a script guarded against keys", script, calls[len(calls)-1]) } } @@ -1963,6 +1997,46 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t } } +// A window of the user's that shows the same site and opens beside the +// Computer's leaves the Computer unable to tell them apart, so it takes +// neither. +func TestLocalToolCuaMacEverydayChromeRefusesTwoWindowsOfTheSite(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("the Computer's window in the user's Chrome is macOS only") + } + inputSettle = 0 + t.Cleanup(func() { inputSettle = 300 * time.Millisecond }) + marker := filepath.Join(t.TempDir(), "calls.jsonl") + c := cuaFixtureSession(t, "tabs-everyday-twin", marker) + pid := everydayChromePID + everydayChromePID = func(context.Context) (int, string, error) { + return 42, "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --disable-backgrounding-occluded-windows", nil + } + t.Cleanup(func() { everydayChromePID = pid }) + open := agentChromeOpen + agentChromeOpen = func(context.Context, []string) ([]byte, error) { + f, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, err + } + defer f.Close() + // One command line shows both windows, as the fixture lists them + // from the second open on. + _, err = f.WriteString(`{"name":"open"}` + "\n" + `{"name":"open"}` + "\n") + return nil, err + } + t.Cleanup(func() { agentChromeOpen = open }) + data := everydayChromeData + everydayChromeData = func() string { return t.TempDir() } + t.Cleanup(func() { everydayChromeData = data }) + settle := everydayChromeSettle + everydayChromeSettle = 0 + t.Cleanup(func() { everydayChromeSettle = settle }) + if _, err := c.launchAgentChrome(t.Context(), agentBrowsers[0], []string{"https://example.com/"}); !strings.Contains(fmt.Sprint(err), "COMPUTER_WINDOW_AMBIGUOUS") { + t.Fatalf("err=%v, want two windows of the site refused", err) + } +} + // A launch opens a fresh window when the listed one has left the // accessibility tree. func TestLocalToolCuaMacAgentChromeOpensAFreshWindowForAnUnreadableOne(t *testing.T) { diff --git a/apps/computer/local-daemon/internal/linger/localtools/manager.go b/apps/computer/local-daemon/internal/linger/localtools/manager.go index ed46edf5..578e1788 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/manager.go +++ b/apps/computer/local-daemon/internal/linger/localtools/manager.go @@ -91,16 +91,11 @@ type Manager struct { agentChromeOwner *operation agentChromePID int agentChromeChecked time.Time - // sharedBrowserBundle, sharedBrowserPID and sharedBrowserWindow are the - // user's own browser the Computer works in, such as their everyday Chrome - // or Safari, and the window in it that the Computer opened as its own; - // see launchEverydayChrome and launchAgentSafari. - sharedBrowserBundle string - sharedBrowserPID int - sharedBrowserWindow float64 - // everydayChromeQuitAsked records that the Computer asked the user to quit - // their Chrome, so the next start of it brings back their windows. - everydayChromeQuitAsked bool + // sharedWindows holds, by bundle id, the user's own browser the Computer + // works in, such as their everyday Chrome or Safari, and the window in it + // that the Computer opened as its own; see launchEverydayChrome and + // launchAgentSafari. + sharedWindows map[string]sharedWindow // paceMu guards the persisted pace record of the Computer's own browser // and paceLast, when that Chrome last received input; see // paceAgentChrome. @@ -133,19 +128,24 @@ func New(options Options) (*Manager, error) { } // retireAgentChromeProfile removes the separate Chrome profile the Computer -// used before it worked in a window of the user's everyday Chrome. A profile -// whose Chrome still runs stays until a later start finds it closed. +// used before it worked in a window of the user's everyday Chrome. A Chrome +// still running on it is the Computer's own, so it is quit first; a profile +// whose Chrome does not quit stays until a later start. func retireAgentChromeProfile(state string) { directory := filepath.Join(state, "computer-tools", "agent-chrome") if _, err := os.Stat(directory); err != nil { return } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), agentChromeWait+5*time.Second) defer cancel() - if pid, _, err := agentChromePID(ctx, directory); err != nil || pid > 0 { - return + pid, _, err := agentChromePID(ctx, directory) + if err == nil && pid > 0 { + _, err = stopAgentChrome(ctx, directory, pid) + } + if err == nil { + err = os.RemoveAll(directory) } - if err := os.RemoveAll(directory); err != nil { + if err != nil { slog.Warn("agent_browser.retire_profile_failed", "error", err) } } diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md index 2d80eed4..6f2aa03e 100644 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -32,7 +32,7 @@ The user watches the window, and each turn of yours costs seconds. The Computer - Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. - Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. - Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Edge and Firefox. -- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; Return bound to the search box goes only while no other Chrome window could take it. +- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; only Return bound to the search box of the Computer's window goes, and only while no other Chrome window could take it. - After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. - Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. From ac0a4aa4a12ad9e32e8c92095405298da46be358 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 11:16:08 +0800 Subject: [PATCH 05/30] fix(computer): send the user's Chrome no key from a script and find it by its executable A script cannot prove which window an element token was read from, so it sends the user's Chrome no key at all and does not start Chrome. Chrome is found by the executable column of ps, so an install path with spaces or dashes still matches. The old agent Chrome is retired in the background, and a tab's Close button beside it no longer takes a button above its level. Co-Authored-By: Claude Code --- apps/computer/local-daemon/README.md | 6 ++- .../linger/localtools/agent_browser.go | 38 ++++++++++--------- .../linger/localtools/agent_browser_test.go | 23 ++++++----- .../linger/localtools/agent_chrome.go | 4 +- .../internal/linger/localtools/cua.go | 4 +- .../internal/linger/localtools/cua_script.go | 25 ++++++------ .../localtools/cua_script_guard_test.go | 28 ++++++-------- .../linger/localtools/cua_script_test.go | 20 +++++----- .../linger/localtools/localtools_test.go | 4 +- .../internal/linger/localtools/manager.go | 4 +- 10 files changed, 82 insertions(+), 74 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index d335a62a..0464be84 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -346,12 +346,14 @@ the tabs are found by role and place in the window, not by their names, which follow the user's language. The Daemon finds the running Chrome by its executable wherever the app is installed. The user's Chrome takes no keys from the Computer (`COMPUTER_KEYS_REFUSED`), in any of its windows, in tool calls, -`run_actions` steps, paged `recognize_text` reads and every script: a shortcut +`run_actions` steps, paged `recognize_text` reads and scripts: a shortcut runs as a menu command in the Chrome window that has the keyboard, and Cua types text as keys that all read as the A key. A key goes to the window its `element_token` was read from, and a key that names no app is refused while that Chrome runs. Only a plain key bound to an element of the Computer's -window goes, and Cua sends it only when no other Chrome window could take it. +window goes, as a tool call, and Cua sends it only when no other Chrome window +could take it. A script sends that Chrome no key at all, because it cannot +prove which window a token was read from, and it does not start Chrome. The Agent fills fields with `set_value`, presses the page's buttons and goes back through its links. The Daemon removes the separate Chrome profile `computer-tools/agent-chrome` diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index 76326860..ee886e8c 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -291,36 +291,40 @@ func (c *cuaSession) waitSafariWindows(ctx context.Context, b agentBrowser, pid // Google Chrome on its default profile, wherever the app is installed. Chrome // instances on a profile of their own, such as a test browser, name // --user-data-dir and are not it. It returns the pid and command line, or 0 -// when that Chrome is not running. +// when that Chrome is not running. The executable comes from its own ps +// column, because a path with spaces cannot be split from the arguments. var everydayChromePID = func(ctx context.Context) (int, string, error) { - output, err := exec.CommandContext(ctx, "/bin/ps", "-axww", "-o", "pid=,args=").Output() + paths, err := exec.CommandContext(ctx, "/bin/ps", "-axww", "-o", "pid=,comm=").Output() if err != nil { return 0, "", err } - for _, line := range strings.Split(string(output), "\n") { + lines, err := exec.CommandContext(ctx, "/bin/ps", "-axww", "-o", "pid=,args=").Output() + if err != nil { + return 0, "", err + } + commands := map[string]string{} + for _, line := range strings.Split(string(lines), "\n") { pidText, args, _ := strings.Cut(strings.TrimSpace(line), " ") - if !everydayChromeCommand(args) { + commands[pidText] = args + } + for _, line := range strings.Split(string(paths), "\n") { + pidText, path, _ := strings.Cut(strings.TrimSpace(line), " ") + if !everydayChromeProcess(path, commands[pidText]) { continue } if pid, err := strconv.Atoi(pidText); err == nil { - return pid, args, nil + return pid, commands[pidText], nil } } return 0, "", nil } -// everydayChromeCommand reports whether args starts Google Chrome's own -// executable on the default profile. The executable path has spaces, so it -// ends where the first argument starts. -func everydayChromeCommand(args string) bool { - index := strings.Index(args, everydayChromeExecutable) - // A command that only names Chrome's path as an argument has an option - // or another path before it. - if !strings.HasPrefix(args, "/") || index < 0 || strings.Contains(args[:index], " -") || strings.Contains(args[:index], " /") { - return false - } - rest := args[index+len(everydayChromeExecutable):] - return (rest == "" || rest[0] == ' ') && !strings.Contains(args, "--type=") && !strings.Contains(args, "--user-data-dir=") +// everydayChromeProcess reports whether the process with executable path +// and command line args is Google Chrome's main process on the default +// profile. +func everydayChromeProcess(path, args string) bool { + return strings.HasPrefix(path, "/") && strings.HasSuffix(path, everydayChromeExecutable) && + !strings.Contains(args, "--type=") && !strings.Contains(args, "--user-data-dir=") } const everydayChromeExecutable = "/Google Chrome.app/Contents/MacOS/Google Chrome" diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go index abcbb537..6c8208de 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go @@ -47,17 +47,20 @@ func TestPickAgentBrowserTakesTheFirstInstalledOrTheNamedOne(t *testing.T) { } } -func TestEverydayChromeCommandFindsChromeInAnyFolder(t *testing.T) { - for args, want := range map[string]bool{ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome": true, - "/Users/a/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --disable-backgrounding-occluded-windows": true, - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test": false, - "/Applications/Google Chrome.app/Contents/Frameworks/Google Chrome Framework.framework/Helpers/Google Chrome Helper.app/Contents/MacOS/Google Chrome Helper --type=renderer": false, - "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary": false, - "tail -f /Applications/Google Chrome.app/Contents/MacOS/Google Chrome": false, +func TestEverydayChromeProcessFindsChromeInAnyFolder(t *testing.T) { + for _, tc := range []struct { + path, args string + want bool + }{ + {"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", true}, + {"/Users/a/Applications/Work - Apps/Google Chrome.app/Contents/MacOS/Google Chrome", "/Users/a/Applications/Work - Apps/Google Chrome.app/Contents/MacOS/Google Chrome --disable-backgrounding-occluded-windows", true}, + {"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test", false}, + {"/Applications/Google Chrome.app/Contents/Frameworks/Google Chrome Framework.framework/Helpers/Google Chrome Helper.app/Contents/MacOS/Google Chrome Helper", "Google Chrome Helper --type=renderer", false}, + {"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", "", false}, + {"/usr/bin/tail", "/usr/bin/tail -f /Applications/Google Chrome.app/Contents/MacOS/Google Chrome", false}, } { - if got := everydayChromeCommand(args); got != want { - t.Errorf("%q: %v, want %v", args, got, want) + if got := everydayChromeProcess(tc.path, tc.args); got != tc.want { + t.Errorf("%q %q: %v, want %v", tc.path, tc.args, got, tc.want) } } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 575ba505..069f6291 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -634,7 +634,9 @@ func countTabs(tree string) int { // A tab holds its Close button, whose name follows the user's // language, so any button inside it counts. for j, next := range rows[i+1:] { - if j > 0 && indent(next) <= indent(row) { + // The next row may be the button beside the tab, at its + // level; a row above that level, such as New Tab, is not. + if j > 0 && indent(next) <= indent(row) || indent(next) < indent(row) { break } if trimmed := strings.TrimSpace(next); strings.Contains(trimmed, "] AXButton") && strings.HasSuffix(trimmed, "[actions=[press]]") { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 86fb750f..8a2fc9d0 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -223,7 +223,7 @@ const MacToolGuide = `Computer tools on this Mac (MCP server "computer"): - report_blocker({kind, url}) as soon as a page in the Computer's own browser shows a check only the user may pass (captcha, verification, login_required, rate_limited) or one item that cannot be viewed (content_unavailable). Never try to pass the check. The Computer's own browser takes input at a person's pace, within a budget per task and per day, and a script there sends at most 20 inputs: return what you have and look at the window before the next script. A tab that a link in the script opens is the task's and is in front afterwards. The browser tools stay off every site a task opened in that browser. Plan as many steps as you can in one run_actions call, as in one script; its read shows their effect, so do not read again before the next input unless you need another window. Keys are often shorter than clicks: in Finder, cmd+shift+g then a path and Return opens a folder, cmd+down opens the selected item, and in the Trash cmd+delete puts the selected item back. -In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. The user's everyday Chrome takes no shortcut, typed text or unbound key in any window; only one plain key bound to an element of the Computer's window, such as Return in its search box, goes. Fill fields with set_value, press the page's buttons, and go back through its links. In Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. +In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. The user's everyday Chrome takes no shortcut, typed text or unbound key in any window; only one plain key bound to an element of the Computer's window, such as Return in its search box, goes, and never from run_script. Fill fields with set_value, press the page's buttons, and go back through its links. In Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. A menu item in the read, such as File > Put Back, can be clicked directly; do not click a menu bar title first, because an open menu holds the app until it closes. A right_click menu does not show in the read, so select the row with click and use the menu bar item or its keyboard shortcut. Before the first shell screenshot, or when one fails or comes back blank, run ` + "`teamhub computer permissions status`" + `; if Screen Recording is missing for Linger, run ` + "`teamhub computer permissions request --permissions screen_recording`" + ` and end your turn — the card wakes you. Never operate System Settings or privacy prompts yourself. ` + browserToolGuide @@ -452,7 +452,7 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage if err != nil { return nil, err } - if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, chrome); err != nil { + if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, chrome.PID); err != nil { return nil, err } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index d36768b4..460938d6 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -69,11 +69,12 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // Computer's own browser: the guard waits a person's pause before each input, // throws once the script tries to send more than inputLimit inputs, and // refuses launch_app, whose site checks run only in the Daemon's own call. -// While the user's everyday Chrome runs as chrome.PID, the guard refuses -// every key to it, and a key that names no app, by the rule of -// everydayChromeKeys: only one plain press_key bound to an element token in -// the Computer's window chrome.Window goes. -func guardedScript(args, body string, deadline int64, inputLimit int, chrome sharedWindow) string { +// While the user's everyday Chrome runs as chrome, the guard refuses every +// key to it and every key that names no app: a script cannot prove which +// window an element token was read from, so even the one key a tool call +// may send there (see everydayChromeKeys) is refused. It also refuses to +// start Chrome, whose pid the guard would not know. +func guardedScript(args, body string, deadline int64, inputLimit int, chrome int) string { pattern := make([]string, 0, len(scriptForbidden)) for _, word := range scriptForbidden { pattern = append(pattern, regexp.QuoteMeta(word)) @@ -84,12 +85,11 @@ const keys = Object.keys, names = Object.getOwnPropertyNames, proto = Object.get const apply = Reflect.apply, stringify = JSON.stringify, parse = JSON.parse, test = RegExp.prototype.test, then = Promise.prototype.then; const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis.cua; const bad = /`+strings.Join(pattern, "|")+`/; -const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome.PID)+`, own = `+strconv.FormatFloat(chrome.Window, 'f', -1, 64)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; -const plainKey = (key, token, win) => typeof key === "string" && key.indexOf("+") < 0 && typeof token === "string" && own !== 0 && win === own; -const keyed = (k, c, o) => { if (chrome === 0) return false; -if (k === "call") { if (c[0] !== "hotkey" && c[0] !== "type_text" && c[0] !== "press_key") return false; const a = c[1] !== null && typeof c[1] === "object" ? c[1] : {}; -if (typeof a.pid !== "number") return true; if (a.pid !== chrome) return false; return !(c[0] === "press_key" && a.modifiers === undefined && plainKey(a.key, a.element_token, a.window_id)); } -if (k !== "hotkey" && k !== "typeText" && k !== "pressKey") return false; if (o.pid !== chrome) return false; return !(k === "pressKey" && c.length === 2 && plainKey(c[0], c[1], o.windowId)); }; +const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; +const keyOps = {__proto__: null, hotkey: true, type_text: true, press_key: true}, keyMethods = {__proto__: null, hotkey: true, typeText: true, pressKey: true}; +const keyed = (k, c, o) => chrome > 0 && (k === "call" ? keyOps[c[0]] === true && (c[1] === null || typeof c[1] !== "object" || typeof c[1].pid !== "number" || c[1].pid === chrome) : keyMethods[k] === true && o.pid === chrome); +const namesChrome = v => v === "Google Chrome" || v === "com.google.Chrome" || v !== null && typeof v === "object" && (v.name === "Google Chrome" || v.bundle_id === "com.google.Chrome" || v.bundleId === "com.google.Chrome"); +const startsChrome = (k, c) => k === "launchApp" && namesChrome(c[0]) || k === "call" && c[0] === "launch_app" && namesChrome(c[1]); const inputMethods = {__proto__: null, click: true, doubleClick: true, rightClick: true, drag: true, scroll: true, typeText: true, pressKey: true, hotkey: true, setValue: true}; const inputOps = {__proto__: null, click: true, double_click: true, right_click: true, drag: true, scroll: true, type_text: true, press_key: true, hotkey: true, set_value: true}; const input = (k, c) => inputMethods[k] === true || k === "call" && typeof c[0] === "string" && inputOps[c[0]] === true; @@ -106,6 +106,7 @@ if (limit > 0 && k === "typeText") { const given = c.length > 2 && c[2] !== null if (limit > 0 && k === "sleep" && o === real && typeof c[0] === "number" && c[0] > `+strconv.Itoa(scriptSleepLimit)+`) { const t = {length: 1}; define(t, 0, {value: `+strconv.Itoa(scriptSleepLimit)+`}); c = t; } const run = () => { const r = apply(v, o, c); return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [out]) : out(r); }; if (limit > 0 && (k === "launchApp" || k === "call" && c[0] === "launch_app")) throw new Error("COMPUTER_SCRIPT_REFUSED: a script in the Computer's own browser opens no app or URL, so the site checks of launch_app hold; call launch_app as its own tool"); +if (startsChrome(k, c)) throw new Error("COMPUTER_SCRIPT_REFUSED: a script does not start Google Chrome; call launch_app as its own tool"); if (keyed(k, c, o)) throw new Error(`+strconv.Quote(everydayChromeKeysRefused)+`); if (limit <= 0 || !input(k, c)) return run(); if (++sent > limit) throw new Error("COMPUTER_SCRIPT_INPUT_LIMIT: a script sends at most " + limit + " inputs to the Computer's own browser; return what you have, look at the window, then continue in a new script"); @@ -145,7 +146,7 @@ func forbiddenKey(value any, word string) bool { // into what remains of the call after waiting for the desktop lane. In a // reviewed Run, whose review scope is not empty, a script file runs only as // the judge allowed it in that scope: the body it read and the args it saw. -func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope string, inputLimit int, chrome sharedWindow) error { +func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope string, inputLimit int, chrome int) error { for key := range arguments { switch key { case "script", "script_file", "args", "timeout_ms", "max_calls": diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 361045f9..970663b8 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -55,7 +55,7 @@ func TestScriptGuardRefusesForbiddenInputInAJSEngine(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{"n":1}`, tc.body, 42, 0, sharedWindow{})), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{"n":1}`, tc.body, 42, 0, 0)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -84,7 +84,7 @@ func TestScriptGuardPacesInputsForTheComputersChrome(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 2, sharedWindow{})), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 2, 0)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -112,7 +112,7 @@ func TestScriptGuardSpeedsTypingAndCapsWaits(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 20, sharedWindow{})), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, 20, 0)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -145,7 +145,7 @@ func TestScriptGuardRefusesLaunchAppForTheComputersChrome(t *testing.T) { want string }{{2, "threw:COMPUTER_SCRIPT_REFUSED"}, {0, "call:launch_app"}} { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, tc.limit, sharedWindow{})), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, tc.limit, 0)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() @@ -170,7 +170,8 @@ func TestScriptGuardRefusesKeysForTheEverydayChrome(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - // The user's Chrome runs as pid 1, and the Computer's window there is 12. + // The user's Chrome runs as pid 1. A script cannot tell which window a + // token was read from, so it sends that Chrome no key at all. for _, tc := range []struct{ body, want string }{ {`return await cua.call("hotkey", {pid: 1, window_id: 12, keys: ["cmd", "["]});`, "threw:COMPUTER_KEYS_REFUSED"}, {`return await cua.call("hotkey", {keys: ["cmd", "w"]});`, "threw:COMPUTER_KEYS_REFUSED"}, @@ -178,21 +179,14 @@ func TestScriptGuardRefusesKeysForTheEverydayChrome(t *testing.T) { {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.typeText("kettle", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, {`const app = await cua.getApp({pid: 1, windowId: 9}); return app.hotkey(["cmd", "w"]);`, "threw:COMPUTER_KEYS_REFUSED"}, {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.hotkey(["cmd", "s"]);`, "hotkeyed"}, - {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return"});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return", element_token: "s1:2", modifiers: ["cmd"]});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return", element_token: "s1:2", modifiers: []});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "cmd+r", element_token: "s1:2"});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, window_id: 9, key: "return", element_token: "s1:2"});`, "threw:COMPUTER_KEYS_REFUSED"}, - {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return", element_token: "s1:2"});`, "call:press_key"}, - {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("cmd+r", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("return");`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("return", [3, 4]);`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1, windowId: 9}); return app.pressKey("return", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, - {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("return", "s1:2");`, "pressed"}, + {`return await cua.call("press_key", {pid: 1, window_id: 12, key: "return", element_token: "s1:2"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 1, windowId: 12}); return app.pressKey("return", "s1:2");`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.pressKey("return", "s1:2");`, "pressed"}, {`return await cua.call("click", {pid: 1, element_token: "s1:2"});`, "call:click"}, + {`return await cua.call("launch_app", {bundle_id: "com.google.Chrome"});`, "threw:COMPUTER_SCRIPT_REFUSED"}, } { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, 20, sharedWindow{PID: 1, Window: 12})), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, 20, 1)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go index 68c9ff1b..850abd6a 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_test.go @@ -16,22 +16,22 @@ import ( func TestScriptArgumentsBoundTheScript(t *testing.T) { arguments := map[string]any{"script": "return 1;"} - if err := scriptArguments(context.Background(), arguments, "", 0, sharedWindow{}); err != nil || arguments["timeout_ms"] != float64(scriptTimeoutMs) { + if err := scriptArguments(context.Background(), arguments, "", 0, 0); err != nil || arguments["timeout_ms"] != float64(scriptTimeoutMs) { t.Fatalf("default: %v %v", err, arguments) } arguments = map[string]any{"script": "return 1;", "timeout_ms": float64(500000)} - if err := scriptArguments(context.Background(), arguments, "", 0, sharedWindow{}); err != nil || arguments["timeout_ms"] != float64(scriptMaxTimeoutMs) { + if err := scriptArguments(context.Background(), arguments, "", 0, 0); err != nil || arguments["timeout_ms"] != float64(scriptMaxTimeoutMs) { t.Fatalf("cap: %v %v", err, arguments) } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() arguments = map[string]any{"script": "return 1;", "timeout_ms": float64(90000)} - if err := scriptArguments(ctx, arguments, "", 0, sharedWindow{}); err != nil || arguments["timeout_ms"].(float64) > 45000 { + if err := scriptArguments(ctx, arguments, "", 0, 0); err != nil || arguments["timeout_ms"].(float64) > 45000 { t.Fatalf("deadline: %v %v", err, arguments) } late, stop := context.WithTimeout(context.Background(), 10*time.Second) defer stop() - if err := scriptArguments(late, map[string]any{"script": "return 1;"}, "", 0, sharedWindow{}); !errors.Is(err, errNotSent) { + if err := scriptArguments(late, map[string]any{"script": "return 1;"}, "", 0, 0); !errors.Is(err, errNotSent) { t.Fatalf("late: %v", err) } for _, refused := range []map[string]any{ @@ -40,7 +40,7 @@ func TestScriptArgumentsBoundTheScript(t *testing.T) { {"script": `await cua.call("click", {pid: 1, x: 2, y: 3, delivery_mode: "foreground"})`}, {"script": `await app.getState({screenshot_out_file: "/tmp/a.png"})`}, } { - if err := scriptArguments(context.Background(), refused, "", 0, sharedWindow{}); !errors.Is(err, errNotSent) { + if err := scriptArguments(context.Background(), refused, "", 0, 0); !errors.Is(err, errNotSent) { t.Fatalf("%v: got %v", refused, err) } } @@ -56,7 +56,7 @@ func TestScriptArgumentsReadASkillScriptWithArgs(t *testing.T) { t.Fatal(err) } arguments := map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}} - if err := scriptArguments(context.Background(), arguments, "", 0, sharedWindow{}); err != nil { + if err := scriptArguments(context.Background(), arguments, "", 0, 0); err != nil { t.Fatal(err) } script, _ := arguments["script"].(string) @@ -70,19 +70,19 @@ func TestScriptArgumentsReadASkillScriptWithArgs(t *testing.T) { {"script": `return cua.call("click", {delivery_mode: "foreground"});`}, {"script_file": file, "args": map[string]any{"click": []any{map[string]any{"delivery_mode": "foreground"}}}}, } { - if err := scriptArguments(context.Background(), refused, "", 0, sharedWindow{}); !errors.Is(err, errNotSent) { + if err := scriptArguments(context.Background(), refused, "", 0, 0); !errors.Is(err, errNotSent) { t.Fatalf("%v: got %v", refused, err) } } // args carry page and user text, which may name a forbidden option. - if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"text": "in the foreground"}}, "", 0, sharedWindow{}); err != nil { + if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"text": "in the foreground"}}, "", 0, 0); err != nil { t.Fatalf("user text in args: %v", err) } // A reviewed Run runs a script file only as a review allowed it. - if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}}, permission.ScriptScope("agent-1", "context-1"), 0, sharedWindow{}); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_SCRIPT_REFUSED") { + if err := scriptArguments(context.Background(), map[string]any{"script_file": file, "args": map[string]any{"pid": float64(7)}}, permission.ScriptScope("agent-1", "context-1"), 0, 0); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_SCRIPT_REFUSED") { t.Fatalf("unreviewed script file in a reviewed Run: %v", err) } - if err := scriptArguments(context.Background(), map[string]any{"script": "return 1;"}, permission.ScriptScope("agent-1", "context-1"), 0, sharedWindow{}); err != nil { + if err := scriptArguments(context.Background(), map[string]any{"script": "return 1;"}, permission.ScriptScope("agent-1", "context-1"), 0, 0); err != nil { t.Fatalf("inline script in a reviewed Run: %v", err) } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 6b87ebd9..683fe91f 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1983,12 +1983,12 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t t.Fatalf("set_value: %v", err) } // Any script, whatever it targets, is guarded against keys to the - // user's Chrome, with the Computer's window 12 as the one exception. + // user's Chrome. for _, script := range []string{`{"script":"return 1"}`, `{"script":"return 1","args":{"pid":42,"window_id":9}}`, `{"script":"return 1","args":{"pid":7}}`} { if _, err := c.call(t.Context(), "run_script", json.RawMessage(script)); err != nil { t.Fatal(err) } - if calls := fixtureCalls(t, marker); !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "chrome = 42, own = 12") { + if calls := fixtureCalls(t, marker); !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "chrome = 42,") { t.Fatalf("%s: last call=%v, want a script guarded against keys", script, calls[len(calls)-1]) } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/manager.go b/apps/computer/local-daemon/internal/linger/localtools/manager.go index 578e1788..9ea9d4f6 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/manager.go +++ b/apps/computer/local-daemon/internal/linger/localtools/manager.go @@ -122,7 +122,9 @@ func New(options Options) (*Manager, error) { if err := m.openBrowserJournals(); err != nil { return nil, err } - retireAgentChromeProfile(options.StateDirectory) + // Quitting a Chrome still on the old profile may take seconds, so + // it does not hold up the start. + go retireAgentChromeProfile(options.StateDirectory) } return m, nil } From e8ab40afa979c0b443f18cb4c0603f4599757d84 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 11:24:27 +0800 Subject: [PATCH 06/30] fix(computer): refuse script keys through an app the guard cannot place A key sent through an app object whose pid the script guard cannot read could reach the user's Chrome, so it is refused while that Chrome runs, as a key that names no app already is. The humanlike-browse Skill says the one allowed Return goes as its own press_key call, never inside run_script. Co-Authored-By: Claude Code --- .../local-daemon/internal/linger/localtools/cua_script.go | 2 +- .../internal/linger/localtools/cua_script_guard_test.go | 3 +++ skills/builtin/humanlike-browse/1.0.5/SKILL.md | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index 460938d6..bf746fb0 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -87,7 +87,7 @@ const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis const bad = /`+strings.Join(pattern, "|")+`/; const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; const keyOps = {__proto__: null, hotkey: true, type_text: true, press_key: true}, keyMethods = {__proto__: null, hotkey: true, typeText: true, pressKey: true}; -const keyed = (k, c, o) => chrome > 0 && (k === "call" ? keyOps[c[0]] === true && (c[1] === null || typeof c[1] !== "object" || typeof c[1].pid !== "number" || c[1].pid === chrome) : keyMethods[k] === true && o.pid === chrome); +const keyed = (k, c, o) => chrome > 0 && (k === "call" ? keyOps[c[0]] === true && (c[1] === null || typeof c[1] !== "object" || typeof c[1].pid !== "number" || c[1].pid === chrome) : keyMethods[k] === true && (typeof o.pid !== "number" || o.pid === chrome)); const namesChrome = v => v === "Google Chrome" || v === "com.google.Chrome" || v !== null && typeof v === "object" && (v.name === "Google Chrome" || v.bundle_id === "com.google.Chrome" || v.bundleId === "com.google.Chrome"); const startsChrome = (k, c) => k === "launchApp" && namesChrome(c[0]) || k === "call" && c[0] === "launch_app" && namesChrome(c[1]); const inputMethods = {__proto__: null, click: true, doubleClick: true, rightClick: true, drag: true, scroll: true, typeText: true, pressKey: true, hotkey: true, setValue: true}; diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 970663b8..259c9a5f 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -184,6 +184,9 @@ func TestScriptGuardRefusesKeysForTheEverydayChrome(t *testing.T) { {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.pressKey("return", "s1:2");`, "pressed"}, {`return await cua.call("click", {pid: 1, element_token: "s1:2"});`, "call:click"}, {`return await cua.call("launch_app", {bundle_id: "com.google.Chrome"});`, "threw:COMPUTER_SCRIPT_REFUSED"}, + // An app object whose pid the guard cannot read could be that Chrome. + {`const apps = await cua.listApps(); return apps[0].hotkey(["cmd", "w"]);`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({name: "Google Chrome"}); return app.hotkey(["cmd", "w"]);`, "threw:COMPUTER_KEYS_REFUSED"}, } { script := filepath.Join(dir, "script.js") if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, 20, 1)), 0o600); err != nil { diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md index 6f2aa03e..837c46ef 100644 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -32,7 +32,7 @@ The user watches the window, and each turn of yours costs seconds. The Computer - Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. - Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. - Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Edge and Firefox. -- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; only Return bound to the search box of the Computer's window goes, and only while no other Chrome window could take it. +- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; only Return bound to the search box of the Computer's window goes, as its own `press_key` call and never inside `run_script`, and only while no other Chrome window could take it. - After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. - Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. From 48a700ec7d93a243b79630879dc625587cd3abb1 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 12:11:52 +0800 Subject: [PATCH 07/30] fix(computer): close script key reroutes and keep every Computer window in the user's Chrome A script key whose arguments name an app or window of their own is refused, since Cua lets them override the app the call is made on, and no script opens an app or URL, so the key rule knows every app a script reaches. Every window the Computer opened in a browser stays its own for pacing and the key rule. press_key takes element_token, as the one allowed key needs. Chrome's arguments are read only for its own processes, and stale comments, dead fields and docs are aligned. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 12 +- .../linger/localtools/agent_browser.go | 104 +++++++++++------- .../linger/localtools/agent_browser_test.go | 19 ++++ .../linger/localtools/agent_chrome.go | 4 +- .../internal/linger/localtools/cua.go | 4 +- .../internal/linger/localtools/cua_script.go | 26 +++-- .../localtools/cua_script_guard_test.go | 30 +++-- .../linger/localtools/localtools_test.go | 14 ++- .../builtin/humanlike-browse/1.0.5/SKILL.md | 2 +- 9 files changed, 138 insertions(+), 77 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 0464be84..e2f4c423 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -340,8 +340,10 @@ restart keeps it, and removes the record once Chrome started. Before the Computer opens its window, it waits until the list of Chrome windows stops changing, so every restored window is there. The Computer's window is the new window with one tab: the one whose address bar shows the site just opened, or -the only new window when the home page moved to another site. Two new windows -of the site are refused with `COMPUTER_WINDOW_AMBIGUOUS`. The address bar and +the only new window when the home page moved to another site. Two or more new +one-tab windows that the site does not tell apart are refused with +`COMPUTER_WINDOW_AMBIGUOUS`. Every window the Computer opened there stays its +own for pacing and the key rule, until Chrome restarts. The address bar and the tabs are found by role and place in the window, not by their names, which follow the user's language. The Daemon finds the running Chrome by its executable wherever the app is installed. The user's Chrome takes no keys from @@ -353,7 +355,9 @@ types text as keys that all read as the A key. A key goes to the window its that Chrome runs. Only a plain key bound to an element of the Computer's window goes, as a tool call, and Cua sends it only when no other Chrome window could take it. A script sends that Chrome no key at all, because it cannot -prove which window a token was read from, and it does not start Chrome. +prove which window a token was read from. It refuses a key whose arguments +name an app or window of their own, and no script opens an app or URL, so the +key rule knows every app a script reaches. The Agent fills fields with `set_value`, presses the page's buttons and goes back through its links. The Daemon removes the separate Chrome profile `computer-tools/agent-chrome` @@ -384,7 +388,7 @@ into a script, to the current snapshot when its row reads the same there. Only the window the task opened with `launch_app` is held. A script that names no window in its `args` is taken to drive that window; a script to another app is neither -paced nor held, and a paced script may not call `launch_app`. The browser +paced nor held, and no script may open an app or URL. The browser window is the largest window whose tab strip reads: a starting Chrome can list a larger surface without an accessibility window, which is skipped. The `launch_app` result tells the Agent to move between pages through the site's diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index ee886e8c..c1dad55a 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -18,9 +18,10 @@ import ( "github.com/simplexai-labs/linger/apps/computer/local-daemon/internal/platform/privatefs" ) -// agentBrowser is one browser the Computer can run as its own: a separate -// instance on a profile of its own, with no remote debugging, so sites see a -// person's browser and the user's everyday browser stays untouched. +// agentBrowser is one browser the Computer browses watched sites in, with no +// remote debugging, so sites see a person's browser: a window of its own in +// the user's browser (Shared), or a separate instance on a profile of its +// own. type agentBrowser struct { // Name is how messages and the model name the browser. Name string @@ -31,7 +32,8 @@ type agentBrowser struct { // Args start the browser on profile, after open's --args. Args func(profile string) []string // Settings are what a running instance's command line must carry for - // the tools to rely on it; one without them is restarted. + // the tools to rely on it. The Computer restarts its own instance that + // lacks them; the user quits their everyday Chrome. Settings []string // AddressRole and AddressLabel find the address bar; an empty label // takes the first element with that role, which in the browser's @@ -102,7 +104,7 @@ var agentBrowsers = []agentBrowser{ // device trust: sites that watch for automated browsing trust that // browser most, and the user signs in only once. {Name: "Chrome", Bundle: "com.google.Chrome", App: "Google Chrome.app", Settings: everydayChromeArgs, - AddressRole: "AXTextField", AddressLabel: "Address and search bar", Shared: true, Everyday: true}, + AddressRole: "AXTextField", Shared: true, Everyday: true}, {Name: "Edge", Bundle: "com.microsoft.edgemac", App: "Microsoft Edge.app", Directory: "agent-edge", Args: chromium, Settings: append(slices.Clone(chromiumArgs), "-AppleLanguages"), AddressRole: "AXTextField", AddressLabel: "Address and search bar"}, // Firefox keeps its window working while covered without a flag. Its @@ -176,9 +178,10 @@ func (b agentBrowser) startedWith(command string) bool { // Safari: the window this task or an earlier one opened there, or a new one. // Each URL opens in a new tab of that window, as in the other browsers, and // the window's first tab stays as its spare tab. -// agentSafariWindowWait bounds the wait for a new Safari window to become +// sharedBrowserWindowWait bounds the wait for a new window of the user's +// Safari or Chrome to become // readable. -var agentSafariWindowWait = 30 * time.Second +var sharedBrowserWindowWait = 30 * time.Second func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls []string) (json.RawMessage, error) { m := c.op.manager @@ -194,7 +197,7 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls return nil, err } m.mu.Lock() - own := m.sharedWindows[b.Bundle].Window + own := m.sharedWindows[b.Bundle].latest() if m.sharedWindows[b.Bundle].PID != pid { own = 0 } @@ -236,7 +239,7 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls if started { // Safari can take longer than a Chromium start before the new // window's accessibility tree resolves; opening a tab waits for it. - c.agentChromeTabsUntil(ctx, pid, window.WindowID, agentSafariWindowWait, func(n int) bool { return n > 0 }) + c.agentChromeTabsUntil(ctx, pid, window.WindowID, sharedBrowserWindowWait, func(n int) bool { return n > 0 }) } m.setSharedWindow(b.Bundle, pid, window.WindowID) for _, url := range urls { @@ -298,22 +301,22 @@ var everydayChromePID = func(ctx context.Context) (int, string, error) { if err != nil { return 0, "", err } - lines, err := exec.CommandContext(ctx, "/bin/ps", "-axww", "-o", "pid=,args=").Output() - if err != nil { - return 0, "", err - } - commands := map[string]string{} - for _, line := range strings.Split(string(lines), "\n") { - pidText, args, _ := strings.Cut(strings.TrimSpace(line), " ") - commands[pidText] = args - } for _, line := range strings.Split(string(paths), "\n") { pidText, path, _ := strings.Cut(strings.TrimSpace(line), " ") - if !everydayChromeProcess(path, commands[pidText]) { + if !strings.HasPrefix(path, "/") || !strings.HasSuffix(path, everydayChromeExecutable) { + continue + } + // Only Chrome's own processes need their arguments read. + args, err := exec.CommandContext(ctx, "/bin/ps", "-ww", "-o", "args=", "-p", pidText).Output() + if err != nil { + continue // it ended between the two reads + } + command := strings.TrimSpace(string(args)) + if !everydayChromeProcess(path, command) { continue } if pid, err := strconv.Atoi(pidText); err == nil { - return pid, commands[pidText], nil + return pid, command, nil } } return 0, "", nil @@ -430,7 +433,7 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u // finds its window. Windows that stay ambiguous are refused. func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, pid int, before map[float64]bool, site string) (listedWindow, int, error) { var seen time.Time - for deadline := time.Now().Add(agentSafariWindowWait); ; { + for deadline := time.Now().Add(sharedBrowserWindowWait); ; { windows, err := c.appWindows(ctx, pid) if err != nil { return listedWindow{}, 0, err @@ -454,11 +457,11 @@ func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, p return matching[0], 1, nil case len(matching) == 0 && len(fresh) == 1 && settled: return fresh[0], 1, nil - case len(fresh) > 1 && settled && len(matching) != 1, len(fresh) > 1 && time.Now().After(deadline): + case len(fresh) > 1 && (settled || time.Now().After(deadline)): return listedWindow{}, 0, fmt.Errorf("COMPUTER_WINDOW_AMBIGUOUS: %d new one-tab windows of the user's Chrome could be the one the Computer opened, so it works in none of them; ask the user to close the windows they do not need, then call launch_app again", len(fresh)) } if time.Now().After(deadline) { - return listedWindow{}, 0, errors.New("launch_app: the user's Chrome did not open a window for the Computer") + return listedWindow{}, 0, fmt.Errorf("launch_app: the user's Chrome showed no readable new window for %s; a window it opened may still be there, so ask the user to close it before trying again", site) } if err := sleepContext(ctx, 200*time.Millisecond); err != nil { return listedWindow{}, 0, err @@ -466,9 +469,11 @@ func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, p } } -// everydayChromeSettle is how long a window list of the user's Chrome must -// stay the same before the Computer trusts it: Chrome reopens restored -// windows one after another. +// everydayChromeSettle is how long the Computer lets the user's Chrome +// settle before it trusts what it sees: the window list must stay the same +// that long after a start, since Chrome reopens restored windows one after +// another, and a new window must have stood that long before it is taken +// without its site. var everydayChromeSettle = 1500 * time.Millisecond // windowAddress returns the web address the address bar of window shows, @@ -510,7 +515,7 @@ func (c *cuaSession) windowAddress(ctx context.Context, b agentBrowser, pid int, // restores is there before the Computer opens its own, and returns its pid. func (c *cuaSession) waitEverydayChrome(ctx context.Context) (int, error) { last, since := "", time.Now() - for deadline := time.Now().Add(agentSafariWindowWait); ; { + for deadline := time.Now().Add(sharedBrowserWindowWait); ; { pid, _, err := everydayChromePID(ctx) if err != nil { return 0, err @@ -563,25 +568,40 @@ func (m *Manager) quitAsked() bool { return err == nil } -// sharedWindow is the window the Computer opened as its own in one of the -// user's own browsers, and that browser's pid. +// sharedWindow is the windows the Computer opened as its own in one of the +// user's browsers, latest last, and that browser's pid. A task may open +// several, one site each, and every one stays the Computer's. type sharedWindow struct { - PID int - Window float64 + PID int + Windows []float64 +} + +// latest is the window the Computer opened last, or 0. +func (s sharedWindow) latest() float64 { + if len(s.Windows) == 0 { + return 0 + } + return s.Windows[len(s.Windows)-1] } -// setSharedWindow records the Computer's window in the user's browser -// bundle. Callers do not hold m.mu. +// setSharedWindow records window as the Computer's in the user's browser +// bundle running as pid; a new pid starts a new record. Callers do not hold +// m.mu. func (m *Manager) setSharedWindow(bundle string, pid int, window float64) { m.mu.Lock() defer m.mu.Unlock() if m.sharedWindows == nil { m.sharedWindows = map[string]sharedWindow{} } - m.sharedWindows[bundle] = sharedWindow{PID: pid, Window: window} + shared := m.sharedWindows[bundle] + if shared.PID != pid { + shared = sharedWindow{PID: pid} + } + shared.Windows = append(slices.DeleteFunc(slices.Clone(shared.Windows), func(w float64) bool { return w == window }), window) + m.sharedWindows[bundle] = shared } -// sharedWindowOf returns the Computer's window in the user's browser that +// sharedWindowOf returns the Computer's windows in the user's browser that // runs as pid. Callers do not hold m.mu. func (m *Manager) sharedWindowOf(pid int) (sharedWindow, bool) { m.mu.Lock() @@ -595,8 +615,8 @@ func (m *Manager) sharedWindowOf(pid int) (sharedWindow, bool) { } // userChromeWindow returns the pid of the user's everyday Chrome, or 0 when -// it does not run, and the window the Computer opened as its own there, or -// 0. Chrome is looked up on each call, so the key rule holds before the +// it does not run, and the windows the Computer opened as its own there. +// Chrome is looked up on each call, so the key rule holds before the // Computer opened a window and after Chrome restarts. func (c *cuaSession) userChromeWindow(ctx context.Context) (sharedWindow, error) { if runtime.GOOS != "darwin" { @@ -611,7 +631,7 @@ func (c *cuaSession) userChromeWindow(ctx context.Context) (sharedWindow, error) own := m.sharedWindows["com.google.Chrome"] m.mu.Unlock() if pid == 0 || own.PID != pid { - own.Window = 0 + own.Windows = nil } own.PID = pid return own, nil @@ -629,7 +649,9 @@ func (c *cuaSession) userChromeKeys(ctx context.Context, name string, arguments pid, window := arguments["pid"], arguments["window_id"] if token, ok := arguments["element_token"].(string); ok { // A token the Computer did not see read cannot be placed in a - // window, so it is not taken for one in the Computer's window. + // window, so it is not taken for one in the Computer's window. Cua + // refuses a token read from another pid than the one named, so a + // token cannot carry a key into another app. window = nil if key, found := c.op.reads.window(token); found { pid, window = float64(key.pid), float64(key.window) @@ -645,8 +667,8 @@ func (c *cuaSession) userChromeKeys(ctx context.Context, name string, arguments if paged { return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) } - id, _ := window.(float64) - return everydayChromeKeys(name, arguments, chrome.Window != 0 && id == chrome.Window) + id, named := window.(float64) + return everydayChromeKeys(name, arguments, named && slices.Contains(chrome.Windows, id)) } // everydayChromeKeysRefused tells the Agent how to work without keys in the diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go index 6c8208de..251334b8 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go @@ -90,3 +90,22 @@ func TestEverydayChromeKeysAllowOnlyOnePlainBoundKey(t *testing.T) { t.Error("a bound Return went to one of the user's windows") } } + +func TestSharedWindowsKeepEveryWindowOfTheBrowser(t *testing.T) { + m := &Manager{} + m.setSharedWindow("com.google.Chrome", 42, 11) + m.setSharedWindow("com.google.Chrome", 42, 12) + m.setSharedWindow("com.apple.Safari", 7, 3) + shared, ok := m.sharedWindowOf(42) + if !ok || len(shared.Windows) != 2 || shared.latest() != 12 { + t.Fatalf("shared=%+v ok=%v, want windows 11 and 12 with 12 latest", shared, ok) + } + // A restarted Chrome has a new pid, and none of the old windows. + m.setSharedWindow("com.google.Chrome", 43, 20) + if shared, _ := m.sharedWindowOf(43); len(shared.Windows) != 1 || shared.latest() != 20 { + t.Fatalf("shared=%+v, want only window 20 after the restart", shared) + } + if safari, ok := m.sharedWindowOf(7); !ok || safari.latest() != 3 { + t.Fatalf("safari=%+v ok=%v, want Safari's window kept apart", safari, ok) + } +} diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 069f6291..ff2cc341 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -431,7 +431,7 @@ func (c *cuaSession) isAgentChrome(ctx context.Context, pid, window any) (bool, m.mu.Unlock() if shared, ok := m.sharedWindowOf(int(number)); ok { id, named := window.(float64) - return !named || id == shared.Window, nil + return !named || slices.Contains(shared.Windows, id), nil } if known > 0 && known == int(number) && time.Since(checked) < agentChromeRecheck { return true, nil @@ -554,7 +554,7 @@ func (c *cuaSession) agentChromeKeys(ctx context.Context, arguments map[string]a } if pid, ok := arguments["pid"].(float64); ok { if shared, found := c.op.manager.sharedWindowOf(int(pid)); found { - arguments["window_id"] = shared.Window + arguments["window_id"] = shared.latest() return nil } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 8a2fc9d0..1f094656 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -223,7 +223,7 @@ const MacToolGuide = `Computer tools on this Mac (MCP server "computer"): - report_blocker({kind, url}) as soon as a page in the Computer's own browser shows a check only the user may pass (captcha, verification, login_required, rate_limited) or one item that cannot be viewed (content_unavailable). Never try to pass the check. The Computer's own browser takes input at a person's pace, within a budget per task and per day, and a script there sends at most 20 inputs: return what you have and look at the window before the next script. A tab that a link in the script opens is the task's and is in front afterwards. The browser tools stay off every site a task opened in that browser. Plan as many steps as you can in one run_actions call, as in one script; its read shows their effect, so do not read again before the next input unless you need another window. Keys are often shorter than clicks: in Finder, cmd+shift+g then a path and Return opens a folder, cmd+down opens the selected item, and in the Trash cmd+delete puts the selected item back. -In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. The user's everyday Chrome takes no shortcut, typed text or unbound key in any window; only one plain key bound to an element of the Computer's window, such as Return in its search box, goes, and never from run_script. Fill fields with set_value, press the page's buttons, and go back through its links. In Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. +In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. The user's everyday Chrome takes no shortcut, typed text or unbound key in any window; only one plain key bound to an element of the Computer's window, such as Return in its search box, goes, and never from run_script. Fill fields with set_value, press the page's buttons, and go back through its links. In Safari, Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. A menu item in the read, such as File > Put Back, can be clicked directly; do not click a menu bar title first, because an open menu holds the app until it closes. A right_click menu does not show in the read, so select the row with click and use the menu bar item or its keyboard shortcut. Before the first shell screenshot, or when one fails or comes back blank, run ` + "`teamhub computer permissions status`" + `; if Screen Recording is missing for Linger, run ` + "`teamhub computer permissions request --permissions screen_recording`" + ` and end your turn — the card wakes you. Never operate System Settings or privacy prompts yourself. ` + browserToolGuide @@ -270,7 +270,7 @@ var macCompactTools = map[string]macCompactTool{ "double_click": {"Double-click an element by element_token, by role and name, or a point x,y of the window's latest screenshot." + macInputResult, macPointer, []string{"pid"}}, "right_click": {"Right-click an element by element_token, by role and name, or a point x,y, to open its context menu; read the window to see the menu." + macInputResult, macPointer, []string{"pid"}}, "hotkey": {macKeyDescriptions["hotkey"] + macKeyRouting + macInputResult, map[string]any{"pid": macPID, "keys": map[string]any{"type": "array", "items": macString}, "window_id": macWindowID}, []string{"pid", "keys"}}, - "press_key": {macKeyDescriptions["press_key"] + macKeyRouting + macInputResult, map[string]any{"pid": macPID, "key": macString, "modifiers": map[string]any{"type": "array", "items": macString}, "window_id": macWindowID}, []string{"pid", "key"}}, + "press_key": {macKeyDescriptions["press_key"] + macKeyRouting + macInputResult, map[string]any{"pid": macPID, "key": macString, "modifiers": map[string]any{"type": "array", "items": macString}, "window_id": macWindowID, "element_token": macToken}, []string{"pid", "key"}}, "type_text": {macKeyDescriptions["type_text"] + macKeyRouting + macInputResult, map[string]any{"pid": macPID, "text": macString, "window_id": macWindowID, "element_token": macToken, "role": macRole, "name": macLabel, "nth": macNth}, []string{"pid", "text"}}, "scroll": {"Scroll a window, or the element element_token, by amount lines in direction." + macInputResult, map[string]any{"pid": macPID, "window_id": macWindowID, "element_token": macToken, "role": macRole, "name": macLabel, "nth": macNth, "direction": map[string]any{"type": "string", "enum": []string{"up", "down", "left", "right"}}, "amount": map[string]any{"type": "integer"}}, []string{"pid", "direction"}}, "get_window_state": {macWindowStateDescription, map[string]any{ diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index bf746fb0..ceb73292 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -19,7 +19,7 @@ const macScriptDescription = "Run a JavaScript program that drives apps on this "- app.getState({query?, max_elements?, timeout_ms?, since?: \"latest\", tree_format?: \"elements\"}) returns {tree_markdown, window_title, snapshot_id, truncated?, elements?}; rows read as in get_window_state. In a script the walk stops after 250 nodes and 1000 ms unless you pass max_elements and timeout_ms, which cuts a browser page: pass about 3000 and 4000 there\n" + "- app.query({role?, name?}) finds named elements {element_token, role, label, frame}; app.waitFor({role?, name?, text?, gone?, timeout_ms?}) waits until the window shows (or no longer shows) them; text also matches static text\n" + "- app.click(t), doubleClick(t), typeText(text, t?), pressKey(\"escape\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, \"down\", amount?), setValue(t, value), where t is {role, name, nth?}, an element_token, a row number of this app's last getState(), or [x, y] pixels of its last screenshot\n" + - "- cua.call(tool, args) for click, press_key, type_text, scroll, get_window_state, list_windows or launch_app with their usual arguments.\n" + + "- cua.call(tool, args) for click, press_key, type_text, scroll, get_window_state or list_windows with their usual arguments. A script opens no app or URL: call launch_app as its own tool first.\n" + "Inputs go to the app in the background, as the other computer tools do; a control that reacts only to a real pointer needs the click tool with delivery_mode \"foreground\". A failed call throws (catch it to recover), and the error names the line. timeout_ms is the wall-time limit (default 60000, at most 100000); max_calls bounds driver calls (default 100, at most 500). A Skill that ships a script names it with script_file and passes its inputs as args, read in the script as the constant args. Return only what you need, such as extracted text; the result is cut at 40000 characters and carries no screenshots: call get_window_state to look at the window." // scriptTimeoutMs bounds a script below the two-minute limit of one local @@ -68,12 +68,14 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // could not verify in time. A positive inputLimit paces the script for the // Computer's own browser: the guard waits a person's pause before each input, // throws once the script tries to send more than inputLimit inputs, and -// refuses launch_app, whose site checks run only in the Daemon's own call. -// While the user's everyday Chrome runs as chrome, the guard refuses every -// key to it and every key that names no app: a script cannot prove which -// window an element token was read from, so even the one key a tool call -// may send there (see everydayChromeKeys) is refused. It also refuses to -// start Chrome, whose pid the guard would not know. +// refuses to open an app or URL, whose site checks run only in the Daemon's +// own launch_app and which could start Chrome with a pid the guard does not +// know. While the user's everyday Chrome runs as chrome, the guard refuses +// every key to it, every key that names no app, and every key whose +// arguments name an app or window of their own, which Cua lets override the +// app the call is made on: a script cannot prove which window an element +// token was read from, so even the one key a tool call may send there (see +// everydayChromeKeys) is refused. func guardedScript(args, body string, deadline int64, inputLimit int, chrome int) string { pattern := make([]string, 0, len(scriptForbidden)) for _, word := range scriptForbidden { @@ -87,9 +89,10 @@ const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis const bad = /`+strings.Join(pattern, "|")+`/; const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; const keyOps = {__proto__: null, hotkey: true, type_text: true, press_key: true}, keyMethods = {__proto__: null, hotkey: true, typeText: true, pressKey: true}; -const keyed = (k, c, o) => chrome > 0 && (k === "call" ? keyOps[c[0]] === true && (c[1] === null || typeof c[1] !== "object" || typeof c[1].pid !== "number" || c[1].pid === chrome) : keyMethods[k] === true && (typeof o.pid !== "number" || o.pid === chrome)); -const namesChrome = v => v === "Google Chrome" || v === "com.google.Chrome" || v !== null && typeof v === "object" && (v.name === "Google Chrome" || v.bundle_id === "com.google.Chrome" || v.bundleId === "com.google.Chrome"); -const startsChrome = (k, c) => k === "launchApp" && namesChrome(c[0]) || k === "call" && c[0] === "launch_app" && namesChrome(c[1]); +const routes = v => v !== null && typeof v === "object" && (v.pid !== undefined || v.app !== undefined || v.target !== undefined || v.window_id !== undefined || v.windowId !== undefined); +const rerouted = c => { for (let i = 0; i < c.length; i++) if (routes(c[i])) return true; return false; }; +const keyed = (k, c, o) => chrome > 0 && (k === "call" ? keyOps[c[0]] === true && (c[1] === null || typeof c[1] !== "object" || typeof c[1].pid !== "number" || c[1].pid === chrome || c[1].app !== undefined || c[1].target !== undefined) : keyMethods[k] === true && (typeof o.pid !== "number" || o.pid === chrome || rerouted(c))); +const launches = (k, c) => k === "launch" || k === "launchApp" || k === "call" && c[0] === "launch_app"; const inputMethods = {__proto__: null, click: true, doubleClick: true, rightClick: true, drag: true, scroll: true, typeText: true, pressKey: true, hotkey: true, setValue: true}; const inputOps = {__proto__: null, click: true, double_click: true, right_click: true, drag: true, scroll: true, type_text: true, press_key: true, hotkey: true, set_value: true}; const input = (k, c) => inputMethods[k] === true || k === "call" && typeof c[0] === "string" && inputOps[c[0]] === true; @@ -105,8 +108,7 @@ define(w, k, {enumerable: true, get() { const v = o[k]; if (typeof v !== "functi if (limit > 0 && k === "typeText") { const given = c.length > 2 && c[2] !== null && typeof c[2] === "object" ? c[2] : {}; const t = {length: 3}; define(t, 0, {value: c[0]}); define(t, 1, {value: c[1]}); define(t, 2, {value: {delay_ms: `+strconv.Itoa(agentChromeTypeDelay)+`, ...given}}); c = t; } if (limit > 0 && k === "sleep" && o === real && typeof c[0] === "number" && c[0] > `+strconv.Itoa(scriptSleepLimit)+`) { const t = {length: 1}; define(t, 0, {value: `+strconv.Itoa(scriptSleepLimit)+`}); c = t; } const run = () => { const r = apply(v, o, c); return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [out]) : out(r); }; -if (limit > 0 && (k === "launchApp" || k === "call" && c[0] === "launch_app")) throw new Error("COMPUTER_SCRIPT_REFUSED: a script in the Computer's own browser opens no app or URL, so the site checks of launch_app hold; call launch_app as its own tool"); -if (startsChrome(k, c)) throw new Error("COMPUTER_SCRIPT_REFUSED: a script does not start Google Chrome; call launch_app as its own tool"); +if (launches(k, c)) throw new Error("COMPUTER_SCRIPT_REFUSED: a script opens no app or URL, so the site checks of launch_app hold and the key rule knows every app the script reaches; call launch_app as its own tool"); if (keyed(k, c, o)) throw new Error(`+strconv.Quote(everydayChromeKeysRefused)+`); if (limit <= 0 || !input(k, c)) return run(); if (++sent > limit) throw new Error("COMPUTER_SCRIPT_INPUT_LIMIT: a script sends at most " + limit + " inputs to the Computer's own browser; return what you have, look at the window, then continue in a new script"); diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 259c9a5f..6cf195e8 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -14,8 +14,9 @@ import ( // the same prelude's semantics. const guardHarness = `import fs from "fs"; const log = []; -class App { constructor(n, pid, windowId) { this.n = n; this.pid = pid; this.windowId = windowId; } hotkey(keys) { log.push("hotkey:" + keys.join("+")); return "hotkeyed"; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } } +class App { constructor(n, pid, windowId) { this.n = n; this.pid = pid; this.windowId = windowId; } hotkey(keys, on, options) { log.push("hotkey:" + keys.join("+") + ":" + JSON.stringify({pid: this.pid, ...on, ...options})); return "hotkeyed"; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } } const fake = { call(t, a) { log.push("call:" + t + ":" + JSON.stringify(a)); return Promise.resolve({ok: true}); }, + launch(name) { log.push("launched:" + name); return Promise.resolve(new App("l", 7, 1)); }, getApp(target) { return Promise.resolve(new App("a", target && target.pid, target && target.windowId)); }, listApps() { return Promise.resolve([new App("b")]); }, box() { return Promise.resolve({inner: {app: new App("c")}}); }, @@ -127,9 +128,9 @@ func TestScriptGuardSpeedsTypingAndCapsWaits(t *testing.T) { } } -// A paced script opens no app or URL, so launch_app's site checks hold; an -// unpaced script may still call it. -func TestScriptGuardRefusesLaunchAppForTheComputersChrome(t *testing.T) { +// No script opens an app or URL, paced or not: launch_app's site checks hold, +// and the key rule knows every app the script reaches. +func TestScriptGuardRefusesLaunchApp(t *testing.T) { node, err := exec.LookPath("node") if err != nil { t.Skip("node is not installed") @@ -139,21 +140,24 @@ func TestScriptGuardRefusesLaunchAppForTheComputersChrome(t *testing.T) { if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { t.Fatal(err) } - body := `return await cua.call("launch_app", {urls: ["https://example.com/"]});` for _, tc := range []struct { + body string limit int - want string - }{{2, "threw:COMPUTER_SCRIPT_REFUSED"}, {0, "call:launch_app"}} { + }{ + {`return await cua.call("launch_app", {urls: ["https://example.com/"]});`, 2}, + {`return await cua.call("launch_app", {urls: ["https://example.com/"]});`, 0}, + {`return await cua.launch("com.google.Chrome");`, 0}, + } { script := filepath.Join(dir, "script.js") - if err := os.WriteFile(script, []byte(guardedScript(`{}`, body, 42, tc.limit, 0)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, tc.limit, 0)), 0o600); err != nil { t.Fatal(err) } out, err := exec.Command(node, harness, script).CombinedOutput() if err != nil { t.Fatalf("node: %v\n%s", err, out) } - if got := string(out); !strings.Contains(got, tc.want) || tc.limit > 0 && strings.Contains(got, "call:launch_app") { - t.Fatalf("limit %d: got %s, want %q", tc.limit, got, tc.want) + if got := string(out); !strings.Contains(got, "threw:COMPUTER_SCRIPT_REFUSED") || strings.Contains(got, "call:launch_app") || strings.Contains(got, "launched") { + t.Fatalf("%s (limit %d): got %s, want refused", tc.body, tc.limit, got) } } } @@ -184,6 +188,12 @@ func TestScriptGuardRefusesKeysForTheEverydayChrome(t *testing.T) { {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.pressKey("return", "s1:2");`, "pressed"}, {`return await cua.call("click", {pid: 1, element_token: "s1:2"});`, "call:click"}, {`return await cua.call("launch_app", {bundle_id: "com.google.Chrome"});`, "threw:COMPUTER_SCRIPT_REFUSED"}, + // Arguments that name an app or window of their own would override + // the app the key is called on. + {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.hotkey(["cmd", "w"], undefined, {pid: 1});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.pressKey("return", {app: "Google Chrome"});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`return await cua.call("press_key", {pid: 2, key: "return", target: {pid: 1}});`, "threw:COMPUTER_KEYS_REFUSED"}, + {`const app = await cua.getApp({pid: 2, windowId: 5}); return app.pressKey("return", {role: "AXTextField", name: "Search"});`, "pressed"}, // An app object whose pid the guard cannot read could be that Chrome. {`const apps = await cua.listApps(); return apps[0].hotkey(["cmd", "w"]);`, "threw:COMPUTER_KEYS_REFUSED"}, {`const app = await cua.getApp({name: "Google Chrome"}); return app.hotkey(["cmd", "w"]);`, "threw:COMPUTER_KEYS_REFUSED"}, diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 683fe91f..225b55cd 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1940,6 +1940,14 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t t.Fatalf("the user's own Chrome window %v read as the Computer's", user) } } + // Token s1 was read from the Computer's window 12, so only the rule for + // keys refuses what follows with it. + c.op.reads.mu.Lock() + if c.op.reads.snapshots == nil { + c.op.reads.windows, c.op.reads.snapshots = map[windowKey]*windowRows{}, map[string]windowKey{} + } + c.op.reads.snapshots["s1"] = windowKey{pid: 42, window: 12} + c.op.reads.mu.Unlock() // No key reaches the user's Chrome, in the Computer's window or theirs. for _, call := range []struct{ name, args string }{ {"hotkey", `{"pid":42,"window_id":12,"keys":["cmd","["]}`}, @@ -1948,7 +1956,7 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t {"press_key", `{"pid":42,"window_id":12,"key":"return"}`}, {"press_key", `{"pid":42,"window_id":12,"key":"return","element_token":"s1:1","modifiers":[]}`}, {"press_key", `{"pid":42,"window_id":12,"key":"cmd+r","element_token":"s1:1"}`}, - {"press_key", `{"pid":42,"window_id":9,"key":"return","element_token":"s1:1"}`}, + {"press_key", `{"pid":42,"window_id":9,"key":"return","element_token":"s9:1"}`}, {"recognize_text", `{"pid":42,"window_id":12,"pages":2,"next_key":"pagedown"}`}, // A token read from the user's window 9 places the key there, // whatever window_id names; one never read places it nowhere. @@ -1972,10 +1980,6 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t if _, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"keys":["cmd","w"],"element_token":"s7:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_TARGET_REQUIRED") { t.Fatalf("hotkey with no pid: err=%v, want refused", err) } - // A token read from the Computer's window 12 takes one plain key. - c.op.reads.mu.Lock() - c.op.reads.snapshots["s1"] = windowKey{pid: 42, window: 12} - c.op.reads.mu.Unlock() if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":12,"key":"return","element_token":"s1:1"}`)); err != nil { t.Fatalf("Return bound to a field: %v", err) } diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md index 837c46ef..4fd77b2f 100644 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -31,7 +31,7 @@ The user watches the window, and each turn of yours costs seconds. The Computer - Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. - Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. -- Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Edge and Firefox. +- Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Safari, Edge and Firefox. - The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; only Return bound to the search box of the Computer's window goes, as its own `press_key` call and never inside `run_script`, and only while no other Chrome window could take it. - After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. - Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. From 5a7dc5dc9ce87882bb0ede5d45f64914c2b5aee3 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 13:59:23 +0800 Subject: [PATCH 08/30] fix(computer): carry the Agent's read tokens over the Computer's own reads Every driver call fenced off the reads before it, the Agent's own get_window_state included. A token from the read the Agent had just made was then behind the fence, so the tab check before its next input staled it: a bound Return, scroll or click failed as stale_element_token. Only a call that may change the page fences now; reads fence nothing. Co-Authored-By: Claude Opus 5.5 --- .../internal/linger/localtools/cua.go | 2 +- .../linger/localtools/cua_snapshots.go | 11 ++++++-- .../linger/localtools/cua_snapshots_test.go | 28 +++++++++++++------ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 1f094656..089debe6 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -543,7 +543,7 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage driverStarted := time.Now() result, err := c.sendInput(ctx, name, arguments, appKeys, agentChrome) if c.snapshots != nil { - c.snapshots.sent() + c.snapshots.sent(name) } driverMs = time.Since(driverStarted).Milliseconds() if err == nil { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go index dd92f290..5b14ece6 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go @@ -153,13 +153,20 @@ func (s *snapshotRows) remap(arguments map[string]any) { // sent fences off every read so far: the input just sent may have changed // the page, so a token of an earlier read must not land on a row that only -// reads the same. -func (s *snapshotRows) sent() { +// reads the same. A read changes nothing, so it fences nothing: the Agent's +// own read must still carry over the Computer's reads before its next input. +func (s *snapshotRows) sent(name string) { + if readTools[name] { + return + } s.mu.Lock() s.fence = s.seq s.mu.Unlock() } +// readTools only read the screen, so the page a token names stays as it was. +var readTools = map[string]bool{"list_windows": true, "get_window_state": true, "get_desktop_state": true, "get_screen_size": true, "get_cursor_position": true, "clipboard_read": true} + // carry returns the token on the current snapshot of its window, or the // token itself. func (s *snapshotRows) carry(token string) string { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go index 19751534..7cf14f81 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go @@ -56,23 +56,33 @@ func TestSnapshotRowsCarriesTokensOverTheComputersOwnReads(t *testing.T) { t.Fatalf("script = %v, want %s", script["script"], want) } + // The Agent's own read is no input: its tokens still carry over the + // tab check that follows it. + s.sent("get_window_state") + s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000005", "pid": 7, "window_id": 9, "tree_markdown": ownTree})) + read := map[string]any{"element_token": "s00000004:4"} + s.remap(read) + if read["element_token"] != "s00000005:4" { + t.Fatalf("token of the Agent's read = %v, want s00000005:4", read["element_token"]) + } + // After an input, an older read is never carried over: the page may // have changed under a row that reads the same. - s.sent() - s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000005", "pid": 7, "window_id": 9, "tree_markdown": agentTree})) - before := map[string]any{"element_token": "s00000004:4"} + s.sent("click") + s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000006", "pid": 7, "window_id": 9, "tree_markdown": agentTree})) + before := map[string]any{"element_token": "s00000005:4"} s.remap(before) - if before["element_token"] != "s00000004:4" { + if before["element_token"] != "s00000005:4" { t.Fatalf("read before an input = %v, want its own stale token", before["element_token"]) } // A diff that says the indices shifted keeps only the rows it lists. - s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000006", "pid": 7, "window_id": 9, "since": "s00000005", "since_status": "diff", - "tree_diff": "since s00000005: 1 added.\nreindexed: rows after 3 moved by 1\n+ [5] AXLink \"Kettle\" [actions=[press]]"})) - s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000007", "pid": 7, "window_id": 9, "tree_markdown": agentTree})) - shifted := map[string]any{"element_token": "s00000006:4"} + s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000007", "pid": 7, "window_id": 9, "since": "s00000006", "since_status": "diff", + "tree_diff": "since s00000006: 1 added.\nreindexed: rows after 3 moved by 1\n+ [5] AXLink \"Kettle\" [actions=[press]]"})) + s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000008", "pid": 7, "window_id": 9, "tree_markdown": agentTree})) + shifted := map[string]any{"element_token": "s00000007:4"} s.remap(shifted) - if shifted["element_token"] != "s00000006:4" { + if shifted["element_token"] != "s00000007:4" { t.Fatalf("row of a reindexed diff = %v, want its own stale token", shifted["element_token"]) } From 7abd74c6e2360093c2a5eb91a10974e4cac3cb02 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 14:04:47 +0800 Subject: [PATCH 09/30] refactor(computer): fence snapshots by the existing read tool set Co-Authored-By: Claude Opus 5.5 --- .../local-daemon/internal/linger/localtools/cua_snapshots.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go index 5b14ece6..379acd3a 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go @@ -156,7 +156,7 @@ func (s *snapshotRows) remap(arguments map[string]any) { // reads the same. A read changes nothing, so it fences nothing: the Agent's // own read must still carry over the Computer's reads before its next input. func (s *snapshotRows) sent(name string) { - if readTools[name] { + if cuaReadTools[name] { return } s.mu.Lock() @@ -164,9 +164,6 @@ func (s *snapshotRows) sent(name string) { s.mu.Unlock() } -// readTools only read the screen, so the page a token names stays as it was. -var readTools = map[string]bool{"list_windows": true, "get_window_state": true, "get_desktop_state": true, "get_screen_size": true, "get_cursor_position": true, "clipboard_read": true} - // carry returns the token on the current snapshot of its window, or the // token itself. func (s *snapshotRows) carry(token string) string { From 64ead42fb7958c7c985b4b9b91131adcf5a45e69 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 14:11:27 +0800 Subject: [PATCH 10/30] fix(computer): send the user's Chrome no key, not even a bound Return Cua refuses a background key while its app has another window (same_pid_keyboard_ambiguity), and the user's everyday Chrome always has one: on the bench a bound Return in the Computer's Xiaohongshu window was refused that way. The one key the rule let through never arrived and only cost the Agent a turn, so every key to that Chrome is refused now, and the tool guide, README and humanlike-browse 1.0.5 say so. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 11 ++- .../linger/localtools/agent_browser.go | 80 ++++++------------- .../linger/localtools/agent_browser_test.go | 26 ------ .../internal/linger/localtools/cua.go | 8 +- .../linger/localtools/localtools_test.go | 19 +---- .../builtin/humanlike-browse/1.0.5/SKILL.md | 2 +- 6 files changed, 38 insertions(+), 108 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index e2f4c423..ffe80145 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -350,12 +350,11 @@ executable wherever the app is installed. The user's Chrome takes no keys from the Computer (`COMPUTER_KEYS_REFUSED`), in any of its windows, in tool calls, `run_actions` steps, paged `recognize_text` reads and scripts: a shortcut runs as a menu command in the Chrome window that has the keyboard, and Cua -types text as keys that all read as the A key. A key goes to the window its -`element_token` was read from, and a key that names no app is refused while -that Chrome runs. Only a plain key bound to an element of the Computer's -window goes, as a tool call, and Cua sends it only when no other Chrome window -could take it. A script sends that Chrome no key at all, because it cannot -prove which window a token was read from. It refuses a key whose arguments +types text as keys that all read as the A key. A key named by its +`element_token` goes to the app the token was read from, and a key that names +no app is refused while that Chrome runs. Not even Return bound to a search box goes: Cua refuses a +background key while its app has another window, and the user's Chrome always +has one. A script's guard refuses a key whose arguments name an app or window of their own, and no script opens an app or URL, so the key rule knows every app a script reaches. The Agent fills fields with diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index c1dad55a..0a2bc5ba 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -614,87 +614,55 @@ func (m *Manager) sharedWindowOf(pid int) (sharedWindow, bool) { return sharedWindow{}, false } -// userChromeWindow returns the pid of the user's everyday Chrome, or 0 when -// it does not run, and the windows the Computer opened as its own there. -// Chrome is looked up on each call, so the key rule holds before the -// Computer opened a window and after Chrome restarts. -func (c *cuaSession) userChromeWindow(ctx context.Context) (sharedWindow, error) { +// userChromePID returns the pid of the user's everyday Chrome, or 0 when it +// does not run. Chrome is looked up on each call, so the key rule holds before +// the Computer opened a window and after Chrome restarts. +func userChromePID(ctx context.Context) (int, error) { if runtime.GOOS != "darwin" { - return sharedWindow{}, nil + return 0, nil } pid, _, err := everydayChromePID(ctx) if err != nil { - return sharedWindow{}, fmt.Errorf("COMPUTER_INPUT_NOT_SENT: could not tell whether the user's Chrome runs: %w: %w", err, errNotSent) + return 0, fmt.Errorf("COMPUTER_INPUT_NOT_SENT: could not tell whether the user's Chrome runs: %w: %w", err, errNotSent) } - m := c.op.manager - m.mu.Lock() - own := m.sharedWindows["com.google.Chrome"] - m.mu.Unlock() - if pid == 0 || own.PID != pid { - own.Windows = nil - } - own.PID = pid - return own, nil + return pid, nil } // userChromeKeys refuses a key tool, or a paged read, that reaches the user's -// everyday Chrome. A key goes to the Chrome window that has the keyboard, so -// the window is the one the element_token was read from, not the one named, -// and a key that names no app is refused while that Chrome runs. -func (c *cuaSession) userChromeKeys(ctx context.Context, name string, arguments map[string]any, paged bool) error { - chrome, err := c.userChromeWindow(ctx) - if err != nil || chrome.PID == 0 { +// everyday Chrome, in any of its windows. A key named by its element_token +// goes to the app the token was read from, and a key that names no app is +// refused while that Chrome runs. +func (c *cuaSession) userChromeKeys(ctx context.Context, arguments map[string]any) error { + chrome, err := userChromePID(ctx) + if err != nil || chrome == 0 { return err } - pid, window := arguments["pid"], arguments["window_id"] + pid := arguments["pid"] if token, ok := arguments["element_token"].(string); ok { - // A token the Computer did not see read cannot be placed in a - // window, so it is not taken for one in the Computer's window. Cua - // refuses a token read from another pid than the one named, so a - // token cannot carry a key into another app. - window = nil + // Cua refuses a token read from another pid than the one named, so + // the token's own app is the one the key reaches. if key, found := c.op.reads.window(token); found { - pid, window = float64(key.pid), float64(key.window) + pid = float64(key.pid) } } number, ok := pid.(float64) if !ok { return fmt.Errorf("COMPUTER_TARGET_REQUIRED: while the user's Chrome runs, a key names its app by pid, so it cannot reach Chrome unseen: %w", errNotSent) } - if int(number) != chrome.PID { + if int(number) != chrome { return nil } - if paged { - return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) - } - id, named := window.(float64) - return everydayChromeKeys(name, arguments, named && slices.Contains(chrome.Windows, id)) + return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) } // everydayChromeKeysRefused tells the Agent how to work without keys in the -// user's everyday Chrome. +// user's everyday Chrome. Shortcuts are menu commands, which Chrome runs in +// the window that has the keyboard, typed text arrives as keys that all read +// as the A key, and Cua refuses even a key bound to an element while Chrome +// has another window, which the user's Chrome always has. The script guard +// (keyed in guardedScript) applies the same rule. const everydayChromeKeysRefused = "COMPUTER_KEYS_REFUSED: the user's everyday Chrome takes no keys from the Computer: a key or shortcut goes to the Chrome window that has the keyboard, which may be the one the user is typing in; nothing was sent. Fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links" -// everydayChromeKeys refuses keys to the user's everyday Chrome. Only one -// plain key bound to an element, such as Return in a search box, may go: -// Cua delivers it only when no other Chrome window could receive it. -// Shortcuts are menu commands, which Chrome runs in the window that has the -// keyboard, and typed text arrives as keys that all read as the A key. own -// says the key names the Computer's window; one of the user's windows takes -// none. The script guard (keyed in guardedScript) applies the same rule. -func everydayChromeKeys(name string, arguments map[string]any, own bool) error { - if name != "hotkey" && name != "type_text" && name != "press_key" { - return nil - } - _, bound := arguments["element_token"].(string) - _, modified := arguments["modifiers"] - key, plain := arguments["key"].(string) - if own && name == "press_key" && bound && !modified && plain && !strings.Contains(key, "+") { - return nil - } - return fmt.Errorf("%s: %w", everydayChromeKeysRefused, errNotSent) -} - // everydayChromeData is the user's Chrome data folder, whose Local State // names the profile Chrome last used. var everydayChromeData = func() string { diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go index 251334b8..343877de 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go @@ -65,32 +65,6 @@ func TestEverydayChromeProcessFindsChromeInAnyFolder(t *testing.T) { } } -func TestEverydayChromeKeysAllowOnlyOnePlainBoundKey(t *testing.T) { - for _, tc := range []struct { - name string - arguments map[string]any - allowed bool - }{ - {"press_key", map[string]any{"key": "return", "element_token": "s1:1"}, true}, - {"set_value", map[string]any{"value": "kettle", "element_token": "s1:1"}, true}, - {"press_key", map[string]any{"key": "return"}, false}, - {"press_key", map[string]any{"key": "return", "element_token": "s1:1", "modifiers": []any{}}, false}, - {"press_key", map[string]any{"key": "return", "element_token": "s1:1", "modifiers": "cmd"}, false}, - {"press_key", map[string]any{"key": "cmd+r", "element_token": "s1:1"}, false}, - {"press_key", map[string]any{"key": 13.0, "element_token": "s1:1"}, false}, - {"hotkey", map[string]any{"keys": []any{"cmd", "["}}, false}, - {"type_text", map[string]any{"text": "kettle", "element_token": "s1:1"}, false}, - } { - if err := everydayChromeKeys(tc.name, tc.arguments, true); (err == nil) != tc.allowed { - t.Errorf("%s %v: err=%v, want allowed=%v", tc.name, tc.arguments, err, tc.allowed) - } - } - // One of the user's windows takes no key at all. - if err := everydayChromeKeys("press_key", map[string]any{"key": "return", "element_token": "s1:1"}, false); err == nil { - t.Error("a bound Return went to one of the user's windows") - } -} - func TestSharedWindowsKeepEveryWindowOfTheBrowser(t *testing.T) { m := &Manager{} m.setSharedWindow("com.google.Chrome", 42, 11) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 089debe6..afc52b4a 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -223,7 +223,7 @@ const MacToolGuide = `Computer tools on this Mac (MCP server "computer"): - report_blocker({kind, url}) as soon as a page in the Computer's own browser shows a check only the user may pass (captcha, verification, login_required, rate_limited) or one item that cannot be viewed (content_unavailable). Never try to pass the check. The Computer's own browser takes input at a person's pace, within a budget per task and per day, and a script there sends at most 20 inputs: return what you have and look at the window before the next script. A tab that a link in the script opens is the task's and is in front afterwards. The browser tools stay off every site a task opened in that browser. Plan as many steps as you can in one run_actions call, as in one script; its read shows their effect, so do not read again before the next input unless you need another window. Keys are often shorter than clicks: in Finder, cmd+shift+g then a path and Return opens a folder, cmd+down opens the selected item, and in the Trash cmd+delete puts the selected item back. -In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. The user's everyday Chrome takes no shortcut, typed text or unbound key in any window; only one plain key bound to an element of the Computer's window, such as Return in its search box, goes, and never from run_script. Fill fields with set_value, press the page's buttons, and go back through its links. In Safari, Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. +In Chrome, launch_app({bundle_id: "com.google.Chrome", urls: [url]}) opens a URL in a new tab without bringing Chrome forward; its pages read as empty while the window is covered or on another Space. Add agent_browser: true to use the Computer's own browser window instead: it keeps working while covered, but not on another desktop (Space). Chrome's own toolbar, such as Back and the address bar, refuses background input. The user's everyday Chrome takes no key in any window, not even Return in a search box. Fill fields with set_value, press the page's buttons, and go back through its links. In Safari, Edge or Firefox press escape to close a page overlay, or hotkey cmd+[ to go back. A web control that ignores a background click, such as an option in a menu that opens on hover, takes one click with delivery_mode "foreground" at x,y of a screenshot. A menu item in the read, such as File > Put Back, can be clicked directly; do not click a menu bar title first, because an open menu holds the app until it closes. A right_click menu does not show in the read, so select the row with click and use the menu bar item or its keyboard shortcut. Before the first shell screenshot, or when one fails or comes back blank, run ` + "`teamhub computer permissions status`" + `; if Screen Recording is missing for Linger, run ` + "`teamhub computer permissions request --permissions screen_recording`" + ` and end your turn — the card wakes you. Never operate System Settings or privacy prompts yourself. ` + browserToolGuide @@ -398,7 +398,7 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage // A key to the user's Chrome reaches the window that has the keyboard, // whichever window it names, so it is refused before anything is taken. if name == "hotkey" || name == "type_text" || name == "press_key" || paged { - if err := c.userChromeKeys(ctx, name, arguments, paged); err != nil { + if err := c.userChromeKeys(ctx, arguments); err != nil { return nil, err } } @@ -448,11 +448,11 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage if name == "run_script" { // A script's body may reach any app, so its guard checks every key // against the user's Chrome, whatever window args names. - chrome, err := c.userChromeWindow(ctx) + chrome, err := userChromePID(ctx) if err != nil { return nil, err } - if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, chrome.PID); err != nil { + if err := scriptArguments(ctx, arguments, c.reviewScope(), scriptInputs, chrome); err != nil { return nil, err } } diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 225b55cd..8f95f44b 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1940,16 +1940,10 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t t.Fatalf("the user's own Chrome window %v read as the Computer's", user) } } - // Token s1 was read from the Computer's window 12, so only the rule for - // keys refuses what follows with it. - c.op.reads.mu.Lock() - if c.op.reads.snapshots == nil { - c.op.reads.windows, c.op.reads.snapshots = map[windowKey]*windowRows{}, map[string]windowKey{} - } - c.op.reads.snapshots["s1"] = windowKey{pid: 42, window: 12} - c.op.reads.mu.Unlock() - // No key reaches the user's Chrome, in the Computer's window or theirs. + // No key reaches the user's Chrome, in the Computer's window or theirs, + // not even Return bound to a field of the Computer's window. for _, call := range []struct{ name, args string }{ + {"press_key", `{"pid":42,"window_id":12,"key":"return","element_token":"s1:1"}`}, {"hotkey", `{"pid":42,"window_id":12,"keys":["cmd","["]}`}, {"hotkey", `{"pid":42,"window_id":9,"keys":["cmd","w"]}`}, {"type_text", `{"pid":42,"window_id":12,"text":"kettle","element_token":"s1:1"}`}, @@ -1958,9 +1952,7 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t {"press_key", `{"pid":42,"window_id":12,"key":"cmd+r","element_token":"s1:1"}`}, {"press_key", `{"pid":42,"window_id":9,"key":"return","element_token":"s9:1"}`}, {"recognize_text", `{"pid":42,"window_id":12,"pages":2,"next_key":"pagedown"}`}, - // A token read from the user's window 9 places the key there, - // whatever window_id names; one never read places it nowhere. - {"press_key", `{"pid":42,"window_id":12,"key":"return","element_token":"s9:1"}`}, + // A token never read leaves the named pid. {"press_key", `{"pid":42,"key":"return","element_token":"s7:1"}`}, } { c.op.reads.mu.Lock() @@ -1980,9 +1972,6 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t if _, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"keys":["cmd","w"],"element_token":"s7:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_TARGET_REQUIRED") { t.Fatalf("hotkey with no pid: err=%v, want refused", err) } - if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":12,"key":"return","element_token":"s1:1"}`)); err != nil { - t.Fatalf("Return bound to a field: %v", err) - } if _, err := c.call(t.Context(), "set_value", json.RawMessage(`{"pid":42,"window_id":12,"value":"kettle","element_token":"s1:1"}`)); err != nil { t.Fatalf("set_value: %v", err) } diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md index 4fd77b2f..2d5e043c 100644 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -32,7 +32,7 @@ The user watches the window, and each turn of yours costs seconds. The Computer - Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. - Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. - Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Safari, Edge and Firefox. -- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; only Return bound to the search box of the Computer's window goes, as its own `press_key` call and never inside `run_script`, and only while no other Chrome window could take it. +- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or `pressKey`, not even Return in the search box. - After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. - Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. From 6f957f823fe5bb2b3ab1fb898933652b7cd4c553 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 14:13:43 +0800 Subject: [PATCH 11/30] docs(computer): state the script guard's key rule without the retired exception Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 13 ++++++------- .../internal/linger/localtools/cua_script.go | 5 ++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index ffe80145..e4147743 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -352,13 +352,12 @@ the Computer (`COMPUTER_KEYS_REFUSED`), in any of its windows, in tool calls, runs as a menu command in the Chrome window that has the keyboard, and Cua types text as keys that all read as the A key. A key named by its `element_token` goes to the app the token was read from, and a key that names -no app is refused while that Chrome runs. Not even Return bound to a search box goes: Cua refuses a -background key while its app has another window, and the user's Chrome always -has one. A script's guard refuses a key whose arguments -name an app or window of their own, and no script opens an app or URL, so the -key rule knows every app a script reaches. -The Agent fills fields with -`set_value`, presses the page's buttons and goes back through its links. +no app is refused while that Chrome runs. Not even Return bound to a search +box goes: Cua refuses a background key while its app has another window, and +the user's Chrome always has one. A script's guard refuses a key whose +arguments name an app or window of their own, and no script opens an app or +URL, so the key rule knows every app a script reaches. The Agent fills fields +with `set_value`, presses the page's buttons and goes back through its links. The Daemon removes the separate Chrome profile `computer-tools/agent-chrome` when it starts, after quitting the Chrome that still runs on it. Edge and Firefox each run as a separate instance with their own profile under diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index ceb73292..ea7808e4 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -73,9 +73,8 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // know. While the user's everyday Chrome runs as chrome, the guard refuses // every key to it, every key that names no app, and every key whose // arguments name an app or window of their own, which Cua lets override the -// app the call is made on: a script cannot prove which window an element -// token was read from, so even the one key a tool call may send there (see -// everydayChromeKeys) is refused. +// app the call is made on. This is the rule tool calls follow: no key +// reaches the user's Chrome. func guardedScript(args, body string, deadline int64, inputLimit int, chrome int) string { pattern := make([]string, 0, len(scriptForbidden)) for _, word := range scriptForbidden { From b9de9fcc81bbbd55fa72e91e4d044abc6f4d5334 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 14:30:41 +0800 Subject: [PATCH 12/30] perf(computer): read pages in scripts without a screenshot Cua's app.getState captures a window screenshot on every read unless told not to. On the bench those reads took 6 to 17 s each on Taobao and Xiaohongshu pages, against under 1 s for the same tree without one, and made up a quarter to a third of each run. A script returns no screenshot, so the guard now passes include_screenshot: false unless the script asks for one. The script API names waitFor's 10 s limit, and humanlike-browse 1.0.5 says a screenshot read costs seconds. Co-Authored-By: Claude Opus 5.5 --- .../internal/linger/localtools/cua_script.go | 9 ++++++--- .../internal/linger/localtools/cua_script_guard_test.go | 3 ++- skills/builtin/humanlike-browse/1.0.5/SKILL.md | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index ea7808e4..d115f460 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -16,8 +16,8 @@ import ( // Computer's rules: background input only, and a bounded result. const macScriptDescription = "Run a JavaScript program that drives apps on this Mac in ONE call: loops, waits, reads and checks between inputs, and only its return value comes back. Use it to repeat a flow over several items, such as open each result, read it and close it, instead of one model turn per input. The script is the body of an async function: use await and return a JSON value. The only API is cua:\n" + "- const app = await cua.getApp({pid, windowId}) or cua.getApp(\"Google Chrome\"); cua.sleep(ms), where ms must be a whole number: a fraction returns at once without waiting, and in the Computer's own browser one sleep lasts at most 1.5 s: wait for a page with app.waitFor; cua.listWindows({pid})\n" + - "- app.getState({query?, max_elements?, timeout_ms?, since?: \"latest\", tree_format?: \"elements\"}) returns {tree_markdown, window_title, snapshot_id, truncated?, elements?}; rows read as in get_window_state. In a script the walk stops after 250 nodes and 1000 ms unless you pass max_elements and timeout_ms, which cuts a browser page: pass about 3000 and 4000 there\n" + - "- app.query({role?, name?}) finds named elements {element_token, role, label, frame}; app.waitFor({role?, name?, text?, gone?, timeout_ms?}) waits until the window shows (or no longer shows) them; text also matches static text\n" + + "- app.getState({query?, max_elements?, timeout_ms?, since?: \"latest\", tree_format?: \"elements\"}) returns {tree_markdown, window_title, snapshot_id, truncated?, elements?}; rows read as in get_window_state. It takes no screenshot, which costs seconds on a busy page, unless you pass include_screenshot: true. In a script the walk stops after 250 nodes and 1000 ms unless you pass max_elements and timeout_ms, which cuts a browser page: pass about 3000 and 4000 there\n" + + "- app.query({role?, name?}) finds named elements {element_token, role, label, frame}; app.waitFor({role?, name?, text?, gone?, timeout_ms?}) waits, at most 10000 ms, until the window shows (or no longer shows) them; text also matches static text\n" + "- app.click(t), doubleClick(t), typeText(text, t?), pressKey(\"escape\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, \"down\", amount?), setValue(t, value), where t is {role, name, nth?}, an element_token, a row number of this app's last getState(), or [x, y] pixels of its last screenshot\n" + "- cua.call(tool, args) for click, press_key, type_text, scroll, get_window_state or list_windows with their usual arguments. A script opens no app or URL: call launch_app as its own tool first.\n" + "Inputs go to the app in the background, as the other computer tools do; a control that reacts only to a real pointer needs the click tool with delivery_mode \"foreground\". A failed call throws (catch it to recover), and the error names the line. timeout_ms is the wall-time limit (default 60000, at most 100000); max_calls bounds driver calls (default 100, at most 500). A Skill that ships a script names it with script_file and passes its inputs as args, read in the script as the constant args. Return only what you need, such as extracted text; the result is cut at 40000 characters and carries no screenshots: call get_window_state to look at the window." @@ -65,7 +65,9 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // that does not compile alone could close the wrapper around it and reach the // driver directly, so it is refused. DEADLINE is when the driver stops the // script, in Unix milliseconds, so it can refuse an input whose result it -// could not verify in time. A positive inputLimit paces the script for the +// could not verify in time. A script reads text, so getState takes no +// screenshot unless the script asks for one: Cua's default captures one per +// read, which took seconds on a busy page. A positive inputLimit paces the script for the // Computer's own browser: the guard waits a person's pause before each input, // throws once the script tries to send more than inputLimit inputs, and // refuses to open an app or URL, whose site checks run only in the Daemon's @@ -105,6 +107,7 @@ const wrap = o => { const w = {}, seen = {}; for (let q = o; q && q !== plainProtos[0]; q = proto(q)) { const ns = names(q); for (let i = 0; i < ns.length; i++) { const k = ns[i]; if (k === "constructor" || seen[k]) continue; seen[k] = true; define(w, k, {enumerable: true, get() { const v = o[k]; if (typeof v !== "function") return out(v); return function (...a) { let c = {length: a.length}; for (let i = 0; i < a.length; i++) define(c, i, {value: plain(a[i])}); if (limit > 0 && k === "typeText") { const given = c.length > 2 && c[2] !== null && typeof c[2] === "object" ? c[2] : {}; const t = {length: 3}; define(t, 0, {value: c[0]}); define(t, 1, {value: c[1]}); define(t, 2, {value: {delay_ms: `+strconv.Itoa(agentChromeTypeDelay)+`, ...given}}); c = t; } +if (k === "getState" && (c.length === 0 || c[0] === undefined || c[0] !== null && typeof c[0] === "object" && c[0].include_screenshot === undefined)) { const t = {length: 1}; define(t, 0, {value: {include_screenshot: false, ...(c.length > 0 && c[0] !== undefined ? c[0] : {})}}); c = t; } if (limit > 0 && k === "sleep" && o === real && typeof c[0] === "number" && c[0] > `+strconv.Itoa(scriptSleepLimit)+`) { const t = {length: 1}; define(t, 0, {value: `+strconv.Itoa(scriptSleepLimit)+`}); c = t; } const run = () => { const r = apply(v, o, c); return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [out]) : out(r); }; if (launches(k, c)) throw new Error("COMPUTER_SCRIPT_REFUSED: a script opens no app or URL, so the site checks of launch_app hold and the key rule knows every app the script reaches; call launch_app as its own tool"); diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 6cf195e8..1e5ac204 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -14,7 +14,7 @@ import ( // the same prelude's semantics. const guardHarness = `import fs from "fs"; const log = []; -class App { constructor(n, pid, windowId) { this.n = n; this.pid = pid; this.windowId = windowId; } hotkey(keys, on, options) { log.push("hotkey:" + keys.join("+") + ":" + JSON.stringify({pid: this.pid, ...on, ...options})); return "hotkeyed"; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } } +class App { constructor(n, pid, windowId) { this.n = n; this.pid = pid; this.windowId = windowId; } hotkey(keys, on, options) { log.push("hotkey:" + keys.join("+") + ":" + JSON.stringify({pid: this.pid, ...on, ...options})); return "hotkeyed"; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } getState(options) { log.push("state:" + JSON.stringify(options)); return {tree_markdown: ""}; } } const fake = { call(t, a) { log.push("call:" + t + ":" + JSON.stringify(a)); return Promise.resolve({ok: true}); }, launch(name) { log.push("launched:" + name); return Promise.resolve(new App("l", 7, 1)); }, getApp(target) { return Promise.resolve(new App("a", target && target.pid, target && target.windowId)); }, @@ -51,6 +51,7 @@ func TestScriptGuardRefusesForbiddenInputInAJSEngine(t *testing.T) { {"closing the wrapper", "return 1; }).call(undefined); })(globalThis.cua, globalThis, eval, Function); (function () { return (async function () {", "threw:COMPUTER_SCRIPT_REFUSED: the script is not one complete function body", "call:"}, {"closing the wrapper to call", `}); await globalThis.cua.call("click", {["deliv" + "ery_mode"]: "foreground"}); (async function () {`, "threw:", "call:"}, {"inputs", `return [args.n, DEADLINE].join(",");`, `"1,42"`, ""}, + {"read without a screenshot", `const app = await cua.getApp({pid: 1}); await app.getState(); await app.getState({query: "Search"}); await app.getState({include_screenshot: true}); return "read";`, `"state:{\"include_screenshot\":false}","state:{\"include_screenshot\":false,\"query\":\"Search\"}","state:{\"include_screenshot\":true}"`, ""}, {"frozen builtins", `try { JSON.parse = () => ({delivery_mode: 1}); } catch (e) { return "refused"; } return "changed";`, "refused", ""}, } for _, tc := range cases { diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md index 2d5e043c..352bf591 100644 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -34,7 +34,7 @@ The user watches the window, and each turn of yours costs seconds. The Computer - Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Safari, Edge and Firefox. - The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or `pressKey`, not even Return in the search box. - After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. -- Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. +- Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. A read with a screenshot takes seconds on a busy page; ask for one only to see what the text does not hold. A visit to a result, in one script: From d48675803b79420c21807e7d1ba2bbcf37a8b3c1 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 15:12:55 +0800 Subject: [PATCH 13/30] fix(skills): keep humanlike-browse 1.0.5 as sealed and ship the edits as 1.0.6 An environment that installed 1.0.5 sealed it, so TeamHub refused to start once the same version's SKILL.md changed. Co-Authored-By: Claude Opus 5.5 --- skills/builtin/catalog.yaml | 5 +- .../builtin/humanlike-browse/1.0.5/SKILL.md | 8 +- .../builtin/humanlike-browse/1.0.6/SKILL.md | 80 +++++++++++++++++++ 3 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 skills/builtin/humanlike-browse/1.0.6/SKILL.md diff --git a/skills/builtin/catalog.yaml b/skills/builtin/catalog.yaml index d1a4f3ef..dab7f2ca 100644 --- a/skills/builtin/catalog.yaml +++ b/skills/builtin/catalog.yaml @@ -219,11 +219,14 @@ skills: version: 1.0.0 package_path: builtin/personal-profile/1.0.0 - skill_id: simplexai_humanlike_browse - bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_5 + bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_6 default_agent_types: - anotherme - general manifests: + - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_6 + version: 1.0.6 + package_path: builtin/humanlike-browse/1.0.6 - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_5 version: 1.0.5 package_path: builtin/humanlike-browse/1.0.5 diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md index 352bf591..8c0b3f7a 100644 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.5/SKILL.md @@ -11,7 +11,7 @@ description: Read, survey, post to or manage a site that watches for automated b - Anything else on a social network, a marketplace or a site where the user is signed in: only the computer tools, through `launch_app({agent_browser: true, urls: [url]})`. In Chrome or Safari this opens a window of the Computer's own in the user's everyday browser, with the user's sign-ins; Edge and Firefox run as separate instances. Pass `bundle_id` only when the user asks for one browser. The browser tools drive a page in a way these sites detect, so never use them on these sites; the Computer keeps them off every site the task opened this way. - If `launch_app` has no `agent_browser`, this needs a Mac Computer: tell the user and stop. - If a call says a computer tool does not exist, take the tool's exact name from your tool list and call it again before telling the user the tools are unavailable. -- If `launch_app` answers `COMPUTER_BROWSER_RESTART_REQUIRED`, ask the user with a question card to quit Chrome from its menu and leave it closed, saying that the Computer starts it again with their windows and tabs; then end your turn. When they answer that Chrome is closed, call `launch_app` again; if they decline, end the task. +- If `launch_app` answers `COMPUTER_BROWSER_RESTART_REQUIRED`, ask the user with a question card whether the Computer may restart Chrome, saying that their windows and tabs come back. Only if they agree, call `launch_app` again with `restart_browser: true`; otherwise end the task. - If the site asks to sign in, ask the user to sign in once in that browser window and stop; the sign-in stays for later tasks. ## Move like a person @@ -31,10 +31,10 @@ The user watches the window, and each turn of yours costs seconds. The Computer - Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. - Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. -- Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Safari, Edge and Firefox. -- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or `pressKey`, not even Return in the search box. +- Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Edge and Firefox. +- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; Return bound to the search box goes only while no other Chrome window could take it. - After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. -- Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. A read with a screenshot takes seconds on a busy page; ask for one only to see what the text does not hold. +- Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. A visit to a result, in one script: diff --git a/skills/builtin/humanlike-browse/1.0.6/SKILL.md b/skills/builtin/humanlike-browse/1.0.6/SKILL.md new file mode 100644 index 00000000..352bf591 --- /dev/null +++ b/skills/builtin/humanlike-browse/1.0.6/SKILL.md @@ -0,0 +1,80 @@ +--- +name: humanlike-browse +description: Read, survey, post to or manage a site that watches for automated browsing, such as a social network, a marketplace or any site where the user is signed in, through the computer tools in the Computer's own window of the user's browser. Use whenever a task reads posts, comments, profiles or search results on such a site, or writes there for the user. +--- + +# Sites that watch for automated browsing + +## Choose the route + +- A public page that needs no sign-in and no clicks: read it with the web search or fetch tools. +- Anything else on a social network, a marketplace or a site where the user is signed in: only the computer tools, through `launch_app({agent_browser: true, urls: [url]})`. In Chrome or Safari this opens a window of the Computer's own in the user's everyday browser, with the user's sign-ins; Edge and Firefox run as separate instances. Pass `bundle_id` only when the user asks for one browser. The browser tools drive a page in a way these sites detect, so never use them on these sites; the Computer keeps them off every site the task opened this way. +- If `launch_app` has no `agent_browser`, this needs a Mac Computer: tell the user and stop. +- If a call says a computer tool does not exist, take the tool's exact name from your tool list and call it again before telling the user the tools are unavailable. +- If `launch_app` answers `COMPUTER_BROWSER_RESTART_REQUIRED`, ask the user with a question card to quit Chrome from its menu and leave it closed, saying that the Computer starts it again with their windows and tabs; then end your turn. When they answer that Chrome is closed, call `launch_app` again; if they decline, end the task. +- If the site asks to sign in, ask the user to sign in once in that browser window and stop; the sign-in stays for later tasks. + +## Move like a person + +These sites watch how a visitor moves, and the user may watch the window. Every page change looks like a person's. + +- Open each site once with `launch_app`, at its home page, such as `https://www.reddit.com/`; in Chrome, one site per call. A page the user names or describes, such as a forum, a category, a search or their order list, is reached from the home page through the site's own controls. Never call `launch_app` again for another page of a site already open. +- After that, reach each page the way a person does: type into the site's own search box, use its menus, filters, sort controls and page links, and click the item you want. +- Never type or paste an address into the browser's address bar, and never build a search, filter or page address yourself, in a tool call or in a script. +- If the page the user wants cannot be reached from the home page, such as a private share link, tell the user and stop. +- Go back through the page's own links, such as the site's logo or its breadcrumb. If the item opened in a new tab, close that tab with its Close button. In Edge or Firefox the `cmd+[` hotkey to the window also goes back. +- To see more items in a list that grows as it scrolls, scroll the list with `scroll`. Use the site's next-page link for a paged list. + +## Keep it quick + +The user watches the window, and each turn of yours costs seconds. The Computer already spaces each input like a quick person. Do not add waits of your own. + +- Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. +- Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. +- Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Safari, Edge and Firefox. +- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or `pressKey`, not even Return in the search box. +- After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. +- Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. A read with a screenshot takes seconds on a busy page; ask for one only to see what the text does not hold. + +A visit to a result, in one script: + +```js +const app = await cua.getApp({pid, windowId}); +await app.setValue({role: "AXTextField", name: "Search"}, "camping kettle"); +await app.click({role: "AXButton", name: "Search"}); +await app.waitFor({role: "AXLink", name: "Next", timeout_ms: 8000}); +const links = await app.query({role: "AXLink"}); +const first = links.find((l) => /kettle/i.test(l.label)); +await app.click({role: "AXLink", name: first.label}); +await app.waitFor({text: "out of 5 stars", timeout_ms: 8000}); +const page = await app.getState({query: "out of 5 stars"}); +await app.click({role: "AXLink", name: "Back to results"}); +return {title: first.label, rating: page.tree_markdown}; +``` + +## Read + +- Read with `get_window_state`. It returns the whole page the browser has loaded, including the part below the window, so text already on the page needs no scrolling. Pass `query` to narrow a long page. If a read says it was cut, read again narrower instead of guessing. +- Text inside pictures: `recognize_text`; in a picture viewer, `pages` with `next_key`. +- A state the tree does not show, such as liked or followed: look with `include_screenshot: true`. +- To repeat a flow over items, use `run_script`. A script there sends at most 20 inputs: return what it extracted, look at the window, then continue. Many sites open a result in a new tab; the script then reads that tab, so read the tab in front before the next step. +- Stop scrolling when two reads add no new items. Read only what the task needs: one search per keyword, and never reopen an item already read. +- Act only in the tabs this task opened. A click that opens a new tab is the task's tab. Close each item's tab with its tab's Close button after reading it, and the task's last tab when done. If a tab the task did not open is in front, stop and tell the user. + +## Write + +Write only when the user asked for that action in this task: publish, comment, like, follow or delete. Text to review or a request to draft is not permission. + +1. Check on the page that the signed-in account is the one the user meant. If not, stop and tell the user. +2. Make sure exactly one item matches the target. If several do, ask. +3. Send once, then read the page again and confirm the result shows, such as the comment under that account. +4. If the result is unclear, read the page. Never send again blind. + +If the user takes over the window, read the page and the account again before the next input. + +## Checks and limits + +- When a page shows a captcha, a code or QR check, a sign-in wall or a too-many-requests message, call `report_blocker` with that kind and the page's URL at once. Then tell the user in the Room what the page asks for and that they can deal with it in that browser window, report what was done, and end the task. +- When only one item cannot be viewed, call `report_blocker` with `content_unavailable` and its URL, and skip it. +- Never pass a check yourself, switch accounts or retry around a limit. +- When the Computer refuses input with `COMPUTER_INPUT_BUDGET`, `COMPUTER_SITE_COOLDOWN`, `COMPUTER_SITE_BLOCKED` or `COMPUTER_TAB_CHANGED`, report what was done and end the task. From f79692f46f0a415d6a661af509cb2d8e706c78ca Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 15:21:19 +0800 Subject: [PATCH 14/30] fix(skills): ship only humanlike-browse 1.0.6 1.0.5 never reached main, so the PR adds one new version instead of freezing a retired text. An environment that sealed 1.0.5 keeps the row and moves its latest Manifest to 1.0.6. Co-Authored-By: Claude Opus 5.5 --- skills/builtin/catalog.yaml | 3 - .../builtin/humanlike-browse/1.0.5/SKILL.md | 80 ------------------- 2 files changed, 83 deletions(-) delete mode 100644 skills/builtin/humanlike-browse/1.0.5/SKILL.md diff --git a/skills/builtin/catalog.yaml b/skills/builtin/catalog.yaml index dab7f2ca..81fed2fd 100644 --- a/skills/builtin/catalog.yaml +++ b/skills/builtin/catalog.yaml @@ -227,9 +227,6 @@ skills: - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_6 version: 1.0.6 package_path: builtin/humanlike-browse/1.0.6 - - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_5 - version: 1.0.5 - package_path: builtin/humanlike-browse/1.0.5 - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_4 version: 1.0.4 package_path: builtin/humanlike-browse/1.0.4 diff --git a/skills/builtin/humanlike-browse/1.0.5/SKILL.md b/skills/builtin/humanlike-browse/1.0.5/SKILL.md deleted file mode 100644 index 8c0b3f7a..00000000 --- a/skills/builtin/humanlike-browse/1.0.5/SKILL.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -name: humanlike-browse -description: Read, survey, post to or manage a site that watches for automated browsing, such as a social network, a marketplace or any site where the user is signed in, through the computer tools in the Computer's own window of the user's browser. Use whenever a task reads posts, comments, profiles or search results on such a site, or writes there for the user. ---- - -# Sites that watch for automated browsing - -## Choose the route - -- A public page that needs no sign-in and no clicks: read it with the web search or fetch tools. -- Anything else on a social network, a marketplace or a site where the user is signed in: only the computer tools, through `launch_app({agent_browser: true, urls: [url]})`. In Chrome or Safari this opens a window of the Computer's own in the user's everyday browser, with the user's sign-ins; Edge and Firefox run as separate instances. Pass `bundle_id` only when the user asks for one browser. The browser tools drive a page in a way these sites detect, so never use them on these sites; the Computer keeps them off every site the task opened this way. -- If `launch_app` has no `agent_browser`, this needs a Mac Computer: tell the user and stop. -- If a call says a computer tool does not exist, take the tool's exact name from your tool list and call it again before telling the user the tools are unavailable. -- If `launch_app` answers `COMPUTER_BROWSER_RESTART_REQUIRED`, ask the user with a question card whether the Computer may restart Chrome, saying that their windows and tabs come back. Only if they agree, call `launch_app` again with `restart_browser: true`; otherwise end the task. -- If the site asks to sign in, ask the user to sign in once in that browser window and stop; the sign-in stays for later tasks. - -## Move like a person - -These sites watch how a visitor moves, and the user may watch the window. Every page change looks like a person's. - -- Open each site once with `launch_app`, at its home page, such as `https://www.reddit.com/`; in Chrome, one site per call. A page the user names or describes, such as a forum, a category, a search or their order list, is reached from the home page through the site's own controls. Never call `launch_app` again for another page of a site already open. -- After that, reach each page the way a person does: type into the site's own search box, use its menus, filters, sort controls and page links, and click the item you want. -- Never type or paste an address into the browser's address bar, and never build a search, filter or page address yourself, in a tool call or in a script. -- If the page the user wants cannot be reached from the home page, such as a private share link, tell the user and stop. -- Go back through the page's own links, such as the site's logo or its breadcrumb. If the item opened in a new tab, close that tab with its Close button. In Edge or Firefox the `cmd+[` hotkey to the window also goes back. -- To see more items in a list that grows as it scrolls, scroll the list with `scroll`. Use the site's next-page link for a paged list. - -## Keep it quick - -The user watches the window, and each turn of yours costs seconds. The Computer already spaces each input like a quick person. Do not add waits of your own. - -- Browse in this conversation yourself. Do not hand the browsing to a background Task: starting one adds a minute, and the user watches this Room. -- Put each page visit in one `run_script`: search, wait, read what you need, open the item, wait, read it, go back. Return to the model only to choose something, such as which items to open. -- Fill a search box with `setValue`, then click the site's search button. Return on the box also works in Edge and Firefox. -- The user's Chrome takes no keys from the Computer: a key could reach the Chrome window the user is typing in. There, never use `typeText`, `hotkey` or a key without an element; Return bound to the search box goes only while no other Chrome window could take it. -- After a click that loads something, wait with `app.waitFor` for what the next page shows, such as a heading or a button, with `timeout_ms` about 8000. Never wait with `cua.sleep`; one sleep lasts at most 1.5 s there. -- Read only what you need: `app.query({role, name})`, `getState({query})` or `get_window_state` with `query`. Read the whole tree only to find your way on a page you have not seen. - -A visit to a result, in one script: - -```js -const app = await cua.getApp({pid, windowId}); -await app.setValue({role: "AXTextField", name: "Search"}, "camping kettle"); -await app.click({role: "AXButton", name: "Search"}); -await app.waitFor({role: "AXLink", name: "Next", timeout_ms: 8000}); -const links = await app.query({role: "AXLink"}); -const first = links.find((l) => /kettle/i.test(l.label)); -await app.click({role: "AXLink", name: first.label}); -await app.waitFor({text: "out of 5 stars", timeout_ms: 8000}); -const page = await app.getState({query: "out of 5 stars"}); -await app.click({role: "AXLink", name: "Back to results"}); -return {title: first.label, rating: page.tree_markdown}; -``` - -## Read - -- Read with `get_window_state`. It returns the whole page the browser has loaded, including the part below the window, so text already on the page needs no scrolling. Pass `query` to narrow a long page. If a read says it was cut, read again narrower instead of guessing. -- Text inside pictures: `recognize_text`; in a picture viewer, `pages` with `next_key`. -- A state the tree does not show, such as liked or followed: look with `include_screenshot: true`. -- To repeat a flow over items, use `run_script`. A script there sends at most 20 inputs: return what it extracted, look at the window, then continue. Many sites open a result in a new tab; the script then reads that tab, so read the tab in front before the next step. -- Stop scrolling when two reads add no new items. Read only what the task needs: one search per keyword, and never reopen an item already read. -- Act only in the tabs this task opened. A click that opens a new tab is the task's tab. Close each item's tab with its tab's Close button after reading it, and the task's last tab when done. If a tab the task did not open is in front, stop and tell the user. - -## Write - -Write only when the user asked for that action in this task: publish, comment, like, follow or delete. Text to review or a request to draft is not permission. - -1. Check on the page that the signed-in account is the one the user meant. If not, stop and tell the user. -2. Make sure exactly one item matches the target. If several do, ask. -3. Send once, then read the page again and confirm the result shows, such as the comment under that account. -4. If the result is unclear, read the page. Never send again blind. - -If the user takes over the window, read the page and the account again before the next input. - -## Checks and limits - -- When a page shows a captcha, a code or QR check, a sign-in wall or a too-many-requests message, call `report_blocker` with that kind and the page's URL at once. Then tell the user in the Room what the page asks for and that they can deal with it in that browser window, report what was done, and end the task. -- When only one item cannot be viewed, call `report_blocker` with `content_unavailable` and its URL, and skip it. -- Never pass a check yourself, switch accounts or retry around a limit. -- When the Computer refuses input with `COMPUTER_INPUT_BUDGET`, `COMPUTER_SITE_COOLDOWN`, `COMPUTER_SITE_BLOCKED` or `COMPUTER_TAB_CHANGED`, report what was done and end the task. From df5a0007e5e0747b258d9cd6ea07ab8bdbd9ff2c Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 15:48:38 +0800 Subject: [PATCH 15/30] fix(computer): carry a script's read tokens into the next script The Daemon reads the window before and after each input in the Computer's own browser, and each read replaces Cua's snapshot. Tokens from a script's reads went stale, since those reads run inside the driver and the Daemon never saw their rows. The guard now returns the reads made after the script's last input beside its value; the Daemon records them and puts the script's own value back into the result. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 2 + .../internal/linger/localtools/cua.go | 9 ++ .../internal/linger/localtools/cua_script.go | 107 ++++++++++++++++-- .../localtools/cua_script_guard_test.go | 7 +- .../linger/localtools/cua_snapshots.go | 36 +++--- .../linger/localtools/cua_snapshots_test.go | 63 +++++++++++ 6 files changed, 202 insertions(+), 22 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index e4147743..0a7d3b22 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -383,6 +383,8 @@ settles, since the browser shows such a tab a moment later. Reading the tab strip replaces Cua's snapshot of the window, so the Daemon remembers the rows of recent reads and moves an Agent's element token, in an input or written into a script, to the current snapshot when its row reads the same there. +A script's reads after its last input count too: the script guard returns +them beside the script's value, and the Daemon takes them out of the result. Only the window the task opened with `launch_app` is held. A script that names no window in its `args` is taken to drive that window; a script to another app is neither diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index afc52b4a..7147f9ec 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -542,8 +542,17 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage } driverStarted := time.Now() result, err := c.sendInput(ctx, name, arguments, appKeys, agentChrome) + var scriptReads []windowRead + if err == nil && name == "run_script" { + result, scriptReads = scriptResultReads(result) + } if c.snapshots != nil { c.snapshots.sent(name) + // The script's reads after its last input come after the fence, so + // their tokens carry over the Computer's reads before the next call. + for _, read := range scriptReads { + c.snapshots.record(read) + } } driverMs = time.Since(driverStarted).Milliseconds() if err == nil { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index d115f460..5efeb152 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -1,6 +1,7 @@ package localtools import ( + "bytes" "context" "encoding/json" "fmt" @@ -67,7 +68,9 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // script, in Unix milliseconds, so it can refuse an input whose result it // could not verify in time. A script reads text, so getState takes no // screenshot unless the script asks for one: Cua's default captures one per -// read, which took seconds on a busy page. A positive inputLimit paces the script for the +// read, which took seconds on a busy page. The guard returns the window reads +// made after the script's last input beside its value, under scriptReadsKey, +// for scriptResultReads to take out. A positive inputLimit paces the script for the // Computer's own browser: the guard waits a person's pause before each input, // throws once the script tries to send more than inputLimit inputs, and // refuses to open an app or URL, whose site checks run only in the Daemon's @@ -85,10 +88,12 @@ func guardedScript(args, body string, deadline int64, inputLimit int, chrome int source, _ := json.Marshal(`"use strict"; ` + body + "\n") return strings.ReplaceAll(`const args = Object.freeze(`+args+`), DEADLINE = `+strconv.FormatInt(deadline, 10)+`; const __guarded = (() => { const keys = Object.keys, names = Object.getOwnPropertyNames, proto = Object.getPrototypeOf, define = Object.defineProperty, freeze = Object.freeze; -const apply = Reflect.apply, stringify = JSON.stringify, parse = JSON.parse, test = RegExp.prototype.test, then = Promise.prototype.then; +const apply = Reflect.apply, stringify = JSON.stringify, parse = JSON.parse, test = RegExp.prototype.test, then = Promise.prototype.then, slice = Array.prototype.slice; const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis.cua; const bad = /`+strings.Join(pattern, "|")+`/; -const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0; +const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0, reads = [], readChars = 0; +const reading = (k, c) => k === "getState" || k === "call" && c[0] === "get_window_state"; +const keep = r => { if (r === null || typeof r !== "object" || typeof r.snapshot_id !== "string" || typeof r.pid !== "number" || typeof r.window_id !== "number") return; const read = {snapshot_id: r.snapshot_id, pid: r.pid, window_id: r.window_id}; for (const f of ["tree_markdown", "tree_diff", "since", "since_status"]) if (typeof r[f] === "string") read[f] = r[f]; const size = (read.tree_markdown || "").length + (read.tree_diff || "").length; reads[reads.length] = read; readChars += size; while (reads.length > `+strconv.Itoa(scriptReadsKept)+` || readChars > `+strconv.Itoa(scriptReadCharacters)+` && reads.length > 0) { const first = reads[0]; readChars -= (first.tree_markdown || "").length + (first.tree_diff || "").length; reads = apply(slice, reads, [1]); } }; const keyOps = {__proto__: null, hotkey: true, type_text: true, press_key: true}, keyMethods = {__proto__: null, hotkey: true, typeText: true, pressKey: true}; const routes = v => v !== null && typeof v === "object" && (v.pid !== undefined || v.app !== undefined || v.target !== undefined || v.window_id !== undefined || v.windowId !== undefined); const rerouted = c => { for (let i = 0; i < c.length; i++) if (routes(c[i])) return true; return false; }; @@ -109,22 +114,23 @@ define(w, k, {enumerable: true, get() { const v = o[k]; if (typeof v !== "functi if (limit > 0 && k === "typeText") { const given = c.length > 2 && c[2] !== null && typeof c[2] === "object" ? c[2] : {}; const t = {length: 3}; define(t, 0, {value: c[0]}); define(t, 1, {value: c[1]}); define(t, 2, {value: {delay_ms: `+strconv.Itoa(agentChromeTypeDelay)+`, ...given}}); c = t; } if (k === "getState" && (c.length === 0 || c[0] === undefined || c[0] !== null && typeof c[0] === "object" && c[0].include_screenshot === undefined)) { const t = {length: 1}; define(t, 0, {value: {include_screenshot: false, ...(c.length > 0 && c[0] !== undefined ? c[0] : {})}}); c = t; } if (limit > 0 && k === "sleep" && o === real && typeof c[0] === "number" && c[0] > `+strconv.Itoa(scriptSleepLimit)+`) { const t = {length: 1}; define(t, 0, {value: `+strconv.Itoa(scriptSleepLimit)+`}); c = t; } -const run = () => { const r = apply(v, o, c); return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [out]) : out(r); }; +const run = () => { const r = apply(v, o, c); const seen = x => { if (reading(k, c)) keep(x); return out(x); }; return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [seen]) : seen(r); }; if (launches(k, c)) throw new Error("COMPUTER_SCRIPT_REFUSED: a script opens no app or URL, so the site checks of launch_app hold and the key rule knows every app the script reaches; call launch_app as its own tool"); if (keyed(k, c, o)) throw new Error(`+strconv.Quote(everydayChromeKeysRefused)+`); +if (input(k, c)) { reads = []; readChars = 0; } if (limit <= 0 || !input(k, c)) return run(); if (++sent > limit) throw new Error("COMPUTER_SCRIPT_INPUT_LIMIT: a script sends at most " + limit + " inputs to the Computer's own browser; return what you have, look at the window, then continue in a new script"); return apply(then, apply(rest, real, [`+strconv.Itoa(scriptInputPauseMin)+` + floor(random() * `+strconv.Itoa(scriptInputPauseSpan)+`)]), [run]); }; }}); } } return freeze(w); }; -return wrap(real); })(); +return {api: wrap(real), done: v => reads.length === 0 ? v : v === undefined ? {`+scriptReadsKey+`: reads} : {`+scriptReadsKey+`: reads, value: v}}; })(); try { new ((async function () {}).constructor)(`+string(source)+`); } catch (e) { throw new Error("COMPUTER_SCRIPT_REFUSED: the script is not one complete function body: " + e.message); } for (const f of [function () {}, async function () {}, function* () {}, async function* () {}]) Object.defineProperty(Object.getPrototypeOf(f), "constructor", {value: undefined, writable: false, configurable: false}); for (const b of [Object, Object.prototype, Array, Array.prototype, Function.prototype, Promise, Promise.prototype, JSON, Reflect, RegExp, RegExp.prototype, String.prototype, Symbol]) Object.freeze(b); -return await (function (cua, globalThis, eval, Function) { return (async function () { "use strict"; `, "\n", " ") + body + scriptEpilogue +return __guarded.done(await (function (cua, globalThis, eval, Function) { return (async function () { "use strict"; `, "\n", " ") + body + scriptEpilogue } // scriptEpilogue closes what guardedScript opened. -const scriptEpilogue = "\n}).call(undefined); })(__guarded, undefined, undefined, undefined);" +const scriptEpilogue = "\n}).call(undefined); })(__guarded.api, undefined, undefined, undefined));" // forbiddenKey reports whether value holds an object key that names word. func forbiddenKey(value any, word string) bool { @@ -213,6 +219,93 @@ func scriptArguments(ctx context.Context, arguments map[string]any, reviewScope return nil } +// scriptReadsKey names the reads the guard adds to a script's return value. +// The guard keeps at most scriptReadsKept of them and scriptReadCharacters of +// tree text, the latest first kept: an Agent acts on its latest read, and a +// whole busy page is a few hundred thousand characters. +const ( + scriptReadsKey = "__computer_reads" + scriptReadsKept = 4 + scriptReadCharacters = 2000000 +) + +// scriptValueCharacters is where Cua cuts a return value in a script's text. +const scriptValueCharacters = 8000 + +// scriptResultReads takes the window reads the guard added to a script's +// return value out of the result, and puts the value the script returned +// back into the text Cua wrote for it. They are the script's reads after its +// last input, so the Computer's own reads before the next call can carry the +// tokens of those rows over, as they do for the Agent's own reads. A result +// without them is returned as it is. +func scriptResultReads(result json.RawMessage) (json.RawMessage, []windowRead) { + var reply map[string]json.RawMessage + var structured map[string]json.RawMessage + var value map[string]json.RawMessage + if json.Unmarshal(result, &reply) != nil || json.Unmarshal(reply["structuredContent"], &structured) != nil || + json.Unmarshal(structured["value"], &value) != nil { + return result, nil + } + _, inner := value["value"] + if _, ok := value[scriptReadsKey]; !ok || len(value) > 2 || len(value) == 2 && !inner { + return result, nil + } + var reads []windowRead + if json.Unmarshal(value[scriptReadsKey], &reads) != nil { + return result, nil + } + returned := json.RawMessage("null") + if inner { + var compact bytes.Buffer + if json.Compact(&compact, value["value"]) != nil { + return result, nil + } + returned = compact.Bytes() + } + structured["value"] = returned + text := string(returned) + if runes := []rune(text); len(runes) > scriptValueCharacters { + text = string(runes[:scriptValueCharacters]) + "…" + } + var content []map[string]any + if json.Unmarshal(reply["content"], &content) == nil { + for _, item := range content { + if body, ok := item["text"].(string); ok { + item["text"] = withReturnValue(body, text) + } + } + } + encodedContent, contentErr := json.Marshal(content) + encodedStructured, structuredErr := json.Marshal(structured) + if contentErr != nil || structuredErr != nil { + return result, nil + } + reply["content"], reply["structuredContent"] = encodedContent, encodedStructured + unwrapped, err := json.Marshal(reply) + if err != nil { + return result, nil + } + return unwrapped, reads +} + +// withReturnValue replaces the return value line of a script's text, which +// runs to the console or call log section after it. +func withReturnValue(text, value string) string { + const label = "\nreturn value: " + start := strings.Index(text, label) + if start < 0 { + return text + } + start += len(label) + end := len(text) + for _, next := range []string{"\nconsole (", "\ncalls:\n"} { + if at := strings.Index(text[start:], next); at >= 0 && start+at < end { + end = start + at + } + } + return text[:start] + value + text[end:] +} + // compactScriptResult keeps a script's result small in the model's context: // the driver call log keeps its first and last lines, the text is cut at a // fixed length, and the screenshots of the reads the script made are dropped. diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index 1e5ac204..caff3f53 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -14,7 +14,7 @@ import ( // the same prelude's semantics. const guardHarness = `import fs from "fs"; const log = []; -class App { constructor(n, pid, windowId) { this.n = n; this.pid = pid; this.windowId = windowId; } hotkey(keys, on, options) { log.push("hotkey:" + keys.join("+") + ":" + JSON.stringify({pid: this.pid, ...on, ...options})); return "hotkeyed"; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } getState(options) { log.push("state:" + JSON.stringify(options)); return {tree_markdown: ""}; } } +class App { constructor(n, pid, windowId) { this.n = n; this.pid = pid; this.windowId = windowId; } hotkey(keys, on, options) { log.push("hotkey:" + keys.join("+") + ":" + JSON.stringify({pid: this.pid, ...on, ...options})); return "hotkeyed"; } click(a) { log.push("raw:" + JSON.stringify(a)); return "clicked"; } typeText(text, on, options) { log.push("type:" + JSON.stringify(options)); return "typed"; } pressKey(key, on) { log.push("key:" + key); return "pressed"; } getState(options) { log.push("state:" + JSON.stringify(options)); return options.query === "rows" ? {snapshot_id: "s0000000a", pid: this.pid, window_id: 9, tree_markdown: "- [3] AXButton \"OK\"", elements: 1} : {tree_markdown: ""}; } } const fake = { call(t, a) { log.push("call:" + t + ":" + JSON.stringify(a)); return Promise.resolve({ok: true}); }, launch(name) { log.push("launched:" + name); return Promise.resolve(new App("l", 7, 1)); }, getApp(target) { return Promise.resolve(new App("a", target && target.pid, target && target.windowId)); }, @@ -48,10 +48,13 @@ func TestScriptGuardRefusesForbiddenInputInAJSEngine(t *testing.T) { {"handle in an object", `const b = await cua.box(); return b.inner.app.click({out_file: "/tmp/x"});`, "threw:COMPUTER_SCRIPT_REFUSED", "raw:"}, {"no global object", `return [typeof globalThis, typeof eval, typeof Function, typeof this].join(",");`, "undefined,undefined,undefined,undefined", ""}, {"no function constructor", `return typeof (async function () {}).constructor;`, "undefined", ""}, - {"closing the wrapper", "return 1; }).call(undefined); })(globalThis.cua, globalThis, eval, Function); (function () { return (async function () {", "threw:COMPUTER_SCRIPT_REFUSED: the script is not one complete function body", "call:"}, + {"closing the wrapper", "return 1; }).call(undefined); })(globalThis.cua, globalThis, eval, Function)); Object(await (function () { return (async function () {", "threw:COMPUTER_SCRIPT_REFUSED: the script is not one complete function body", "call:"}, {"closing the wrapper to call", `}); await globalThis.cua.call("click", {["deliv" + "ery_mode"]: "foreground"}); (async function () {`, "threw:", "call:"}, {"inputs", `return [args.n, DEADLINE].join(",");`, `"1,42"`, ""}, {"read without a screenshot", `const app = await cua.getApp({pid: 1}); await app.getState(); await app.getState({query: "Search"}); await app.getState({include_screenshot: true}); return "read";`, `"state:{\"include_screenshot\":false}","state:{\"include_screenshot\":false,\"query\":\"Search\"}","state:{\"include_screenshot\":true}"`, ""}, + {"reads after the last input", `const app = await cua.getApp({pid: 4}); await app.click(1); await app.getState({query: "rows"}); return "done";`, `"result":{"__computer_reads":[{"snapshot_id":"s0000000a","pid":4,"window_id":9,"tree_markdown":"- [3] AXButton \"OK\""}],"value":"done"}`, ""}, + {"a read before an input", `const app = await cua.getApp({pid: 4}); await app.getState({query: "rows"}); await app.click(1); return "done";`, `"result":"done"`, ""}, + {"no return value", `const app = await cua.getApp({pid: 4}); await app.getState({query: "rows"});`, `"result":{"__computer_reads":[{"snapshot_id":"s0000000a","pid":4,"window_id":9,"tree_markdown":"- [3] AXButton \"OK\""}]}`, ""}, {"frozen builtins", `try { JSON.parse = () => ({delivery_mode: 1}); } catch (e) { return "refused"; } return "changed";`, "refused", ""}, } for _, tc := range cases { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go index 379acd3a..e0ea73d3 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go @@ -46,7 +46,8 @@ func newSnapshotRows() *snapshotRows { } // observe records the rows of a read, or forgets the current snapshots after -// a script, whose reads run inside the driver. +// a script, whose reads run inside the driver: the reads it made after its +// last input are recorded from its result once the fence for it is set. func (s *snapshotRows) observe(params any, result json.RawMessage) { call, _ := params.(map[string]any) name, _ := call["name"].(string) @@ -60,20 +61,29 @@ func (s *snapshotRows) observe(params any, result json.RawMessage) { return } var reply struct { - Structured struct { - Snapshot string `json:"snapshot_id"` - PID float64 `json:"pid"` - Window float64 `json:"window_id"` - Tree string `json:"tree_markdown"` - Diff string `json:"tree_diff"` - Since string `json:"since"` - Status string `json:"since_status"` - } `json:"structuredContent"` - } - if json.Unmarshal(result, &reply) != nil || reply.Structured.Snapshot == "" { + Structured windowRead `json:"structuredContent"` + } + if json.Unmarshal(result, &reply) == nil { + s.record(reply.Structured) + } +} + +// windowRead is the part of a get_window_state result that names its rows. +type windowRead struct { + Snapshot string `json:"snapshot_id"` + PID float64 `json:"pid"` + Window float64 `json:"window_id"` + Tree string `json:"tree_markdown"` + Diff string `json:"tree_diff"` + Since string `json:"since"` + Status string `json:"since_status"` +} + +// record keeps the rows of read as its window's current snapshot. +func (s *snapshotRows) record(read windowRead) { + if read.Snapshot == "" { return } - read := reply.Structured window := fmt.Sprintf("%g:%g", read.PID, read.Window) rows := map[string]string{} s.mu.Lock() diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go index 7cf14f81..955beff3 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go @@ -2,6 +2,7 @@ package localtools import ( "encoding/json" + "strings" "testing" ) @@ -94,3 +95,65 @@ func TestSnapshotRowsCarriesTokensOverTheComputersOwnReads(t *testing.T) { t.Fatalf("after a script = %v, want the token unchanged", after["element_token"]) } } + +// A script's reads after its last input come back in its result, so a token +// it returned carries over the Computer's own reads into the next script. +func TestSnapshotRowsCarriesAScriptsTokensIntoTheNextScript(t *testing.T) { + tree := "- [0] AXWindow \"Shop\"\n - [4] AXLink \"Kettle\" [actions=[press]]" + value := map[string]any{scriptReadsKey: []any{map[string]any{"snapshot_id": "s00000011", "pid": 7, "window_id": 9, "tree_markdown": tree}}, "value": map[string]any{"z": 1, "a": "s00000011:4"}} + encodedValue, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + text := "run_script: ok in 5 ms, 2 driver call(s)\nreturn value: " + string(encodedValue)[:40] + "…\nconsole (1 line(s)):\n hi\ncalls:\n 1. line 1 get_window_state ok (3 ms): window_id=9" + result, err := json.Marshal(map[string]any{"content": []any{map[string]any{"type": "text", "text": text}}, + "structuredContent": map[string]any{"ok": true, "value": json.RawMessage(`{"__computer_reads":` + string(mustJSON(t, value[scriptReadsKey])) + `,"value":{"z":1,"a":"s00000011:4"}}`)}}) + if err != nil { + t.Fatal(err) + } + unwrapped, reads := scriptResultReads(result) + var reply struct { + Content []struct{ Text string } `json:"content"` + Structured struct { + Value json.RawMessage `json:"value"` + } `json:"structuredContent"` + } + if err := json.Unmarshal(unwrapped, &reply); err != nil || len(reads) != 1 { + t.Fatalf("reply = %s, reads = %v, error = %v", unwrapped, reads, err) + } + if want := "return value: {\"z\":1,\"a\":\"s00000011:4\"}\nconsole (1 line(s)):\n hi\ncalls:\n"; !strings.Contains(reply.Content[0].Text, want) { + t.Fatalf("text = %q, want it to hold %q", reply.Content[0].Text, want) + } + if string(reply.Structured.Value) != `{"z":1,"a":"s00000011:4"}` { + t.Fatalf("structured value = %s", reply.Structured.Value) + } + + s := newSnapshotRows() + s.observe(map[string]any{"name": "run_script"}, unwrapped) + s.sent("run_script") + for _, read := range reads { + s.record(read) + } + // The Computer counts the tabs after the script and checks the tab + // before the next one; each read replaces the script's snapshot. + s.observe(readCall(nil), readResult(t, map[string]any{"snapshot_id": "s00000012", "pid": 7, "window_id": 9, "tree_markdown": tree})) + next := map[string]any{"script": `await app.click("s00000011:4")`} + s.remap(next) + if want := `await app.click("s00000012:4")`; next["script"] != want { + t.Fatalf("next script = %v, want %s", next["script"], want) + } + + plain := json.RawMessage(`{"content":[{"type":"text","text":"run_script: ok\nreturn value: 1"}],"structuredContent":{"value":1}}`) + if same, reads := scriptResultReads(plain); string(same) != string(plain) || reads != nil { + t.Fatalf("a result without reads = %s, %v; want it unchanged", same, reads) + } +} + +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return encoded +} From 9b858e3ee76bbdca27da502e1283a48c5b0358a8 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 16:03:18 +0800 Subject: [PATCH 16/30] refactor(computer): remove recognize_text Agents almost never used it: two calls in about a thousand Computer tool calls, both after a screenshot that already showed the text, and one misread it. A page's text is in the accessibility tree, and text inside a picture shows in a screenshot. Its paged read sends keys, which the user's everyday Chrome refuses, so it could not work on the main browsing route. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 2 +- .../linger/localtools/agent_browser.go | 2 +- .../linger/localtools/agent_chrome_pace.go | 2 +- .../internal/linger/localtools/cua.go | 27 +- .../internal/linger/localtools/cua_text.go | 352 ------------------ .../linger/localtools/cua_text_darwin_test.go | 126 ------- .../linger/localtools/cua_text_test.go | 45 --- .../internal/linger/localtools/lanes_test.go | 2 - .../linger/localtools/localtools_test.go | 2 - .../internal/permission/allowlist.go | 9 +- .../internal/permission/allowlist_test.go | 3 - .../internal/permission/gui_reuse.go | 24 +- .../internal/permission/gui_reuse_test.go | 3 - .../builtin/humanlike-browse/1.0.6/SKILL.md | 3 +- 14 files changed, 12 insertions(+), 590 deletions(-) delete mode 100644 apps/computer/local-daemon/internal/linger/localtools/cua_text.go delete mode 100644 apps/computer/local-daemon/internal/linger/localtools/cua_text_darwin_test.go delete mode 100644 apps/computer/local-daemon/internal/linger/localtools/cua_text_test.go diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 0a7d3b22..8582e206 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -348,7 +348,7 @@ the tabs are found by role and place in the window, not by their names, which follow the user's language. The Daemon finds the running Chrome by its executable wherever the app is installed. The user's Chrome takes no keys from the Computer (`COMPUTER_KEYS_REFUSED`), in any of its windows, in tool calls, -`run_actions` steps, paged `recognize_text` reads and scripts: a shortcut +`run_actions` steps and scripts: a shortcut runs as a menu command in the Chrome window that has the keyboard, and Cua types text as keys that all read as the A key. A key named by its `element_token` goes to the app the token was read from, and a key that names diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index 0a2bc5ba..0102fe34 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -628,7 +628,7 @@ func userChromePID(ctx context.Context) (int, error) { return pid, nil } -// userChromeKeys refuses a key tool, or a paged read, that reaches the user's +// userChromeKeys refuses a key tool that reaches the user's // everyday Chrome, in any of its windows. A key named by its element_token // goes to the app the token was read from, and a key that names no app is // refused while that Chrome runs. diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go index 1c736822..20ad7b36 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go @@ -20,7 +20,7 @@ import ( // The Computer's own browser is where Agents use sites that watch for // automated browsing. Every input it receives, from a tool call, a batch step, -// a script or a paged text read, passes the same pace: a pause like a person's +// or a script, passes the same pace: a pause like a person's // between inputs, a budget per task and per day, and a stop once the Agent // reports a check that only the user may pass. The model decides what a page // shows; the Computer only enforces what follows from that decision. diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 7147f9ec..3e0eca62 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -190,7 +190,6 @@ func (m *Manager) openCua(ctx context.Context, op *operation) (*cuaSession, []to } tools = append(tools, tool{Name: "run_actions", Description: batchDescription, InputSchema: schema(batchProperties, "actions")}) if runtime.GOOS == "darwin" { - tools = append(tools, tool{Name: "recognize_text", Description: recognizeTextDescription, InputSchema: recognizeTextSchema()}) tools = append(tools, tool{Name: "report_blocker", Description: reportBlockerDescription, InputSchema: reportBlockerSchema()}) } tools = append(tools, tool{Name: checkPermissionsTool, Description: "Read Cua application permissions without prompting. Also reports an unresolved prior action.", InputSchema: schema(map[string]any{})}) @@ -219,7 +218,6 @@ const MacToolGuide = `Computer tools on this Mac (MCP server "computer"): A click, scroll or type_text, in a step or alone, can name its target with role and name instead of element_token: the one element with that role whose label or value is exactly name, or nth (from 0) when several match. It is found when it runs, so one batch can cross pages; finding it reads the window, which stales the tokens of your earlier read, so later steps name their targets the same way. - run_script({script}) runs JavaScript that loops, waits and reads between inputs and returns only what you extract. Use it to repeat a flow over several items, such as open each result, read it and close it, in one call instead of a turn per item. - bring_to_front({pid, window_id}) raises one window. -- recognize_text({pid, window_id, region?, pages?, next_key?}) reads the text in a window's pixels, such as words inside a picture, in any language; it types nothing. pages with next_key reads a picture viewer page by page in one call. - report_blocker({kind, url}) as soon as a page in the Computer's own browser shows a check only the user may pass (captcha, verification, login_required, rate_limited) or one item that cannot be viewed (content_unavailable). Never try to pass the check. The Computer's own browser takes input at a person's pace, within a budget per task and per day, and a script there sends at most 20 inputs: return what you have and look at the window before the next script. A tab that a link in the script opens is the task's and is in front afterwards. The browser tools stay off every site a task opened in that browser. Plan as many steps as you can in one run_actions call, as in one script; its read shows their effect, so do not read again before the next input unless you need another window. Keys are often shorter than clicks: in Finder, cmd+shift+g then a path and Return opens a folder, cmd+down opens the selected item, and in the Trash cmd+delete puts the selected item back. @@ -374,12 +372,6 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage return c.reportBlocker(args) } } - // A one-page read sends nothing. A paged read presses keys, so it takes - // the lease and the action journal below as other input does. - paged := name == "recognize_text" && runtime.GOOS == "darwin" && pagedRead(args) - if name == "recognize_text" && runtime.GOOS == "darwin" && !paged { - return c.recognizeText(ctx, args, false) - } arguments, err := cuaCallArguments(args) if err != nil { return nil, err @@ -397,12 +389,12 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage } // A key to the user's Chrome reaches the window that has the keyboard, // whichever window it names, so it is refused before anything is taken. - if name == "hotkey" || name == "type_text" || name == "press_key" || paged { + if name == "hotkey" || name == "type_text" || name == "press_key" { if err := c.userChromeKeys(ctx, arguments); err != nil { return nil, err } } - agentChrome, err := c.claimAgentChromeInput(ctx, name, arguments, paged) + agentChrome, err := c.claimAgentChromeInput(ctx, name, arguments) if err != nil { return nil, err } @@ -420,7 +412,7 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage return nil, err } } - paced := agentChrome && (inputTools[name] || paged || ownScript) + paced := agentChrome && (inputTools[name] || ownScript) scriptInputs := 0 if paced { // A script reserves nothing: its own pause runs inside it, and its @@ -495,12 +487,6 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage if err := c.requirePermissions(ctx); err != nil { return nil, err } - if paged { - result, err := c.recognizeText(ctx, args, agentChrome) - confirmed = !errors.Is(err, errOutcomeUnknown) - c.stampInput(arguments["pid"]) - return result, err - } // Only a script to the Computer's own browser wakes it first. wake := scriptTarget if !ownScript { @@ -589,9 +575,6 @@ func (c *cuaSession) needsFocus(ctx context.Context, name string, args json.RawM switch { case cuaReadTools[name]: return false - case name == "recognize_text": - // A paged read presses keys in the window. - return runtime.GOOS != "darwin" || pagedRead(args) case runtime.GOOS != "darwin" || !backgroundTools[name]: return true } @@ -652,13 +635,13 @@ func cuaCallArguments(args json.RawMessage) (map[string]any, error) { // claimAgentChromeInput reports whether the call reaches the Computer's own // Chrome and, if so, claims it for this Operation. -func (c *cuaSession) claimAgentChromeInput(ctx context.Context, name string, arguments map[string]any, paged bool) (bool, error) { +func (c *cuaSession) claimAgentChromeInput(ctx context.Context, name string, arguments map[string]any) (bool, error) { // The Computer's own browser takes input and window changes from one // Operation at a time. A script may reach any app from its body, and an // input without a pid may reach it by its element_token alone, so both // take the lease while that Chrome runs. agentChrome := false - leased := inputTools[name] || windowTools[name] || paged + leased := inputTools[name] || windowTools[name] if _, pid := arguments["pid"]; runtime.GOOS == "darwin" && (name == "run_script" || leased && !pid) { own, err := c.agentChromeRunning(ctx) if err != nil { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_text.go b/apps/computer/local-daemon/internal/linger/localtools/cua_text.go deleted file mode 100644 index 1ebf1afd..00000000 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_text.go +++ /dev/null @@ -1,352 +0,0 @@ -package localtools - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "image" - "image/png" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - // The window screenshot is PNG or JPEG. - _ "image/jpeg" -) - -// recognizeTextDescription describes the macOS text recognition tool. -const recognizeTextDescription = "Read the text in a window's pixels, such as words inside a picture that the accessibility tree does not have. It recognizes the languages macOS knows, without being told which, and returns one line per run of text with its position as fractions of the window or region: x from the left, y from the top. region crops the window first, in pixels of the window's screenshot. pages reads a picture viewer page by page: after each page it presses next_key (right, down or pagedown) to the window in the background, and reads only the part of the window that changed between pages. It stops early, returning the pages read and why, when a key changes nothing or is refused, or the call's time runs out. It types nothing and keeps the element tokens of your last read." - -// recognizePagesMax bounds how many pages one call reads. -const recognizePagesMax = 20 - -// recognizeNextKeys are the keys that turn a page: they move, never submit. -var recognizeNextKeys = map[string]bool{"right": true, "down": true, "pagedown": true} - -// recognizeTextFields are the arguments recognize_text takes. -var recognizeTextFields = map[string]bool{"pid": true, "window_id": true, "region": true, "pages": true, "next_key": true} - -// recognizePageSettle is how long a viewer gets to show the next page. -var recognizePageSettle = 700 * time.Millisecond - -func recognizeTextSchema() json.RawMessage { - encoded, _ := json.Marshal(schema(map[string]any{ - "pid": macPID, - "window_id": map[string]any{"type": "integer"}, - "region": map[string]any{"type": "object", "description": "Optional part of the window, in screenshot pixels.", "properties": map[string]any{ - "x": map[string]any{"type": "number"}, "y": map[string]any{"type": "number"}, "width": map[string]any{"type": "number"}, "height": map[string]any{"type": "number"}}}, - "pages": map[string]any{"type": "integer", "minimum": 1, "maximum": recognizePagesMax, "description": "How many pages of a picture viewer to read, from the one shown. Needs next_key."}, - "next_key": map[string]any{"type": "string", "enum": []string{"right", "down", "pagedown"}, "description": "The key that shows the next page."}, - }, "pid", "window_id")) - return encoded -} - -// visionScript runs macOS Vision text recognition on each image file named -// by its arguments and prints one JSON object per recognized line, top first, -// with the index of its file. One process loads the recognizer once. -const visionScript = `ObjC.import('Vision'); -function run(argv) { - const lines = []; - for (let f = 0; f < argv.length; f++) { - const handler = $.VNImageRequestHandler.alloc.initWithURLOptions($.NSURL.fileURLWithPath(argv[f]), $({})); - const request = $.VNRecognizeTextRequest.alloc.init; - request.recognitionLevel = 0; - request.usesLanguageCorrection = true; - request.automaticallyDetectsLanguage = true; - if (!handler.performRequestsError($([request]), null)) throw new Error('text recognition failed'); - for (let i = 0; i < request.results.count; i++) { - const observation = request.results.objectAtIndex(i); - const candidate = observation.topCandidates(1).objectAtIndex(0); - const box = observation.boundingBox; - lines.push(JSON.stringify({file: f, text: ObjC.unwrap(candidate.string), x: box.origin.x, y: 1 - box.origin.y - box.size.height, confidence: candidate.confidence})); - } - } - return lines.join('\n'); -}` - -// recognizedLine is one line that Vision returned. -type recognizedLine struct { - File int `json:"file"` - Text string `json:"text"` - X float64 `json:"x"` - Y float64 `json:"y"` - Confidence float64 `json:"confidence"` -} - -// recognizePagingBudget bounds the time one call spends turning pages, so -// the pages it read come back before the Runtime gives up on the call. -var recognizePagingBudget = 30 * time.Second - -// recognizeText captures a window as Cua shows it, also while it is covered -// or on another Space, and reads its text with macOS Vision. The capture is a -// display-only read, so the Agent's element tokens stay valid. With pages, a -// movement key between captures turns the pages of a picture viewer, and only -// the part of the window that changed is read, so the text around the viewer -// is not repeated for every page. A paged read sends keys, so the call path -// treats it as input (see paged); a one-page read is a read. -func (c *cuaSession) recognizeText(ctx context.Context, args json.RawMessage, agentChrome bool) (output json.RawMessage, callErr error) { - // The lease and the review read these fields by their exact names, so a - // field they would not see is refused rather than decoded. - var fields map[string]json.RawMessage - if json.Unmarshal(args, &fields) != nil { - return nil, fmt.Errorf("recognize_text: arguments are not an object: %w", errNotSent) - } - for field := range fields { - if !recognizeTextFields[field] { - return nil, fmt.Errorf("recognize_text: unknown argument %q: %w", field, errNotSent) - } - } - var params struct { - PID *float64 `json:"pid"` - Window *float64 `json:"window_id"` - Region *struct { - X, Y, Width, Height float64 - } `json:"region"` - Pages int `json:"pages"` - NextKey string `json:"next_key"` - } - if json.Unmarshal(args, ¶ms) != nil || params.PID == nil || params.Window == nil { - return nil, fmt.Errorf("COMPUTER_TARGET_REQUIRED: recognize_text needs pid and window_id: %w", errNotSent) - } - pages := max(params.Pages, 1) - if pages > recognizePagesMax || (pages > 1 && !recognizeNextKeys[params.NextKey]) { - return nil, fmt.Errorf("recognize_text: pages is 1 to %d, and more than one page needs next_key right, down or pagedown: %w", recognizePagesMax, errNotSent) - } - if err := c.checkDesktop(); err != nil { - return nil, err - } - if err := c.startSession(ctx, "auto"); err != nil { - return nil, err - } - first, refused, err := c.windowPixels(ctx, *params.PID, *params.Window) - if err != nil || refused != nil { - return refused, err - } - // The region is checked on the first page, before any key is sent. - region := image.Rectangle{} - if params.Region != nil { - region = image.Rect(int(params.Region.X), int(params.Region.Y), int(params.Region.X+params.Region.Width), int(params.Region.Y+params.Region.Height)).Intersect(first.Bounds()) - if region.Empty() { - return nil, fmt.Errorf("recognize_text: region is outside the %dx%d screenshot: %w", first.Bounds().Dx(), first.Bounds().Dy(), errNotSent) - } - } - shots, changed, note, sent := []image.Image{first}, image.Rectangle{}, "", 0 - defer func() { - if callErr != nil && sent > 0 { - callErr = fmt.Errorf("recognize_text: %d %s key(s) were sent, so the viewer may now show page %d: %w", sent, params.NextKey, sent+1, callErr) - } - }() - budget := time.Now().Add(recognizePagingBudget) - if deadline, ok := ctx.Deadline(); ok && deadline.Add(-10*time.Second).Before(budget) { - budget = deadline.Add(-10 * time.Second) - } - for page := 1; page < pages; page++ { - if time.Now().Add(recognizePageSettle).After(budget) { - note = fmt.Sprintf("\nStopped after page %d: the time for one call ran out; read the next pages in another call.", page) - break - } - var pressed json.RawMessage - if agentChrome { - pressed, err = c.agentChromeKey(ctx, int(*params.PID), *params.Window, "press_key", map[string]any{"key": params.NextKey}) - } else { - pressed, err = c.client.request(ctx, "tools/call", map[string]any{"name": "press_key", "arguments": map[string]any{ - "pid": *params.PID, "window_id": *params.Window, "key": params.NextKey, "session": c.label}}) - } - if err != nil { - sent++ - return nil, err - } - if failed(pressed) { - note = fmt.Sprintf("\nStopped after page %d: the %s key was refused: %s", page, params.NextKey, strings.TrimSpace(string(pressed))) - break - } - sent++ - if err := sleepContext(ctx, recognizePageSettle); err != nil { - return nil, err - } - shot, refused, err := c.windowPixels(ctx, *params.PID, *params.Window) - if err != nil { - return nil, err - } - if refused != nil { - note = fmt.Sprintf("\nStopped after page %d: the window could not be read after the %s key: %s", page, params.NextKey, strings.TrimSpace(string(refused))) - break - } - if shot.Bounds() != first.Bounds() { - note = fmt.Sprintf("\nStopped after page %d: the window changed size after the %s key, so its pages cannot be compared.", page, params.NextKey) - break - } - area := changedArea(shots[len(shots)-1], shot) - if area.Empty() { - note = fmt.Sprintf("\nStopped after page %d: the %s key changed nothing.", page, params.NextKey) - break - } - changed = changed.Union(area) - shots = append(shots, shot) - } - area := region - if area.Empty() && len(shots) > 1 { - area = changed - } - encoded := make([][]byte, len(shots)) - for i, shot := range shots { - if encoded[i], err = encodeArea(shot, area); err != nil { - return nil, err - } - } - lines, err := visionText(ctx, encoded) - if err != nil { - return nil, err - } - var text strings.Builder - switch { - case len(shots) == 1: - fmt.Fprintf(&text, "%d line(s) of text in window %v:", len(lines), *params.Window) - case params.Region != nil: - fmt.Fprintf(&text, "%d page(s) of window %v, read in the given region:", len(shots), *params.Window) - default: - fmt.Fprintf(&text, "%d page(s) of window %v, read in the area that changed between pages (x=%d y=%d %dx%d screenshot pixels):", len(shots), *params.Window, area.Min.X, area.Min.Y, area.Dx(), area.Dy()) - } - for page := range shots { - if len(shots) > 1 { - fmt.Fprintf(&text, "\n## Page %d", page+1) - } - for _, line := range lines { - if line.File != page { - continue - } - fmt.Fprintf(&text, "\n- x=%.2f y=%.2f %q", line.X, line.Y, line.Text) - if line.Confidence < 0.5 { - text.WriteString(" (unsure)") - } - } - } - text.WriteString(note) - return json.Marshal(map[string]any{"content": []any{map[string]string{"type": "text", "text": text.String()}}}) -} - -// pagedRead reports whether a recognize_text call turns pages, and so sends -// keys to the window. -func pagedRead(args json.RawMessage) bool { - var params struct { - Pages int `json:"pages"` - } - return json.Unmarshal(args, ¶ms) == nil && params.Pages > 1 -} - -// windowPixels captures window as a display-only read and decodes it. A -// refusal from Cua comes back as the result to return. -func (c *cuaSession) windowPixels(ctx context.Context, pid, window float64) (image.Image, json.RawMessage, error) { - shot, err := c.client.request(ctx, "tools/call", map[string]any{"name": "get_window_state", "arguments": map[string]any{ - "pid": pid, "window_id": window, "include_accessibility_tree": false, "display_only": true, "session": c.label}}) - if err != nil { - return nil, nil, err - } - var reply struct { - IsError bool `json:"isError"` - Content []struct { - Type string `json:"type"` - Data string `json:"data"` - } `json:"content"` - } - if err := json.Unmarshal(shot, &reply); err != nil { - return nil, nil, fmt.Errorf("recognize_text: unreadable window state: %w", err) - } - if reply.IsError { - return nil, shot, nil - } - for _, item := range reply.Content { - if item.Type == "image" { - pixels, err := base64.StdEncoding.DecodeString(item.Data) - if err != nil { - return nil, nil, errors.New("recognize_text: the window screenshot is not base64") - } - picture, _, err := image.Decode(bytes.NewReader(pixels)) - if err != nil { - return nil, nil, fmt.Errorf("recognize_text: decode the screenshot: %w", err) - } - return picture, nil, nil - } - } - return nil, nil, errors.New("COMPUTER_WINDOW_UNREADABLE: the window has no screenshot to read") -} - -// changedArea returns the bounding box of the pixels that differ between two -// screenshots of one size, or an empty rectangle when they match. Small color -// noise does not count as a change. -func changedArea(before, after image.Image) image.Rectangle { - area := image.Rectangle{} - bounds := before.Bounds() - for y := bounds.Min.Y; y < bounds.Max.Y; y++ { - for x := bounds.Min.X; x < bounds.Max.X; x++ { - r1, g1, b1, _ := before.At(x, y).RGBA() - r2, g2, b2, _ := after.At(x, y).RGBA() - if absDiff(r1, r2)+absDiff(g1, g2)+absDiff(b1, b2) > 3*0x1800 { - area = area.Union(image.Rect(x, y, x+1, y+1)) - } - } - } - return area -} - -func absDiff(a, b uint32) uint32 { - if a > b { - return a - b - } - return b - a -} - -// encodeArea returns the part of picture inside area as PNG, or all of it -// when area is empty. -func encodeArea(picture image.Image, area image.Rectangle) ([]byte, error) { - if !area.Empty() { - part, ok := picture.(interface { - SubImage(image.Rectangle) image.Image - }) - if !ok { - return nil, errors.New("recognize_text: the screenshot cannot be cropped") - } - picture = part.SubImage(area) - } - var out bytes.Buffer - if err := png.Encode(&out, picture); err != nil { - return nil, err - } - return out.Bytes(), nil -} - -// visionText runs visionScript on each image in a private temporary -// directory that is removed afterwards. A line's File is the index of its -// image. -var visionText = func(ctx context.Context, images [][]byte) ([]recognizedLine, error) { - directory, err := os.MkdirTemp("", "linger-text-") - if err != nil { - return nil, err - } - defer os.RemoveAll(directory) - arguments := []string{"-l", "JavaScript", "-e", visionScript} - for i, pixels := range images { - path := filepath.Join(directory, fmt.Sprintf("page-%d.png", i)) - if err := os.WriteFile(path, pixels, 0o600); err != nil { - return nil, err - } - arguments = append(arguments, path) - } - output, err := exec.CommandContext(ctx, "/usr/bin/osascript", arguments...).Output() - if err != nil { - return nil, fmt.Errorf("recognize_text: macOS text recognition failed: %w", err) - } - var lines []recognizedLine - for _, row := range strings.Split(strings.TrimSpace(string(output)), "\n") { - var line recognizedLine - if row != "" && json.Unmarshal([]byte(row), &line) == nil && line.Text != "" { - lines = append(lines, line) - } - } - return lines, nil -} diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_text_darwin_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_text_darwin_test.go deleted file mode 100644 index 113c9ce2..00000000 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_text_darwin_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package localtools - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "image" - "image/png" - "path/filepath" - "strings" - "testing" - "time" -) - -// pagedReadSession stubs the lock screen check and Vision, and reports the -// size of each image Vision was given. -func pagedReadSession(t *testing.T, mode string) (*cuaSession, string, *[]image.Rectangle) { - t.Helper() - locked, vision, settle := screenLocked, visionText, recognizePageSettle - t.Cleanup(func() { screenLocked, visionText, recognizePageSettle = locked, vision, settle }) - screenLocked = func() bool { return false } - recognizePageSettle = 0 - var sizes []image.Rectangle - visionText = func(_ context.Context, images [][]byte) ([]recognizedLine, error) { - lines := []recognizedLine{} - for i, encoded := range images { - picture, err := png.Decode(bytes.NewReader(encoded)) - if err != nil { - return nil, err - } - sizes = append(sizes, picture.Bounds()) - lines = append(lines, recognizedLine{File: i, Text: "page text", Confidence: 1}) - } - return lines, nil - } - marker := filepath.Join(t.TempDir(), "calls.jsonl") - return cuaFixtureSession(t, mode, marker), marker, &sizes -} - -func TestRecognizeTextReadsPagesUntilAKeyChangesNothing(t *testing.T) { - c, marker, sizes := pagedReadSession(t, "pages") - result, err := c.call(t.Context(), "recognize_text", json.RawMessage(`{"pid":42,"window_id":9,"pages":5,"next_key":"right"}`)) - if err != nil { - t.Fatal(err) - } - text := string(result) - // Pages 1-3 differ; the third key changes nothing, so a fourth is never read. - if !strings.Contains(text, "3 page(s)") || !strings.Contains(text, "## Page 3") || !strings.Contains(text, "Stopped after page 3: the right key changed nothing") { - t.Fatalf("result = %s", text) - } - // Only the changed pixels, from (10,6) to (14,6), are read on each page. - if len(*sizes) != 3 || (*sizes)[0].Dx() != 5 || (*sizes)[0].Dy() != 1 || !strings.Contains(text, "x=10 y=6 5x1") { - t.Fatalf("Vision images = %v", *sizes) - } - if calls := fixtureCalls(t, marker); len(calls) != 3 { - t.Fatalf("keys sent = %v, want 3", calls) - } -} - -func TestRecognizeTextKeepsThePagesReadBeforeARefusedKey(t *testing.T) { - c, _, sizes := pagedReadSession(t, "pages-refused") - result, err := c.call(t.Context(), "recognize_text", json.RawMessage(`{"pid":42,"window_id":9,"pages":5,"next_key":"right"}`)) - if err != nil { - t.Fatal(err) - } - if text := string(result); !strings.Contains(text, "2 page(s)") || !strings.Contains(text, "Stopped after page 2: the right key was refused") || len(*sizes) != 2 { - t.Fatalf("result = %s, images %v", text, *sizes) - } -} - -func TestRecognizeTextChecksTheRegionBeforeAnyKey(t *testing.T) { - c, marker, _ := pagedReadSession(t, "pages") - _, err := c.call(t.Context(), "recognize_text", json.RawMessage(`{"pid":42,"window_id":9,"pages":5,"next_key":"right","region":{"x":100,"y":100,"width":5,"height":5}}`)) - if !errors.Is(err, errNotSent) || len(fixtureCalls(t, marker)) != 0 { - t.Fatalf("err = %v, keys %v", err, fixtureCalls(t, marker)) - } - // A region inside the window is read as given on every page. - c, _, sizes := pagedReadSession(t, "pages") - result, err := c.call(t.Context(), "recognize_text", json.RawMessage(`{"pid":42,"window_id":9,"pages":2,"next_key":"right","region":{"x":0,"y":0,"width":20,"height":10}}`)) - if err != nil || !strings.Contains(string(result), "read in the given region") || len(*sizes) != 2 || (*sizes)[1] != image.Rect(0, 0, 20, 10) { - t.Fatalf("result = %s err=%v images %v", result, err, *sizes) - } -} - -func TestRecognizeTextStopsPagingWhenTheBudgetRunsOut(t *testing.T) { - c, marker, _ := pagedReadSession(t, "pages") - budget := recognizePagingBudget - t.Cleanup(func() { recognizePagingBudget = budget }) - recognizePagingBudget = -time.Second - result, err := c.call(t.Context(), "recognize_text", json.RawMessage(`{"pid":42,"window_id":9,"pages":5,"next_key":"right"}`)) - if err != nil || !strings.Contains(string(result), "the time for one call ran out") || len(fixtureCalls(t, marker)) != 0 { - t.Fatalf("result = %s err=%v", result, err) - } -} - -func TestRecognizeTextRecordsAnUnknownKeyOutcome(t *testing.T) { - c, marker, _ := pagedReadSession(t, "pages-hang") - ctx, cancel := context.WithCancel(t.Context()) - done := make(chan error, 1) - go func() { - _, err := c.call(ctx, "recognize_text", json.RawMessage(`{"pid":42,"window_id":9,"pages":3,"next_key":"right"}`)) - done <- err - }() - for deadline := time.Now().Add(5 * time.Second); len(fixtureCalls(t, marker)) == 0; time.Sleep(time.Millisecond) { - if time.Now().After(deadline) { - t.Fatal("the key was not sent") - } - } - cancel() - err := <-done - if !errors.Is(err, errOutcomeUnknown) || !strings.Contains(err.Error(), "1 right key(s) were sent") { - t.Fatalf("err = %v, want an unknown outcome that names the key sent", err) - } - if !c.op.manager.journal.blocked() { - t.Fatal("a key with an unknown outcome left the journal open for new input") - } -} - -func TestRecognizeTextRefusesFieldsTheReviewDoesNotRead(t *testing.T) { - c, marker, _ := pagedReadSession(t, "pages") - _, err := c.call(t.Context(), "recognize_text", json.RawMessage(`{"pid":42,"window_id":9,"pages":3,"next_key":"right","PID":7}`)) - if !errors.Is(err, errNotSent) || len(fixtureCalls(t, marker)) != 0 { - t.Fatalf("err = %v, keys %v", err, fixtureCalls(t, marker)) - } -} diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_text_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_text_test.go deleted file mode 100644 index ce673f5a..00000000 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_text_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package localtools - -import ( - "bytes" - "image" - "image/color" - "image/png" - "testing" -) - -func TestEncodeAreaCropsToTheArea(t *testing.T) { - picture := image.NewRGBA(image.Rect(0, 0, 40, 30)) - for _, test := range []struct { - area image.Rectangle - width, height int - }{ - {image.Rect(30, 20, 40, 30), 10, 10}, - {image.Rectangle{}, 40, 30}, - } { - encoded, err := encodeArea(picture, test.area) - if err != nil { - t.Fatal(err) - } - part, err := png.Decode(bytes.NewReader(encoded)) - if err != nil || part.Bounds().Dx() != test.width || part.Bounds().Dy() != test.height { - t.Fatalf("%v: bounds=%v err=%v", test.area, part.Bounds(), err) - } - } -} - -func TestChangedAreaBoundsTheDifferingPixels(t *testing.T) { - blank := image.NewRGBA(image.Rect(0, 0, 40, 30)) - page := image.NewRGBA(blank.Bounds()) - page.Set(5, 6, color.White) - page.Set(20, 12, color.White) - if got := changedArea(blank, page); got != image.Rect(5, 6, 21, 13) { - t.Fatalf("area = %v", got) - } - // Faint noise is not a new page. - noise := image.NewRGBA(blank.Bounds()) - noise.Set(5, 6, color.RGBA{8, 8, 8, 255}) - if got := changedArea(blank, noise); !got.Empty() { - t.Fatalf("noise area = %v", got) - } -} diff --git a/apps/computer/local-daemon/internal/linger/localtools/lanes_test.go b/apps/computer/local-daemon/internal/linger/localtools/lanes_test.go index 4ddccc52..d6ef1480 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/lanes_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/lanes_test.go @@ -425,8 +425,6 @@ func TestCuaInputNeedsFocusUnlessBackgroundToAnotherApp(t *testing.T) { darwin bool }{ {"get_window_state", `{"pid":42}`, false}, - {"recognize_text", `{"pid":42}`, false}, - {"recognize_text", `{"pid":42,"pages":3}`, true}, {"click", `{"pid":42,"window_id":3,"element_token":"s1:5"}`, false}, {"double_click", `{"element_token":"s1:5"}`, true}, {"scroll", `{"pid":9}`, false}, diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 8f95f44b..bc0d9975 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -992,7 +992,6 @@ func TestLocalToolCuaMacAgentChromeServesOneTaskAndNeverComesForward(t *testing. {"launch_app", `{"bundle_id":"com.google.Chrome","urls":["https://example.com/"]}`}, {"launch_app", `{"name":"Google Chrome","urls":["https://example.com/"]}`}, {"run_script", `{"script":"return 1"}`}, - {"recognize_text", `{"pid":42,"window_id":9,"pages":3,"next_key":"right"}`}, } { if _, err := c.call(t.Context(), call.name, json.RawMessage(call.args)); err == nil || !strings.Contains(err.Error(), "COMPUTER_BUSY") { t.Fatalf("%s err=%v, want busy", call.name, err) @@ -1951,7 +1950,6 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t {"press_key", `{"pid":42,"window_id":12,"key":"return","element_token":"s1:1","modifiers":[]}`}, {"press_key", `{"pid":42,"window_id":12,"key":"cmd+r","element_token":"s1:1"}`}, {"press_key", `{"pid":42,"window_id":9,"key":"return","element_token":"s9:1"}`}, - {"recognize_text", `{"pid":42,"window_id":12,"pages":2,"next_key":"pagedown"}`}, // A token never read leaves the named pid. {"press_key", `{"pid":42,"key":"return","element_token":"s7:1"}`}, } { diff --git a/apps/computer/local-daemon/internal/permission/allowlist.go b/apps/computer/local-daemon/internal/permission/allowlist.go index 73c3e584..1222b0c6 100644 --- a/apps/computer/local-daemon/internal/permission/allowlist.go +++ b/apps/computer/local-daemon/internal/permission/allowlist.go @@ -13,8 +13,7 @@ import ( // effect stays on this screen: reading it, or moving the pointer, scrolling, // hovering, going back a page, and arranging windows. Clicking, typing, keys, // dragging, navigation to a new address, uploads, and dialogs are judged. -// recognize_text is read-only only for one page: a paged read presses its -// next_key between pages. report_blocker stops the task's own input to the +// report_blocker stops the task's own input to the // Computer's own browser, and rate_limited also cools the site down there for // every task on this Computer. var readOnlyLocalTools = map[string]bool{ @@ -55,12 +54,6 @@ func ReadOnlyLocalTool(tool, input string) bool { if readOnlyLocalTools[tool] { return true } - if tool == "mcp__computer__recognize_text" { - var arguments struct { - Pages int `json:"pages"` - } - return json.Unmarshal([]byte(input), &arguments) == nil && arguments.Pages <= 1 - } if tool == "mcp__browser__browser_tabs" { // Listing tabs and switching to an open one change nothing; opening // and closing tabs are judged. diff --git a/apps/computer/local-daemon/internal/permission/allowlist_test.go b/apps/computer/local-daemon/internal/permission/allowlist_test.go index d0a2bc89..6b8c115f 100644 --- a/apps/computer/local-daemon/internal/permission/allowlist_test.go +++ b/apps/computer/local-daemon/internal/permission/allowlist_test.go @@ -21,9 +21,6 @@ func TestLocalWriteToolJudgesEveryNonReadOnlyLocalAction(t *testing.T) { {"mcp__computer__click", `{"x":1,"y":2}`, true}, {"mcp__computer__type_text", `{"text":"hi"}`, true}, {"mcp__computer__scroll", `{}`, false}, - {"mcp__computer__recognize_text", `{"pid":1,"window_id":2}`, false}, - {"mcp__computer__recognize_text", `{"pid":1,"window_id":2,"pages":1}`, false}, - {"mcp__computer__recognize_text", `{"pid":1,"window_id":2,"pages":4,"next_key":"right"}`, true}, {"mcp__linger__send_message", `{}`, false}, {"Bash", `{"command":"ls"}`, false}, } diff --git a/apps/computer/local-daemon/internal/permission/gui_reuse.go b/apps/computer/local-daemon/internal/permission/gui_reuse.go index cab7d854..83191a2d 100644 --- a/apps/computer/local-daemon/internal/permission/gui_reuse.go +++ b/apps/computer/local-daemon/internal/permission/gui_reuse.go @@ -32,9 +32,8 @@ var reusableHotkeys = map[string]bool{ } // GUIReuseKey names the shape of a desktop navigation call: a movement or -// dismissal key, a tab or history shortcut, a scroll, a paged recognize_text -// (which has the shape of its next_key), or a run_actions batch made only of -// those. Calls of one shape share an allow verdict within a Run +// dismissal key, a tab or history shortcut, a scroll, or a run_actions batch +// made only of those. Calls of one shape share an allow verdict within a Run // (see Reviewer). Clicks, typing, dragging, scripts and other shortcuts have // no shape: each is judged on its own. func GUIReuseKey(tool, input string) (string, bool) { @@ -63,25 +62,6 @@ func GUIReuseKey(tool, input string) (string, bool) { if !ok { return "", false } - if name == "recognize_text" { - // The read itself is display-only; the keys between pages are what - // is judged, so the call shares the verdict of that key. - for field := range arguments { - switch field { - case "pid", "window_id", "region", "pages", "next_key": - default: - return "", false - } - } - key, _ := arguments["next_key"].(string) - step := map[string]any{"key": key} - for _, field := range []string{"pid", "window_id"} { - if value, present := arguments[field]; present { - step[field] = value - } - } - return stepReuseKey("press_key", step) - } return stepReuseKey(name, arguments) } diff --git a/apps/computer/local-daemon/internal/permission/gui_reuse_test.go b/apps/computer/local-daemon/internal/permission/gui_reuse_test.go index e5fb6257..67e6b831 100644 --- a/apps/computer/local-daemon/internal/permission/gui_reuse_test.go +++ b/apps/computer/local-daemon/internal/permission/gui_reuse_test.go @@ -29,9 +29,6 @@ func TestGUIReuseKeyNamesNavigationShapesOnly(t *testing.T) { {"no pid", "mcp__computer__click", `{"x":1,"y":2}`, ""}, {"foreground scroll", "mcp__computer__scroll", `{"pid":719,"window_id":433,"direction":"down","delivery_mode":"foreground"}`, ""}, {"batch with a foreground scroll", "mcp__computer__run_actions", `{"actions":[{"tool":"scroll","pid":719,"window_id":433,"direction":"down"},{"tool":"scroll","pid":719,"window_id":433,"direction":"down","delivery_mode":"foreground"}]}`, ""}, - {"paged text read has its key's shape", "mcp__computer__recognize_text", `{"pid":719,"window_id":433,"pages":4,"next_key":"right","region":{"x":1,"y":2,"width":3,"height":4}}`, "press_key:right@719/433"}, - {"paged text read with another field", "mcp__computer__recognize_text", `{"pid":719,"window_id":433,"pages":4,"next_key":"right","PID":999}`, ""}, - {"paged text read without a known key", "mcp__computer__recognize_text", `{"pid":719,"window_id":433,"pages":4,"next_key":"return"}`, ""}, {"background scroll", "mcp__computer__scroll", `{"pid":719,"window_id":433,"direction":"down","delivery_mode":"background"}`, "scroll@719/433"}, } { t.Run(test.name, func(t *testing.T) { diff --git a/skills/builtin/humanlike-browse/1.0.6/SKILL.md b/skills/builtin/humanlike-browse/1.0.6/SKILL.md index 352bf591..9d2e0515 100644 --- a/skills/builtin/humanlike-browse/1.0.6/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.6/SKILL.md @@ -55,8 +55,7 @@ return {title: first.label, rating: page.tree_markdown}; ## Read - Read with `get_window_state`. It returns the whole page the browser has loaded, including the part below the window, so text already on the page needs no scrolling. Pass `query` to narrow a long page. If a read says it was cut, read again narrower instead of guessing. -- Text inside pictures: `recognize_text`; in a picture viewer, `pages` with `next_key`. -- A state the tree does not show, such as liked or followed: look with `include_screenshot: true`. +- Text inside pictures, or a state the tree does not show, such as liked or followed: look with `include_screenshot: true`. - To repeat a flow over items, use `run_script`. A script there sends at most 20 inputs: return what it extracted, look at the window, then continue. Many sites open a result in a new tab; the script then reads that tab, so read the tab in front before the next step. - Stop scrolling when two reads add no new items. Read only what the task needs: one search per keyword, and never reopen an item already read. - Act only in the tabs this task opened. A click that opens a new tab is the task's tab. Close each item's tab with its tab's Close button after reading it, and the task's last tab when done. If a tab the task did not open is in front, stop and tell the user. From a92dd47419f889643e44b5698fd2e121607c772e Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 16:04:48 +0800 Subject: [PATCH 17/30] docs(computer): list the window fields a script read returns Cua's getState result carries pid and window_id, which the guard uses to return a script's reads for the token carry. Co-Authored-By: Claude Opus 5.5 --- .../local-daemon/internal/linger/localtools/cua_script.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index 5efeb152..5e6b9af2 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -17,7 +17,7 @@ import ( // Computer's rules: background input only, and a bounded result. const macScriptDescription = "Run a JavaScript program that drives apps on this Mac in ONE call: loops, waits, reads and checks between inputs, and only its return value comes back. Use it to repeat a flow over several items, such as open each result, read it and close it, instead of one model turn per input. The script is the body of an async function: use await and return a JSON value. The only API is cua:\n" + "- const app = await cua.getApp({pid, windowId}) or cua.getApp(\"Google Chrome\"); cua.sleep(ms), where ms must be a whole number: a fraction returns at once without waiting, and in the Computer's own browser one sleep lasts at most 1.5 s: wait for a page with app.waitFor; cua.listWindows({pid})\n" + - "- app.getState({query?, max_elements?, timeout_ms?, since?: \"latest\", tree_format?: \"elements\"}) returns {tree_markdown, window_title, snapshot_id, truncated?, elements?}; rows read as in get_window_state. It takes no screenshot, which costs seconds on a busy page, unless you pass include_screenshot: true. In a script the walk stops after 250 nodes and 1000 ms unless you pass max_elements and timeout_ms, which cuts a browser page: pass about 3000 and 4000 there\n" + + "- app.getState({query?, max_elements?, timeout_ms?, since?: \"latest\", tree_format?: \"elements\"}) returns {tree_markdown, window_title, snapshot_id, pid, window_id, truncated?, elements?}; rows read as in get_window_state. It takes no screenshot, which costs seconds on a busy page, unless you pass include_screenshot: true. In a script the walk stops after 250 nodes and 1000 ms unless you pass max_elements and timeout_ms, which cuts a browser page: pass about 3000 and 4000 there\n" + "- app.query({role?, name?}) finds named elements {element_token, role, label, frame}; app.waitFor({role?, name?, text?, gone?, timeout_ms?}) waits, at most 10000 ms, until the window shows (or no longer shows) them; text also matches static text\n" + "- app.click(t), doubleClick(t), typeText(text, t?), pressKey(\"escape\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, \"down\", amount?), setValue(t, value), where t is {role, name, nth?}, an element_token, a row number of this app's last getState(), or [x, y] pixels of its last screenshot\n" + "- cua.call(tool, args) for click, press_key, type_text, scroll, get_window_state or list_windows with their usual arguments. A script opens no app or URL: call launch_app as its own tool first.\n" + From f7441bc04a0874f7a3b10a2d5b2fab9c3a5eedb4 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 16:05:32 +0800 Subject: [PATCH 18/30] docs(computer): state which fields the script guard keeps a read by Co-Authored-By: Claude Opus 5.5 --- .../local-daemon/internal/linger/localtools/cua_script.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index 5e6b9af2..eea19e6d 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -70,7 +70,9 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // screenshot unless the script asks for one: Cua's default captures one per // read, which took seconds on a busy page. The guard returns the window reads // made after the script's last input beside its value, under scriptReadsKey, -// for scriptResultReads to take out. A positive inputLimit paces the script for the +// for scriptResultReads to take out. It keeps a read by the pid and window_id +// that Cua puts in every getState and get_window_state result, as in a tool +// read, so a script read lands on the same window key as the Computer's own. A positive inputLimit paces the script for the // Computer's own browser: the guard waits a person's pause before each input, // throws once the script tries to send more than inputLimit inputs, and // refuses to open an app or URL, whose site checks run only in the Daemon's From 75872f5c56e9f4cb944aaf5abfe2b9bf8db45dfb Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 16:12:59 +0800 Subject: [PATCH 19/30] fix(skills): ship humanlike-browse as 1.0.7 An environment that installed the PR's 1.0.6 sealed it, and sealed Manifests cannot be changed or deleted, so the edits since then go out as the PR's one new version, 1.0.7. Co-Authored-By: Claude Opus 5.5 --- skills/builtin/catalog.yaml | 8 ++++---- skills/builtin/humanlike-browse/{1.0.6 => 1.0.7}/SKILL.md | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename skills/builtin/humanlike-browse/{1.0.6 => 1.0.7}/SKILL.md (100%) diff --git a/skills/builtin/catalog.yaml b/skills/builtin/catalog.yaml index 81fed2fd..c9da046f 100644 --- a/skills/builtin/catalog.yaml +++ b/skills/builtin/catalog.yaml @@ -219,14 +219,14 @@ skills: version: 1.0.0 package_path: builtin/personal-profile/1.0.0 - skill_id: simplexai_humanlike_browse - bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_6 + bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_7 default_agent_types: - anotherme - general manifests: - - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_6 - version: 1.0.6 - package_path: builtin/humanlike-browse/1.0.6 + - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_7 + version: 1.0.7 + package_path: builtin/humanlike-browse/1.0.7 - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_4 version: 1.0.4 package_path: builtin/humanlike-browse/1.0.4 diff --git a/skills/builtin/humanlike-browse/1.0.6/SKILL.md b/skills/builtin/humanlike-browse/1.0.7/SKILL.md similarity index 100% rename from skills/builtin/humanlike-browse/1.0.6/SKILL.md rename to skills/builtin/humanlike-browse/1.0.7/SKILL.md From c9c2fa2b6428178ba5f0eeae862d8bcaa6c2de70 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 16:33:23 +0800 Subject: [PATCH 20/30] test(computer): give the cancelled-start cleanup its full budget The test shortened the cleanup budget to 2 s, but cleanup must finish inside it, and a loaded CI runner took longer and reported the exit as unconfirmed. A cleanup that waits for a turn that cannot exist still exhausts the default budget, so the test keeps its meaning. Co-Authored-By: Claude Opus 5.5 --- .../local-daemon/internal/provider/native/codex_runtime_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/computer/local-daemon/internal/provider/native/codex_runtime_test.go b/apps/computer/local-daemon/internal/provider/native/codex_runtime_test.go index e4ec818c..4c679cbb 100644 --- a/apps/computer/local-daemon/internal/provider/native/codex_runtime_test.go +++ b/apps/computer/local-daemon/internal/provider/native/codex_runtime_test.go @@ -841,8 +841,6 @@ func jsonText(value any) string { func TestCodexCancelledStartOnResidentKeepsHomeUsable(t *testing.T) { fake := newCodexFake(t, []codexFakeTurn{{Status: "completed", Items: []map[string]any{codexAgentMessage("m", "ok", nil)}}}, nil) - previous := codexCleanupBudget.set(2 * time.Second) - t.Cleanup(func() { codexCleanupBudget.set(previous) }) rt := fake.runtime(nil) spec := fake.spec() out, _ := codexDrain(t, rt, "park", spec) From d8f56514946446eb237eac88f4695e67c5c546e3 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 17:23:47 +0800 Subject: [PATCH 21/30] feat(computer): tell the Agent what an input changed in its browser Taobao runs spent turns on four things the tools did not say: - A click result never said a link opened a tab, so the Agent listed windows and took screenshots to learn where the page went. The tab read after each input now reports a tab opened or closed, or a new front tab title, with no extra round trip, and names the Close button and the read that reaches the tab strip past the page. - Cua warns that a press on page text may have done nothing, but Chrome clicks it; the Agent then clicked the review sort menu by x,y, which missed for two minutes. Such a click now says so. - A role and name target missed a label with a no-break space; white space now matches any white space. - The script API now says that query matches part of a label, and that a row number belongs to a getState earlier in the same script. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 8 +- .../linger/localtools/agent_browser_test.go | 5 +- .../linger/localtools/agent_chrome.go | 84 +++++++++++++------ .../internal/linger/localtools/cua.go | 24 +++++- .../internal/linger/localtools/cua_script.go | 4 +- .../linger/localtools/cua_snapshots.go | 18 ++++ .../linger/localtools/cua_snapshots_test.go | 12 +++ .../linger/localtools/localtools_test.go | 32 +++++-- .../internal/linger/localtools/manager.go | 4 +- 9 files changed, 150 insertions(+), 41 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 8582e206..9a1d7be0 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -312,7 +312,7 @@ changes in the same result. By default it reads the last step's window, or the application's focused window when that step gave no `window_id`; pass `read` to read another window or add a `query`. When a step is refused or its result is unknown, the result lists the completed, unsent and possibly applied steps. -An `element_token` from an older read is refused, never rebound to another element with the same role and label. A step can name its target by role and name instead; the Daemon finds it when the step runs. +An `element_token` from an older read is refused, never rebound to another element with the same role and label. A step can name its target by role and name instead; the Daemon finds it when the step runs, and white space in a label, such as a no-break space, matches any white space. On Windows, after a successful input the Daemon waits 100 ms before reading the window diff, because Cua already waits for its own read-back after input. The MCP initialize result carries no usage instructions, because Codex prepends them to @@ -379,7 +379,11 @@ A change in the number of tabs between the task's inputs stops the task with `COMPUTER_TAB_CHANGED`, because the next input could reach another page. A tab that the task's own input or script opens or closes is the task's: the task holds the browser's input lease, and the count is kept once the tab strip -settles, since the browser shows such a tab a moment later. Reading the tab +settles, since the browser shows such a tab a moment later. The input's +result then says, from that same read, when it opened or closed a tab or +changed the title of the tab in front. A click there on page text whose row +lists no press notes that Chrome clicks it anyway, since Cua warns that such +a press may have done nothing. Reading the tab strip replaces Cua's snapshot of the window, so the Daemon remembers the rows of recent reads and moves an Agent's element token, in an input or written into a script, to the current snapshot when its row reads the same there. diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go index 343877de..7e1a0556 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go @@ -15,8 +15,9 @@ func TestCountTabsReadsChromeEdgeAndFirefoxStrips(t *testing.T) { "safari": " - [17] AXRadioButton \"Trail checklist - Pinboard\" [id=TabBarTab?isPinned=false&isNarrow=false&isActive=false actions=[press,name:close tab]]\n" + " - [18] AXRadioButton \"Start Page\" [id=TabBarTab?isPinned=false&isNarrow=false&isActive=true actions=[press,name:close tab]]", } { - if got := countTabs(tree); got != 2 { - t.Errorf("%s: %d tabs, want 2", name, got) + want := map[string]string{"chrome": "Close", "edge": "Close tab", "firefox": "Close tab"}[name] + if got, close := tabStrip(tree); got != 2 || close != want { + t.Errorf("%s: %d tabs closed by %q, want 2 closed by %q", name, got, close, want) } } // A page's own radio buttons are not tabs. diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index ff2cc341..36c8f9b9 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -577,24 +577,32 @@ func (c *cuaSession) agentChromeKeys(ctx context.Context, arguments map[string]a // It returns 0 when the read fails or is cut short: the tab strip follows the // page in the tree, so a cut read may miss it. func (c *cuaSession) agentChromeTabs(ctx context.Context, pid int, window float64) int { + count, _, _ := c.agentChromeTabsAndTitle(ctx, pid, window) + return count +} + +// agentChromeTabsAndTitle is agentChromeTabs with the title of the window's +// front tab and the label of a tab's Close button, from the same read. +func (c *cuaSession) agentChromeTabsAndTitle(ctx context.Context, pid int, window float64) (int, string, string) { read, err := c.client.request(ctx, "tools/call", map[string]any{"name": "get_window_state", "arguments": map[string]any{ "pid": pid, "window_id": window, "include_screenshot": false, "max_elements": 6000, "timeout_ms": 6000, "session": c.label}}) if err != nil { slog.Warn("agent_browser.tabs_unread", "pid", pid, "window_id", window, "error", err) - return 0 + return 0, "", "" } var state struct { Structured struct { Tree string `json:"tree_markdown"` + Title string `json:"window_title"` Truncated bool `json:"truncated"` Degraded string `json:"degraded_reason"` } `json:"structuredContent"` } if err := json.Unmarshal(read, &state); err != nil || state.Structured.Truncated { slog.Warn("agent_browser.tabs_unread", "pid", pid, "window_id", window, "truncated", state.Structured.Truncated, "error", err) - return 0 + return 0, "", "" } - count := countTabs(state.Structured.Tree) + count, close := tabStrip(state.Structured.Tree) if count == 0 && strings.Contains(state.Structured.Tree, "id=WEB_BROWSER_ADDRESS_AND_SEARCH_FIELD") { // Safari hides its tab strip while a window has one tab. count = 1 @@ -602,15 +610,21 @@ func (c *cuaSession) agentChromeTabs(ctx context.Context, pid int, window float6 if count == 0 && state.Structured.Degraded != "" { slog.Warn("agent_browser.tabs_unread", "pid", pid, "window_id", window, "reason", state.Structured.Degraded) } - return count + return count, state.Structured.Title, close } // countTabs counts the tab rows of a browser window's tree: a radio button // outside the page that holds a button, its Close button. func countTabs(tree string) int { + count, _ := tabStrip(tree) + return count +} + +// tabStrip is countTabs with the label of a tab's Close button, which follows +// the user's language; it is empty for a Safari tab, which has none. +func tabStrip(tree string) (count int, close string) { rows := strings.Split(tree, "\n") indent := func(row string) int { return len(row) - len(strings.TrimLeft(row, " ")) } - count := 0 // web is the indent of the page being skipped, or -1: a radio button // in a page is not a tab. web := -1 @@ -641,11 +655,14 @@ func countTabs(tree string) int { } if trimmed := strings.TrimSpace(next); strings.Contains(trimmed, "] AXButton") && strings.HasSuffix(trimmed, "[actions=[press]]") { count++ + if _, label, ok := strings.Cut(trimmed, "] AXButton \""); ok && close == "" { + close, _, _ = strings.Cut(label, "\"") + } break } } } - return count + return count, close } // agentChromeTabWindow names the window whose tabs hold the task: the one @@ -697,40 +714,59 @@ func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) // script, so a tab it opened or closed, such as a link to a new tab, is the // task's. While it runs, the task holds this browser's input lease, so no // other task changes the tabs. Only a task that opened its tab with -// launch_app holds a count. -func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any) { +// launch_app holds a count. It returns a note on what the input changed in +// the window, from the same read: a click result says nothing of where the +// page went, and the Agent read the window only to learn that. +func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any) string { c.op.mu.Lock() - held := c.op.chromeTabs > 0 + before, title := c.op.chromeTabs, c.op.chromeTitle c.op.mu.Unlock() - if !held { - return + if before == 0 { + return "" } - if got := c.settledAgentChromeTabs(ctx, pid, window); got > 0 { - c.op.mu.Lock() - c.op.chromeTabs = got - c.op.mu.Unlock() + got, front, close := c.settledAgentChromeTabs(ctx, pid, window) + if got == 0 { + return "" } + c.op.mu.Lock() + c.op.chromeTabs, c.op.chromeTitle = got, front + c.op.mu.Unlock() + switch { + case got > before && close != "": + // The tab strip follows the page in the tree, past the default + // walk, so the Agent's search for the button came back empty. + return fmt.Sprintf("This input opened %d new tab(s); the one in front shows %q. To go back, close it with the %q button inside its tab: the tab strip follows the page in the tree, so read it with get_window_state query %q and max_elements 6000, or app.getState with the same in a script.", got-before, front, close, close) + case got > before: + return fmt.Sprintf("This input opened %d new tab(s); the one in front shows %q.", got-before, front) + case got < before: + return fmt.Sprintf("This input closed %d tab(s); the tab in front shows %q.", before-got, front) + case front != title: + return fmt.Sprintf("The tab in front now shows %q.", front) + } + return "" } -// settledAgentChromeTabs reads the window's tab count once two reads a -// settle apart agree: the browser adds or drops a tab a moment after the -// input that opened or closed it returns, and the next input's pause already -// covers that wait. It is 0 when the strip cannot be read. -func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any) int { +// settledAgentChromeTabs reads the window's tab count, its front tab's title +// and the label of a tab's Close button once two reads a settle apart agree: the browser adds or drops a tab +// a moment after the input that opened or closed it returns, and the next +// input's pause already covers that wait. The count is 0 when the strip +// cannot be read. +func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any) (int, string, string) { number, id, ok := c.agentChromeTabWindow(pid, window) if !ok { - return 0 + return 0, "", "" } - got := c.agentChromeTabs(ctx, number, id) + got, title, close := c.agentChromeTabsAndTitle(ctx, number, id) for range 3 { if got == 0 || sleepContext(ctx, inputSettle) != nil { break } - next := c.agentChromeTabs(ctx, number, id) + next, nextTitle, nextClose := c.agentChromeTabsAndTitle(ctx, number, id) + title, close = nextTitle, nextClose if next == got { break } got = next } - return got + return got, title, close } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 3e0eca62..121910ea 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -215,7 +215,7 @@ const MacToolGuide = `Computer tools on this Mac (MCP server "computer"): {tool: "click" | "double_click" | "right_click", pid, window_id, element_token, modifier?: ["cmd"]} on a row of the latest read; cmd-clicks add rows to a selection, so one cmd+c copies several files; {tool: "hotkey", pid, keys: ["cmd", "s"]}, {tool: "press_key", pid, key: "return"}, {tool: "type_text", pid, text} go to the app's focused field, such as a sheet, panel or rename field; a new Save sheet's name field already has focus with its text selected, so type the name without clicking, and text fields refuse click; {tool: "scroll", pid, window_id, direction}. - A click, scroll or type_text, in a step or alone, can name its target with role and name instead of element_token: the one element with that role whose label or value is exactly name, or nth (from 0) when several match. It is found when it runs, so one batch can cross pages; finding it reads the window, which stales the tokens of your earlier read, so later steps name their targets the same way. + A click, scroll or type_text, in a step or alone, can name its target with role and name instead of element_token: the one element with that role whose label or value is exactly name, apart from white space, or nth (from 0) when several match. It is found when it runs, so one batch can cross pages; finding it reads the window, which stales the tokens of your earlier read, so later steps name their targets the same way. - run_script({script}) runs JavaScript that loops, waits and reads between inputs and returns only what you extract. Use it to repeat a flow over several items, such as open each result, read it and close it, in one call instead of a turn per item. - bring_to_front({pid, window_id}) raises one window. - report_blocker({kind, url}) as soon as a page in the Computer's own browser shows a check only the user may pass (captcha, verification, login_required, rate_limited) or one item that cannot be viewed (content_unavailable). Never try to pass the check. @@ -526,8 +526,16 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage // check, replaced the snapshot the Agent's token names. c.snapshots.remap(arguments) } + token, _ := arguments["element_token"].(string) + pagePress := agentChrome && name == "click" && c.snapshots != nil && c.snapshots.unpressable(token) driverStarted := time.Now() result, err := c.sendInput(ctx, name, arguments, appKeys, agentChrome) + if err == nil && pagePress && !failed(result) { + // Chrome clicks a page element on a press even where the element + // lists no press, as the sort menu of a review list did; the Agent + // took Cua's warning as a miss and clicked by x,y, which missed. + result = withNote(result, "In the Computer's own browser, a press on page text clicks it as a person would, even where the row lists no press: check its effect in the next read, and do not repeat it by x,y.") + } var scriptReads []windowRead if err == nil && name == "run_script" { result, scriptReads = scriptResultReads(result) @@ -551,7 +559,9 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage } // A script that throws has still sent the inputs before the throw. if paced && (err == nil || name == "run_script") { - c.recordAgentChromeTabs(ctx, target["pid"], target["window_id"]) + if note := c.recordAgentChromeTabs(ctx, target["pid"], target["window_id"]); note != "" && result != nil { + result = withNote(result, note) + } } return c.finishInput(ctx, name, arguments, appKeys, result, err) } @@ -1333,8 +1343,10 @@ func (c *cuaSession) locate(ctx context.Context, name string, arguments map[stri return fmt.Errorf("COMPUTER_TARGET_UNREADABLE: window %v was read only in part, so a %s labeled %q cannot be proven to be the only one; use an element_token from a read: %w", arguments["window_id"], role, label, errNotSent) } var matches []string + label = sameSpaces(label) for _, element := range state.Structured.Elements { - if element.Role == role && (element.Label == label || element.Value == label) { + value, _ := element.Value.(string) + if element.Role == role && (sameSpaces(element.Label) == label || sameSpaces(value) == label) { matches = append(matches, element.Token) } } @@ -1354,6 +1366,12 @@ func (c *cuaSession) locate(ctx context.Context, name string, arguments map[stri return nil } +// sameSpaces writes every run of white space as one space: a page label may +// hold a no-break space, which reads the same as the space the Agent writes. +func sameSpaces(text string) string { + return strings.Join(strings.Fields(text), " ") +} + // listedWindow is one window from Cua list_windows. type listedWindow struct { WindowID float64 `json:"window_id"` diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index eea19e6d..d6090661 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -18,8 +18,8 @@ import ( const macScriptDescription = "Run a JavaScript program that drives apps on this Mac in ONE call: loops, waits, reads and checks between inputs, and only its return value comes back. Use it to repeat a flow over several items, such as open each result, read it and close it, instead of one model turn per input. The script is the body of an async function: use await and return a JSON value. The only API is cua:\n" + "- const app = await cua.getApp({pid, windowId}) or cua.getApp(\"Google Chrome\"); cua.sleep(ms), where ms must be a whole number: a fraction returns at once without waiting, and in the Computer's own browser one sleep lasts at most 1.5 s: wait for a page with app.waitFor; cua.listWindows({pid})\n" + "- app.getState({query?, max_elements?, timeout_ms?, since?: \"latest\", tree_format?: \"elements\"}) returns {tree_markdown, window_title, snapshot_id, pid, window_id, truncated?, elements?}; rows read as in get_window_state. It takes no screenshot, which costs seconds on a busy page, unless you pass include_screenshot: true. In a script the walk stops after 250 nodes and 1000 ms unless you pass max_elements and timeout_ms, which cuts a browser page: pass about 3000 and 4000 there\n" + - "- app.query({role?, name?}) finds named elements {element_token, role, label, frame}; app.waitFor({role?, name?, text?, gone?, timeout_ms?}) waits, at most 10000 ms, until the window shows (or no longer shows) them; text also matches static text\n" + - "- app.click(t), doubleClick(t), typeText(text, t?), pressKey(\"escape\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, \"down\", amount?), setValue(t, value), where t is {role, name, nth?}, an element_token, a row number of this app's last getState(), or [x, y] pixels of its last screenshot\n" + + "- app.query({role?, name?}) finds the elements whose label holds name, ignoring case, {element_token, role, label, frame}; app.waitFor({role?, name?, text?, gone?, timeout_ms?}) waits, at most 10000 ms, until the window shows (or no longer shows) them; text also matches static text\n" + + "- app.click(t), doubleClick(t), typeText(text, t?), pressKey(\"escape\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, \"down\", amount?), setValue(t, value), where t is {role, name, nth?}, whose name must equal the label exactly, spaces included (for a long label, query a short part of it and click the element_token it returns), an element_token, a row number of a getState() earlier in this same script (a row of a get_window_state call is its element_token string), or [x, y] pixels of its last screenshot\n" + "- cua.call(tool, args) for click, press_key, type_text, scroll, get_window_state or list_windows with their usual arguments. A script opens no app or URL: call launch_app as its own tool first.\n" + "Inputs go to the app in the background, as the other computer tools do; a control that reacts only to a real pointer needs the click tool with delivery_mode \"foreground\". A failed call throws (catch it to recover), and the error names the line. timeout_ms is the wall-time limit (default 60000, at most 100000); max_calls bounds driver calls (default 100, at most 500). A Skill that ships a script names it with script_file and passes its inputs as args, read in the script as the constant args. Return only what you need, such as extracted text; the result is cut at 40000 characters and carries no screenshots: call get_window_state to look at the window." diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go index e0ea73d3..8901a527 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots.go @@ -174,6 +174,24 @@ func (s *snapshotRows) sent(name string) { s.mu.Unlock() } +// unpressable reports whether token names a remembered row that lists no +// press action, such as a web page's static text, which lists none; Cua then +// warns that the press may not have taken. +func (s *snapshotRows) unpressable(token string) bool { + snapshot, index, ok := strings.Cut(token, ":") + if !ok { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + row, ok := s.reads[snapshot].rows[index] + if !ok { + return false + } + _, actions, _ := strings.Cut(row, "[actions=[") + return !strings.Contains(actions, "press") +} + // carry returns the token on the current snapshot of its window, or the // token itself. func (s *snapshotRows) carry(token string) string { diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go index 955beff3..618066f1 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_snapshots_test.go @@ -157,3 +157,15 @@ func mustJSON(t *testing.T, value any) []byte { } return encoded } + +// A row that lists no press, such as a page's static text, is told apart +// from a link that lists one; a token the Computer does not remember is not. +func TestSnapshotRowsTellsTextFromAPressableRow(t *testing.T) { + rows := newSnapshotRows() + rows.record(windowRead{Snapshot: "s00000001", PID: 42, Window: 9, Tree: "- [2] AXWebArea \"Shop\"\n - [3] AXStaticText = \"Sort by time\"\n - [4] AXLink \"Kettle\" [actions=[press]]\n - [5] AXGroup [actions=[show_menu]]"}) + for token, want := range map[string]bool{"s00000001:3": true, "s00000001:5": true, "s00000001:4": false, "s00000001:9": false, "s00000002:3": false, "bad": false} { + if got := rows.unpressable(token); got != want { + t.Fatalf("unpressable(%s)=%v, want %v", token, got, want) + } + } +} diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index bc0d9975..a3f647f8 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -238,7 +238,7 @@ func runToolFixture(mode string) { if address == "" { address = "news.example.org/today" } - result = map[string]any{"content": []any{}, "structuredContent": map[string]any{"tree_markdown": tree.String(), "elements": []any{ + result = map[string]any{"content": []any{}, "structuredContent": map[string]any{"tree_markdown": tree.String(), "window_title": fmt.Sprintf("Tab %d", tabs-1), "elements": []any{ map[string]any{"element_token": "s1:1", "role": "AXTextField", "label": "Address and search bar", "value": address}, }}} } @@ -329,6 +329,7 @@ func runToolFixture(mode string) { map[string]any{"element_token": "s2:4", "role": "AXButton", "label": "Delete all"}, map[string]any{"element_token": "s2:5", "role": "AXButton", "label": "Send"}, map[string]any{"element_token": "s2:6", "role": "AXButton", "label": "Send"}, + map[string]any{"element_token": "s2:7", "role": "AXLink", "label": "Kettle\u00a0[snapshot]"}, }}} } break @@ -1125,21 +1126,34 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsOwnInputChangesLate(t *testing.T) t.Cleanup(func() { agentChromePID = original }) c.op.manager.agentChromeOwner = c.op c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = 1, 42, 9 + // The click below presses a row of page text, which lists no press. + c.snapshots = newSnapshotRows() + c.snapshots.record(windowRead{Snapshot: "s1", PID: 42, Window: 9, Tree: `- [1] AXStaticText = "Sort by time"`}) // A tab the task's own input opens or closes may show only after the // input returns; the task keeps the count the strip settles on. + // The input's result says what it changed in the window, and nothing + // when it changed neither the tabs nor the front tab. for _, step := range []struct { - keys string - want int - }{{`["cmd","t"]`, 2}, {`["cmd","w"]`, 1}} { - if _, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"pid":42,"window_id":9,"keys":`+step.keys+`}`)); err != nil { + keys, note string + want int + }{{`["cmd","t"]`, `This input opened 1 new tab(s); the one in front shows \"Tab 1\". To go back, close it with the \"Close\" button`, 2}, {`["cmd","w"]`, `This input closed 1 tab(s); the tab in front shows \"Tab 0\".`, 1}} { + result, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"pid":42,"window_id":9,"keys":`+step.keys+`}`)) + if err != nil { t.Fatal(err) } if c.op.chromeTabs != step.want { t.Fatalf("tabs after the task's own %s=%d, want %d", step.keys, c.op.chromeTabs, step.want) } - if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); err != nil { + if !strings.Contains(string(result), step.note) { + t.Fatalf("result after the task's own %s=%s, want note %s", step.keys, result, step.note) + } + result, err = c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)) + if err != nil { t.Fatalf("input after the task's own %s: %v", step.keys, err) } + if strings.Contains(string(result), "tab") || !strings.Contains(string(result), "a press on page text clicks it") { + t.Fatalf("result of a click on page text that changed no tab=%s", result) + } } if err := c.drain(); err != nil { t.Fatal(err) @@ -1528,6 +1542,10 @@ func TestLocalToolCuaMacInputsFindTheirTargetWhenTheyRun(t *testing.T) { if result, err := c.call(t.Context(), "type_text", json.RawMessage(`{"pid":42,"window_id":9,"role":"AXTextArea","name":"Comment","text":"hi"}`)); err != nil || failed(result) { t.Fatalf("located type_text result=%s err=%v", result, err) } + // A page label's no-break space matches the space the Agent writes. + if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"role":"AXLink","name":"Kettle [snapshot]"}`)); err != nil { + t.Fatalf("label with a no-break space: %v", err) + } if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"role":"AXButton","name":"Delete"}`)); err == nil || !strings.Contains(err.Error(), "COMPUTER_TARGET_NOT_FOUND") { t.Fatalf("partial label err=%v, want not found", err) } @@ -1559,7 +1577,7 @@ func TestLocalToolCuaMacInputsFindTheirTargetWhenTheyRun(t *testing.T) { } sent = append(sent, call.Name+" "+fmt.Sprint(call.Arguments["element_token"])) } - if want := []string{"click s1:15", "click s1:15", "type_text s2:3", "click s2:6"}; !slices.Equal(sent, want) { + if want := []string{"click s1:15", "click s1:15", "type_text s2:3", "click s2:7", "click s2:6"}; !slices.Equal(sent, want) { t.Fatalf("sent=%v, want %v", sent, want) } if err := c.drain(); err != nil { diff --git a/apps/computer/local-daemon/internal/linger/localtools/manager.go b/apps/computer/local-daemon/internal/linger/localtools/manager.go index 9ea9d4f6..e031fe87 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/manager.go +++ b/apps/computer/local-daemon/internal/linger/localtools/manager.go @@ -232,8 +232,10 @@ type operation struct { // chromeTabs is the tab count of the Computer's own browser after this // task's last input there, and chromePID and chromeWindow the browser // and window this task opened with launch_app; see checkAgentChromeTabs. - // Guarded by mu. + // chromeTitle is the front tab's title read with chromeTabs, empty until + // the task's first input there. Guarded by mu. chromeTabs int + chromeTitle string chromePID int chromeWindow float64 closeOnce sync.Once From 964052f684fe9a02533c7dcb50845d2170aaec92 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 17:28:21 +0800 Subject: [PATCH 22/30] fix(computer): keep the front tab title from launch_app The first input after launch_app compared the front tab's title with an empty one, so a click that changed nothing reported a new front tab, and a second launch_app kept the earlier window's title. Each launch path now stores the title with the tab count; the everyday Chrome path reads it once, since its window search keeps no title. Co-Authored-By: Claude Opus 5.5 --- .../internal/linger/localtools/agent_browser.go | 9 ++++++--- .../internal/linger/localtools/agent_chrome.go | 4 ++-- .../internal/linger/localtools/localtools_test.go | 9 +++++++-- .../local-daemon/internal/linger/localtools/manager.go | 4 ++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index 0102fe34..c4ed7799 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -247,9 +247,9 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls return nil, err } } - tabs := c.agentChromeTabs(ctx, pid, window.WindowID) + tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID) c.op.mu.Lock() - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = tabs, pid, window.WindowID + c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID c.op.mu.Unlock() note := "This is the Computer's own window in the user's Safari. Safari has no separate profile, so this window has the user's Safari sign-ins; work only in this window, never in the user's other Safari windows. It works while other windows cover it, but only on the desktop (Space) the user is on. Keys without an element go to this window in the background. Type into a page field with type_text and the field's element_token. Move between pages as a person does: through the page's search box, menus and links, back with cmd+[; never type an address into the address bar. Close a tab with its tab's close action, not cmd+w." if started { @@ -413,8 +413,11 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u return nil, err } m.setSharedWindow(b.Bundle, pid, window.WindowID) + // The front tab's title, so the first input reports only a change it + // made; the window search above keeps no title. + _, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID) c.op.mu.Lock() - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = tabs, pid, window.WindowID + c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID c.op.mu.Unlock() note := "This is the Computer's own window in the user's everyday Chrome, with the user's sign-ins; work only in this window, never in the user's other Chrome windows. It works while other windows cover it, but only on the desktop (Space) the user is on. The user's Chrome takes no keys from the Computer, because a key could reach the window the user is typing in: fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links. Close a tab with the Close button of its tab; closing the window's last tab closes the window. If a site asks to sign in, ask the user to sign in once in this window." if restored { diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 36c8f9b9..fec93667 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -195,9 +195,9 @@ func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls m.mu.Lock() m.agentChromePID, m.agentChromeChecked = pid, time.Now() m.mu.Unlock() - tabs = c.agentChromeTabs(ctx, pid, window.WindowID) + tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID) c.op.mu.Lock() - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = tabs, pid, window.WindowID + c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID c.op.mu.Unlock() opened := "No URL was opened, so no tab is this task's." if len(urls) > 0 { diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index a3f647f8..2e0729e4 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1988,8 +1988,13 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t if _, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"keys":["cmd","w"],"element_token":"s7:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_TARGET_REQUIRED") { t.Fatalf("hotkey with no pid: err=%v, want refused", err) } - if _, err := c.call(t.Context(), "set_value", json.RawMessage(`{"pid":42,"window_id":12,"value":"kettle","element_token":"s1:1"}`)); err != nil { - t.Fatalf("set_value: %v", err) + // launch_app keeps the front tab's title, so the first input, which + // changes no tab, reports no change. + if c.op.chromeTitle != "Tab 0" { + t.Fatalf("front title after launch_app=%q, want Tab 0", c.op.chromeTitle) + } + if result, err := c.call(t.Context(), "set_value", json.RawMessage(`{"pid":42,"window_id":12,"value":"kettle","element_token":"s1:1"}`)); err != nil || strings.Contains(string(result), "tab in front") { + t.Fatalf("set_value result=%s err=%v, want no tab note", result, err) } // Any script, whatever it targets, is guarded against keys to the // user's Chrome. diff --git a/apps/computer/local-daemon/internal/linger/localtools/manager.go b/apps/computer/local-daemon/internal/linger/localtools/manager.go index e031fe87..26393aee 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/manager.go +++ b/apps/computer/local-daemon/internal/linger/localtools/manager.go @@ -232,8 +232,8 @@ type operation struct { // chromeTabs is the tab count of the Computer's own browser after this // task's last input there, and chromePID and chromeWindow the browser // and window this task opened with launch_app; see checkAgentChromeTabs. - // chromeTitle is the front tab's title read with chromeTabs, empty until - // the task's first input there. Guarded by mu. + // chromeTitle is the front tab's title read with chromeTabs, from + // launch_app on. Guarded by mu. chromeTabs int chromeTitle string chromePID int From 5359bbff2b64f5771be27179df1fe32b416cd03f Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 19:54:36 +0800 Subject: [PATCH 23/30] perf(computer): read the browser's tab strip only as far as it needs Each input to the Computer's own browser cost about 2.2 s beyond the driver call: a read of the whole page before it and at least two after it, only to count tabs. Now: - Before an input, the strip is read only when the window's title in list_windows differs from the front tab's title after the last input. - After an input, the reads stop at the strip's depth, unless a script returned reads the Agent may act on, and a script that sent no input reads nothing, so the tokens its queries returned stay current. - A role and name target reads up to 6000 elements, as the tab reads do, so a large shop page is no longer refused as read only in part. On Taobao a click now takes about 1.3 s instead of 3.4 s. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 6 +- .../linger/localtools/agent_browser.go | 4 +- .../linger/localtools/agent_chrome.go | 84 +++++++++++++--- .../linger/localtools/agent_chrome_pace.go | 38 ++++---- .../internal/linger/localtools/cua.go | 15 ++- .../linger/localtools/localtools_test.go | 95 ++++++++++++++++++- 6 files changed, 203 insertions(+), 39 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 9a1d7be0..97fd7471 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -376,7 +376,11 @@ the task; `rate_limited` also puts the site (its registrable domain) in a Once a task opens a site there, the browser tools refuse that site for the rest of the task, and `launch_app` refuses to open it in another browser. A change in the number of tabs between the task's inputs stops the task with -`COMPUTER_TAB_CHANGED`, because the next input could reach another page. A +`COMPUTER_TAB_CHANGED`, because the next input could reach another page. The +Daemon reads the tab strip before an input only when the window's title in +`list_windows` differs from the front tab's title after the last input; after +an input it reads the strip only to its depth, unless a script returned reads +the Agent may act on, and not at all after a script that sent no input. A tab that the task's own input or script opens or closes is the task's: the task holds the browser's input lease, and the count is kept once the tab strip settles, since the browser shows such a tab a moment later. The input's diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index c4ed7799..422d88a3 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -247,7 +247,7 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls return nil, err } } - tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID) + tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID, 0) c.op.mu.Lock() c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID c.op.mu.Unlock() @@ -415,7 +415,7 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u m.setSharedWindow(b.Bundle, pid, window.WindowID) // The front tab's title, so the first input reports only a change it // made; the window search above keeps no title. - _, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID) + _, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID, 0) c.op.mu.Lock() c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID c.op.mu.Unlock() diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index fec93667..080736f8 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -195,7 +195,7 @@ func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls m.mu.Lock() m.agentChromePID, m.agentChromeChecked = pid, time.Now() m.mu.Unlock() - tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID) + tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID, 0) c.op.mu.Lock() c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID c.op.mu.Unlock() @@ -577,15 +577,25 @@ func (c *cuaSession) agentChromeKeys(ctx context.Context, arguments map[string]a // It returns 0 when the read fails or is cut short: the tab strip follows the // page in the tree, so a cut read may miss it. func (c *cuaSession) agentChromeTabs(ctx context.Context, pid int, window float64) int { - count, _, _ := c.agentChromeTabsAndTitle(ctx, pid, window) + count, _, _ := c.agentChromeTabsAndTitle(ctx, pid, window, 0) return count } +// agentChromeTabDepth bounds a read of the tab strip alone: the strip lies +// within four levels of the window, while a shop's page runs much deeper, so +// the bounded read skips most of the page. +const agentChromeTabDepth = 6 + // agentChromeTabsAndTitle is agentChromeTabs with the title of the window's -// front tab and the label of a tab's Close button, from the same read. -func (c *cuaSession) agentChromeTabsAndTitle(ctx context.Context, pid int, window float64) (int, string, string) { - read, err := c.client.request(ctx, "tools/call", map[string]any{"name": "get_window_state", "arguments": map[string]any{ - "pid": pid, "window_id": window, "include_screenshot": false, "max_elements": 6000, "timeout_ms": 6000, "session": c.label}}) +// front tab and the label of a tab's Close button, from the same read. A +// depth above 0 bounds the read; Cua then keeps only the rows within it, so +// an earlier token of a deeper row cannot be carried over that read. +func (c *cuaSession) agentChromeTabsAndTitle(ctx context.Context, pid int, window float64, depth int) (int, string, string) { + arguments := map[string]any{"pid": pid, "window_id": window, "include_screenshot": false, "max_elements": 6000, "timeout_ms": 6000, "session": c.label} + if depth > 0 { + arguments["max_depth"] = depth + } + read, err := c.client.request(ctx, "tools/call", map[string]any{"name": "get_window_state", "arguments": arguments}) if err != nil { slog.Warn("agent_browser.tabs_unread", "pid", pid, "window_id", window, "error", err) return 0, "", "" @@ -665,6 +675,27 @@ func tabStrip(tree string) (count int, close string) { return count, close } +// agentChromeWindowTitle returns the title list_windows gives window, which +// for a browser is its front tab's, or "" when it is not listed. +func (c *cuaSession) agentChromeWindowTitle(ctx context.Context, pid int, window float64) string { + result, err := c.client.request(ctx, "tools/call", map[string]any{"name": "list_windows", "arguments": map[string]any{"pid": pid}}) + if err != nil { + return "" + } + var listed struct { + Structured struct { + Windows []listedWindow `json:"windows"` + } `json:"structuredContent"` + } + _ = json.Unmarshal(result, &listed) + for _, w := range listed.Structured.Windows { + if w.WindowID == window { + return w.Title + } + } + return "" +} + // agentChromeTabWindow names the window whose tabs hold the task: the one // it opened with launch_app, when an input goes to that browser and names // no other window. An input to another app or window, such as one of the @@ -688,10 +719,11 @@ func (c *cuaSession) agentChromeTabWindow(pid, window any) (int, float64, bool) // different count means someone else opened or closed a tab, and the tab in // front may not be this task's. The task then stops, as after a reported // check. A strip that cannot be read, or a task that has no count yet, is not -// held. +// held. The strip is read only when the window's title, which list_windows +// gives without a read, differs from the front tab's title after that input. func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) error { c.op.mu.Lock() - want := c.op.chromeTabs + want, title := c.op.chromeTabs, c.op.chromeTitle c.op.mu.Unlock() if want == 0 { return nil @@ -700,8 +732,19 @@ func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) if !ok { return nil } - got := c.agentChromeTabs(ctx, number, id) + if title != "" && c.agentChromeWindowTitle(ctx, number, id) == title { + // The front tab still shows what this task's last input left. A + // tab someone opened behind it changes nothing this input reaches, + // and a full read here would cost a walk of the page. + return nil + } + got, front, _ := c.agentChromeTabsAndTitle(ctx, number, id, 0) if got == 0 || got == want { + if got > 0 { + c.op.mu.Lock() + c.op.chromeTitle = front + c.op.mu.Unlock() + } return nil } c.op.mu.Lock() @@ -716,15 +759,17 @@ func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) // other task changes the tabs. Only a task that opened its tab with // launch_app holds a count. It returns a note on what the input changed in // the window, from the same read: a click result says nothing of where the -// page went, and the Agent read the window only to learn that. -func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any) string { +// page went, and the Agent read the window only to learn that. Only full +// keeps the read whole: after a script whose reads the Agent may act on, a +// bounded read would drop their rows from Cua's snapshot. +func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any, full bool) string { c.op.mu.Lock() before, title := c.op.chromeTabs, c.op.chromeTitle c.op.mu.Unlock() if before == 0 { return "" } - got, front, close := c.settledAgentChromeTabs(ctx, pid, window) + got, front, close := c.settledAgentChromeTabs(ctx, pid, window, full) if got == 0 { return "" } @@ -751,17 +796,26 @@ func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any) // a moment after the input that opened or closed it returns, and the next // input's pause already covers that wait. The count is 0 when the strip // cannot be read. -func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any) (int, string, string) { +func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any, full bool) (int, string, string) { number, id, ok := c.agentChromeTabWindow(pid, window) if !ok { return 0, "", "" } - got, title, close := c.agentChromeTabsAndTitle(ctx, number, id) + depth := agentChromeTabDepth + if full { + depth = 0 + } + got, title, close := c.agentChromeTabsAndTitle(ctx, number, id, depth) + if got == 0 && depth > 0 { + // A strip deeper than the bound, as in a browser laid out otherwise. + depth = 0 + got, title, close = c.agentChromeTabsAndTitle(ctx, number, id, depth) + } for range 3 { if got == 0 || sleepContext(ctx, inputSettle) != nil { break } - next, nextTitle, nextClose := c.agentChromeTabsAndTitle(ctx, number, id) + next, nextTitle, nextClose := c.agentChromeTabsAndTitle(ctx, number, id, depth) title, close = nextTitle, nextClose if next == got { break diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go index 20ad7b36..f4f28c9e 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome_pace.go @@ -37,6 +37,27 @@ const ( agentChromeCooldown = time.Hour ) +// scriptInputsSent counts the inputs in a script result's call log; known is +// false when the result holds no call log to count. +func scriptInputsSent(result json.RawMessage) (sent int, known bool) { + var reply struct { + Structured struct { + Calls *[]struct { + Op string `json:"op"` + } `json:"calls"` + } `json:"structuredContent"` + } + if json.Unmarshal(result, &reply) != nil || reply.Structured.Calls == nil { + return 0, false + } + for _, call := range *reply.Structured.Calls { + if inputTools[call.Op] || call.Op == "set_value" { + sent++ + } + } + return sent, true +} + // agentChromePause is the gap kept between two inputs to the Computer's own // browser; a test replaces it. var agentChromePause = func() time.Duration { @@ -203,22 +224,7 @@ func (c *cuaSession) paceAgentChrome(ctx context.Context, reserve int) (int, err // lists them, to both budgets. A failed call may still have taken effect, so // every listed input counts. func (c *cuaSession) countScriptInputs(result json.RawMessage) { - var reply struct { - Structured struct { - Calls []struct { - Op string `json:"op"` - } `json:"calls"` - } `json:"structuredContent"` - } - if json.Unmarshal(result, &reply) != nil { - return - } - sent := 0 - for _, call := range reply.Structured.Calls { - if inputTools[call.Op] || call.Op == "set_value" { - sent++ - } - } + sent, _ := scriptInputsSent(result) if sent == 0 { return } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 121910ea..45191934 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -557,9 +557,16 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage if paced && name == "run_script" && err == nil { c.countScriptInputs(result) } - // A script that throws has still sent the inputs before the throw. - if paced && (err == nil || name == "run_script") { - if note := c.recordAgentChromeTabs(ctx, target["pid"], target["window_id"]); note != "" && result != nil { + // A script that throws has still sent the inputs before the throw. One + // that sent none changed no tab, and a read of the strip after it would + // stale the tokens its queries returned, which the Daemon keeps no rows of. + sentNone := false + if name == "run_script" { + sent, known := scriptInputsSent(result) + sentNone = known && sent == 0 + } + if paced && (err == nil || name == "run_script") && !sentNone { + if note := c.recordAgentChromeTabs(ctx, target["pid"], target["window_id"], len(scriptReads) > 0); note != "" && result != nil { result = withNote(result, note) } } @@ -1318,7 +1325,7 @@ func (c *cuaSession) locate(ctx context.Context, name string, arguments map[stri } read, err := c.client.request(ctx, "tools/call", map[string]any{"name": "get_window_state", "arguments": map[string]any{ "pid": arguments["pid"], "window_id": arguments["window_id"], "tree_format": "elements", "include_screenshot": false, - "max_elements": 3000, "timeout_ms": float64(windowReadTimeoutMs), "session": arguments["session"]}}) + "max_elements": 6000, "timeout_ms": float64(windowReadTimeoutMs), "session": arguments["session"]}}) if err != nil { return err } diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 2e0729e4..787e4930 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -196,6 +196,15 @@ func runToolFixture(mode string) { } result = map[string]any{"content": []any{}} switch request.Params.Name { + case "run_script": + if mode == "tabs-titled" { + // The driver's call log: a click when the script clicks. + calls := []any{} + if script, _ := request.Params.Arguments["script"].(string); strings.Contains(script, "app.click(") { + calls = append(calls, map[string]any{"op": "click"}) + } + result = map[string]any{"content": []any{}, "structuredContent": map[string]any{"calls": calls}} + } case "hotkey": if mode == "tabs-off-space" && strings.Count(string(data), `"keys":["cmd","t"]`) == 0 { result = map[string]any{"isError": true, "content": []any{map[string]string{"type": "text", "text": "off_space_or_ax_unresolved"}}} @@ -204,7 +213,12 @@ func runToolFixture(mode string) { result = map[string]any{"isError": true, "content": []any{map[string]string{"type": "text", "text": "keys were not delivered"}}} } case "list_windows": - windows := []any{map[string]any{"pid": 42, "window_id": 9, "title": "b", "z_index": 3, "is_on_screen": true, "bounds": map[string]any{"width": 800, "height": 600}}} + title := "b" + if mode == "tabs-titled" { + // The window's title is its front tab's, as Chrome's is. + title = fmt.Sprintf("Tab %d", tabs-1) + } + windows := []any{map[string]any{"pid": 42, "window_id": 9, "title": title, "z_index": 3, "is_on_screen": true, "bounds": map[string]any{"width": 800, "height": 600}}} if strings.HasPrefix(mode, "tabs-everyday") && strings.Count(string(data), `"name":"open"`) >= 2 { // After the start and the command line, a window of // the user's that Chrome restored late lists before @@ -224,6 +238,13 @@ func runToolFixture(mode string) { result = map[string]any{"content": []any{}, "structuredContent": map[string]any{"tree_markdown": "", "degraded_reason": "ax_window_unresolved"}} break } + if mode == "tabs-titled" { + // Each read logs its depth bound, 0 for a whole read. + depth, _ := request.Params.Arguments["max_depth"].(float64) + f, _ := os.OpenFile(marker, os.O_APPEND|os.O_WRONLY, 0o600) + _, _ = fmt.Fprintf(f, `{"name":"read","arguments":{"max_depth":%v}}`+"\n", depth) + _ = f.Close() + } var tree strings.Builder for i := range tabs { fmt.Fprintf(&tree, "- [%d] AXRadioButton \"Tab %d\"\n- [%d] AXButton \"Close\" [actions=[press]]\n", 2*i, i, 2*i+1) @@ -1160,6 +1181,78 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsOwnInputChangesLate(t *testing.T) } } +// An input whose window still shows the front tab the last input left skips +// the read of the tab strip before it, and the reads after it stop at the +// strip's depth, unless a script's reads must stay in Cua's snapshot. +func TestLocalToolCuaMacAgentChromeReadsTheTabStripOnlyAsFarAsItNeeds(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("the Computer's own Chrome is macOS only") + } + inputSettle = 0 + t.Cleanup(func() { inputSettle = 300 * time.Millisecond }) + pause := agentChromePause + agentChromePause = func() time.Duration { return 0 } + t.Cleanup(func() { agentChromePause = pause }) + marker := filepath.Join(t.TempDir(), "calls.jsonl") + c := cuaFixtureSession(t, "tabs-titled", marker) + original := agentChromePID + agentChromePID = func(context.Context, string) (int, string, error) { return 42, "--mute-audio", nil } + t.Cleanup(func() { agentChromePID = original }) + c.op.manager.agentChromeOwner = c.op + c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = 1, "Tab 0", 42, 9 + reads := func() []string { + var got []string + for _, call := range fixtureCalls(t, marker) { + if call.Name == "read" { + got = append(got, fmt.Sprint(call.Arguments["max_depth"])) + } else { + got = append(got, call.Name) + } + } + return got + } + if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); err != nil { + t.Fatal(err) + } + if got, want := reads(), []string{"click", "6", "6"}; !slices.Equal(got, want) { + t.Fatalf("calls=%v, want the click alone, then two bounded reads", got) + } + // After a script that returned reads, the reads after it are whole. + if note := c.recordAgentChromeTabs(t.Context(), float64(42), float64(9), true); note != "" { + t.Fatalf("note=%q, want none", note) + } + if got := reads(); !slices.Equal(got[len(got)-2:], []string{"0", "0"}) { + t.Fatalf("calls=%v, want two whole reads after a script with reads", got) + } + // A script that sent no input reads no tab strip after it, so the + // tokens its queries returned stay current; one that clicked does. + before := len(reads()) + if _, err := c.call(t.Context(), "run_script", json.RawMessage(`{"script":"return 1"}`)); err != nil { + t.Fatal(err) + } + if got := reads()[before:]; !slices.Equal(got, []string{"run_script"}) { + t.Fatalf("calls for a script with no input=%v, want the script alone", got) + } + before = len(reads()) + if _, err := c.call(t.Context(), "run_script", json.RawMessage(`{"script":"const app = await cua.getApp({pid: 42, windowId: 9}); await app.click(\"s1:1\"); return 1"}`)); err != nil { + t.Fatal(err) + } + if got := reads()[before:]; !slices.Equal(got, []string{"run_script", "6", "6"}) { + t.Fatalf("calls for a script that clicked=%v, want two bounded reads after it", got) + } + // A front tab that changed since the last input is read whole first. + c.op.chromeTitle = "Elsewhere" + if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); err != nil { + t.Fatal(err) + } + if got := reads(); !slices.Equal(got[len(got)-4:], []string{"0", "click", "6", "6"}) { + t.Fatalf("calls=%v, want a whole read before the click", got) + } + if err := c.drain(); err != nil { + t.Fatal(err) + } +} + func TestLocalToolCuaMacAgentChromeKeepsTabsItsScriptsChange(t *testing.T) { if runtime.GOOS != "darwin" { t.Skip("the Computer's own Chrome is macOS only") From 65a2e0216ab68c6262ef6e80868520b87b62514d Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 19:59:48 +0800 Subject: [PATCH 24/30] fix(computer): report a tab change behind the front tab neutrally The check before an input reads the tab strip only when the front tab's title changed, so a tab someone opened or closed behind it no longer stops the task. The result after the input then said the input opened or closed that tab and told the Agent to close it to go back. When the front tab is unchanged, the result now reports the count alone, and the README states which tab changes stop a task. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 11 +++++++---- .../internal/linger/localtools/agent_chrome.go | 5 +++++ .../internal/linger/localtools/localtools_test.go | 10 ++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 97fd7471..06b1a47e 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -375,10 +375,13 @@ the task; `rate_limited` also puts the site (its registrable domain) in a 60-minute cooldown. The record lives in `computer-tools/agent-chrome-pace.json`. Once a task opens a site there, the browser tools refuse that site for the rest of the task, and `launch_app` refuses to open it in another browser. -A change in the number of tabs between the task's inputs stops the task with -`COMPUTER_TAB_CHANGED`, because the next input could reach another page. The -Daemon reads the tab strip before an input only when the window's title in -`list_windows` differs from the front tab's title after the last input; after +A change of the front tab between the task's inputs that also changed the +number of tabs stops the task with `COMPUTER_TAB_CHANGED`, because the next +input could reach another page. The Daemon reads the tab strip before an input +only when the window's title in `list_windows` differs from the front tab's +title after the last input, so a tab opened or closed behind the front tab +does not stop the task: the input still reaches the task's page, and its +result reports the new count without saying the input changed it; after an input it reads the strip only to its depth, unless a script returned reads the Agent may act on, and not at all after a script that sent no input. A tab that the task's own input or script opens or closes is the task's: the diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 080736f8..07b6854e 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -777,6 +777,11 @@ func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any, c.op.chromeTabs, c.op.chromeTitle = got, front c.op.mu.Unlock() switch { + case got != before && front == title: + // The front tab is the one the input went to. A tab opened or + // closed behind it may be someone else's, since the check before + // the input reads the strip only when the front tab changed. + return fmt.Sprintf("The window now has %d tab(s), %d before this input; the tab in front still shows %q.", got, before, front) case got > before && close != "": // The tab strip follows the page in the tree, past the default // walk, so the Agent's search for the button came back empty. diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 787e4930..a4cd3ad4 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1240,6 +1240,16 @@ func TestLocalToolCuaMacAgentChromeReadsTheTabStripOnlyAsFarAsItNeeds(t *testing if got := reads()[before:]; !slices.Equal(got, []string{"run_script", "6", "6"}) { t.Fatalf("calls for a script that clicked=%v, want two bounded reads after it", got) } + // A tab closed behind the front tab does not stop the task, and the + // result reports the count without saying the input closed it. + c.op.chromeTabs = 2 + result, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)) + if err != nil { + t.Fatal(err) + } + if note := `The window now has 1 tab(s), 2 before this input; the tab in front still shows \"Tab 0\".`; !strings.Contains(string(result), note) || strings.Contains(string(result), "This input closed") { + t.Fatalf("result=%s, want the neutral note %s", result, note) + } // A front tab that changed since the last input is read whole first. c.op.chromeTitle = "Elsewhere" if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); err != nil { From 4762974467b00f77f642de2f28f4755007b1b1e5 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 21:11:39 +0800 Subject: [PATCH 25/30] fix(computer): prove fewer tabs and keep locate and click notes exact A bounded tab strip read that shows fewer tabs than before is read again in full, since a strip split across the depth bound would read as fewer. launch_app takes the front tab's title from list_windows instead of a second whole read. A role and name target whose name is only white space is refused instead of matching an element with no value. The note on a press to a row that lists no press says a repeat by x,y waits for a read that shows no effect. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 8 ++++--- .../linger/localtools/agent_browser.go | 8 +++---- .../linger/localtools/agent_chrome.go | 10 +++++---- .../internal/linger/localtools/cua.go | 5 +++-- .../linger/localtools/localtools_test.go | 21 ++++++++++++++----- 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 06b1a47e..8dc19f0b 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -384,13 +384,15 @@ does not stop the task: the input still reaches the task's page, and its result reports the new count without saying the input changed it; after an input it reads the strip only to its depth, unless a script returned reads the Agent may act on, and not at all after a script that sent no input. A +bounded read that shows no tabs or fewer than before is read again in full. A tab that the task's own input or script opens or closes is the task's: the task holds the browser's input lease, and the count is kept once the tab strip settles, since the browser shows such a tab a moment later. The input's result then says, from that same read, when it opened or closed a tab or -changed the title of the tab in front. A click there on page text whose row -lists no press notes that Chrome clicks it anyway, since Cua warns that such -a press may have done nothing. Reading the tab +changed the title of the tab in front. A click there on an element whose row +lists no press notes that the press reaches it anyway, since Cua warns that +such a press may have done nothing; a repeat by x,y waits for a read that +shows no effect. Reading the tab strip replaces Cua's snapshot of the window, so the Daemon remembers the rows of recent reads and moves an Agent's element token, in an input or written into a script, to the current snapshot when its row reads the same there. diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index 422d88a3..8431fbb6 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -413,11 +413,11 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u return nil, err } m.setSharedWindow(b.Bundle, pid, window.WindowID) - // The front tab's title, so the first input reports only a change it - // made; the window search above keeps no title. - _, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID, 0) + // list_windows gives the front tab's title, so the first input reports + // only a change it made. A title that moved on since then costs the + // first input's check one read of the strip. c.op.mu.Lock() - c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID + c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, window.Title, pid, window.WindowID c.op.mu.Unlock() note := "This is the Computer's own window in the user's everyday Chrome, with the user's sign-ins; work only in this window, never in the user's other Chrome windows. It works while other windows cover it, but only on the desktop (Space) the user is on. The user's Chrome takes no keys from the Computer, because a key could reach the window the user is typing in: fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links. Close a tab with the Close button of its tab; closing the window's last tab closes the window. If a site asks to sign in, ask the user to sign in once in this window." if restored { diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 07b6854e..2c563153 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -769,7 +769,7 @@ func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any, if before == 0 { return "" } - got, front, close := c.settledAgentChromeTabs(ctx, pid, window, full) + got, front, close := c.settledAgentChromeTabs(ctx, pid, window, full, before) if got == 0 { return "" } @@ -801,7 +801,7 @@ func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any, // a moment after the input that opened or closed it returns, and the next // input's pause already covers that wait. The count is 0 when the strip // cannot be read. -func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any, full bool) (int, string, string) { +func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any, full bool, before int) (int, string, string) { number, id, ok := c.agentChromeTabWindow(pid, window) if !ok { return 0, "", "" @@ -811,8 +811,10 @@ func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any depth = 0 } got, title, close := c.agentChromeTabsAndTitle(ctx, number, id, depth) - if got == 0 && depth > 0 { - // A strip deeper than the bound, as in a browser laid out otherwise. + if (got == 0 || got < before) && depth > 0 { + // A strip deeper than the bound, as in a browser laid out otherwise, + // reads as no tabs, and one split across the bound as fewer. A + // closed tab is rare, so proving it with a whole read costs little. depth = 0 got, title, close = c.agentChromeTabsAndTitle(ctx, number, id, depth) } diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 45191934..5eb31dfd 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -534,7 +534,7 @@ func (c *cuaSession) call(ctx context.Context, name string, args json.RawMessage // Chrome clicks a page element on a press even where the element // lists no press, as the sort menu of a review list did; the Agent // took Cua's warning as a miss and clicked by x,y, which missed. - result = withNote(result, "In the Computer's own browser, a press on page text clicks it as a person would, even where the row lists no press: check its effect in the next read, and do not repeat it by x,y.") + result = withNote(result, "In the Computer's own browser, a press goes to the element as a person's click would, even where its row lists no press. Check its effect in the next read; repeat it by x,y only if that read shows no effect.") } var scriptReads []windowRead if err == nil && name == "run_script" { @@ -1320,6 +1320,8 @@ func (c *cuaSession) locate(ctx context.Context, name string, arguments map[stri } role, _ := arguments["role"].(string) label, _ := arguments["name"].(string) + // A name of only white space would match every element without a label. + label = sameSpaces(label) if role == "" || label == "" || arguments["pid"] == nil || arguments["window_id"] == nil { return fmt.Errorf("COMPUTER_TARGET_REQUIRED: a role and name target needs pid, window_id, role and name: %w", errNotSent) } @@ -1350,7 +1352,6 @@ func (c *cuaSession) locate(ctx context.Context, name string, arguments map[stri return fmt.Errorf("COMPUTER_TARGET_UNREADABLE: window %v was read only in part, so a %s labeled %q cannot be proven to be the only one; use an element_token from a read: %w", arguments["window_id"], role, label, errNotSent) } var matches []string - label = sameSpaces(label) for _, element := range state.Structured.Elements { value, _ := element.Value.(string) if element.Role == role && (sameSpaces(element.Label) == label || sameSpaces(value) == label) { diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index a4cd3ad4..42d23889 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -224,7 +224,8 @@ func runToolFixture(mode string) { // the user's that Chrome restored late lists before // the Computer's window. for _, id := range []int{11, 12} { - windows = append(windows, map[string]any{"pid": 42, "window_id": id, "title": "Home", "z_index": 2, "is_on_screen": true, "bounds": map[string]any{"width": 900, "height": 700}}) + // Its title is its front tab's, as Chrome's is. + windows = append(windows, map[string]any{"pid": 42, "window_id": id, "title": fmt.Sprintf("Tab %d", tabs-1), "z_index": 2, "is_on_screen": true, "bounds": map[string]any{"width": 900, "height": 700}}) } } if mode == "tabs-ghost" { @@ -1172,7 +1173,7 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsOwnInputChangesLate(t *testing.T) if err != nil { t.Fatalf("input after the task's own %s: %v", step.keys, err) } - if strings.Contains(string(result), "tab") || !strings.Contains(string(result), "a press on page text clicks it") { + if strings.Contains(string(result), "tab") || !strings.Contains(string(result), "even where its row lists no press") { t.Fatalf("result of a click on page text that changed no tab=%s", result) } } @@ -1241,12 +1242,18 @@ func TestLocalToolCuaMacAgentChromeReadsTheTabStripOnlyAsFarAsItNeeds(t *testing t.Fatalf("calls for a script that clicked=%v, want two bounded reads after it", got) } // A tab closed behind the front tab does not stop the task, and the - // result reports the count without saying the input closed it. + // result reports the count without saying the input closed it. Fewer + // tabs in a bounded read are proven by whole reads, since a strip split + // across the bound would read as fewer too. c.op.chromeTabs = 2 + before = len(reads()) result, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)) if err != nil { t.Fatal(err) } + if got := reads()[before:]; !slices.Equal(got, []string{"click", "6", "0", "0"}) { + t.Fatalf("calls for fewer tabs=%v, want a bounded read, then whole reads", got) + } if note := `The window now has 1 tab(s), 2 before this input; the tab in front still shows \"Tab 0\".`; !strings.Contains(string(result), note) || strings.Contains(string(result), "This input closed") { t.Fatalf("result=%s, want the neutral note %s", result, note) } @@ -1649,6 +1656,10 @@ func TestLocalToolCuaMacInputsFindTheirTargetWhenTheyRun(t *testing.T) { if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"role":"AXLink","name":"Kettle [snapshot]"}`)); err != nil { t.Fatalf("label with a no-break space: %v", err) } + // A name of only white space names nothing, not the element without a value. + if _, err := c.call(t.Context(), "type_text", json.RawMessage(`{"pid":42,"window_id":9,"role":"AXTextArea","name":"\u00a0 ","text":"hi"}`)); err == nil || !strings.Contains(err.Error(), "COMPUTER_TARGET_REQUIRED") { + t.Fatalf("white space name err=%v, want target required", err) + } if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"role":"AXButton","name":"Delete"}`)); err == nil || !strings.Contains(err.Error(), "COMPUTER_TARGET_NOT_FOUND") { t.Fatalf("partial label err=%v, want not found", err) } @@ -2091,8 +2102,8 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t if _, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"keys":["cmd","w"],"element_token":"s7:1"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_TARGET_REQUIRED") { t.Fatalf("hotkey with no pid: err=%v, want refused", err) } - // launch_app keeps the front tab's title, so the first input, which - // changes no tab, reports no change. + // launch_app keeps the front tab's title from list_windows, so the + // first input, which changes no tab, reports no change. if c.op.chromeTitle != "Tab 0" { t.Fatalf("front title after launch_app=%q, want Tab 0", c.op.chromeTitle) } From 87e692bc5679a4b834c2d6e8e00d69833111901d Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 21:54:00 +0800 Subject: [PATCH 26/30] fix(computer): hold each launched window and fence script reads by input Each window a task opens with launch_app keeps its own tab count and front title, so going back to an earlier window keeps its tab check. The script guard drops a read that started before an input finished, such as one made during the pause of an input the script did not await, so stale rows are not carried. The everyday Chrome window search restarts its settle wait when the lone candidate window changes or leaves. The picture viewer test fixture and the countTabs wrapper, left without callers, are removed. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 5 +- .../linger/localtools/agent_browser.go | 18 ++-- .../linger/localtools/agent_browser_test.go | 24 ++++- .../linger/localtools/agent_chrome.go | 92 ++++++++++--------- .../internal/linger/localtools/cua_script.go | 13 ++- .../localtools/cua_script_guard_test.go | 31 +++++++ .../linger/localtools/localtools_test.go | 70 ++++---------- .../internal/linger/localtools/manager.go | 12 +-- 8 files changed, 148 insertions(+), 117 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 8dc19f0b..43658b50 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -398,8 +398,9 @@ of recent reads and moves an Agent's element token, in an input or written into a script, to the current snapshot when its row reads the same there. A script's reads after its last input count too: the script guard returns them beside the script's value, and the Daemon takes them out of the result. -Only the window -the task opened with `launch_app` is held. A script that names no window in +A read that started before an input finished, such as one beside an input +the script did not await, does not count. Only the windows +the task opened with `launch_app` are held, each to its own tab count. A script that names no window in its `args` is taken to drive that window; a script to another app is neither paced nor held, and no script may open an app or URL. The browser window is the largest window whose tab strip reads: a starting Chrome can list diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index 8431fbb6..d19ae54f 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -248,9 +248,7 @@ func (c *cuaSession) launchAgentSafari(ctx context.Context, b agentBrowser, urls } } tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID, 0) - c.op.mu.Lock() - c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID - c.op.mu.Unlock() + c.holdAgentChromeWindow(pid, window.WindowID, tabs, title) note := "This is the Computer's own window in the user's Safari. Safari has no separate profile, so this window has the user's Safari sign-ins; work only in this window, never in the user's other Safari windows. It works while other windows cover it, but only on the desktop (Space) the user is on. Keys without an element go to this window in the background. Type into a page field with type_text and the field's element_token. Move between pages as a person does: through the page's search box, menus and links, back with cmd+[; never type an address into the address bar. Close a tab with its tab's close action, not cmd+w." if started { note = "Opened a new Safari window for the Computer. " + note @@ -416,9 +414,7 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u // list_windows gives the front tab's title, so the first input reports // only a change it made. A title that moved on since then costs the // first input's check one read of the strip. - c.op.mu.Lock() - c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, window.Title, pid, window.WindowID - c.op.mu.Unlock() + c.holdAgentChromeWindow(pid, window.WindowID, tabs, window.Title) note := "This is the Computer's own window in the user's everyday Chrome, with the user's sign-ins; work only in this window, never in the user's other Chrome windows. It works while other windows cover it, but only on the desktop (Space) the user is on. The user's Chrome takes no keys from the Computer, because a key could reach the window the user is typing in: fill a field with set_value, press the page's own buttons, such as its search button, and go back through the page's links. Close a tab with the Close button of its tab; closing the window's last tab closes the window. If a site asks to sign in, ask the user to sign in once in this window." if restored { note = "Started the user's Chrome again with their windows and tabs. " + note @@ -436,6 +432,7 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u // finds its window. Windows that stay ambiguous are refused. func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, pid int, before map[float64]bool, site string) (listedWindow, int, error) { var seen time.Time + var candidate float64 for deadline := time.Now().Add(sharedBrowserWindowWait); ; { windows, err := c.appWindows(ctx, pid) if err != nil { @@ -451,7 +448,14 @@ func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, p matching = append(matching, w) } } - if len(fresh) > 0 && seen.IsZero() { + // The wait counts from when the lone candidate showed: one that + // left, or another in its place, starts it again. + switch { + case len(fresh) == 0: + seen = time.Time{} + case len(fresh) == 1 && fresh[0].WindowID != candidate: + seen, candidate = time.Now(), fresh[0].WindowID + case seen.IsZero(): seen = time.Now() } settled := !seen.IsZero() && time.Since(seen) >= everydayChromeSettle diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go index 7e1a0556..7d56ca16 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go @@ -21,7 +21,7 @@ func TestCountTabsReadsChromeEdgeAndFirefoxStrips(t *testing.T) { } } // A page's own radio buttons are not tabs. - if got := countTabs("- [9] AXRadioButton \"Small\"\n- [10] AXRadioButton \"Large\"\n- [11] AXButton \"Close\" [actions=[press]]"); got != 1 { + if got, _ := tabStrip("- [9] AXRadioButton \"Small\"\n- [10] AXRadioButton \"Large\"\n- [11] AXButton \"Close\" [actions=[press]]"); got != 1 { t.Errorf("page radios: %d tabs, want 1", got) } } @@ -84,3 +84,25 @@ func TestSharedWindowsKeepEveryWindowOfTheBrowser(t *testing.T) { t.Fatalf("safari=%+v ok=%v, want Safari's window kept apart", safari, ok) } } + +// Each window a task opened stays held to its own tabs after it opens +// another, and an input that names no window goes to the latest one. +func TestAgentChromeHoldsEachWindowTheTaskOpened(t *testing.T) { + c := &cuaSession{op: &operation{}} + c.holdAgentChromeWindow(42, 9, 2, "Shop") + c.holdAgentChromeWindow(42, 13, 1, "News") + if _, id, held, ok := c.agentChromeTabWindow(float64(42), float64(9)); !ok || id != 9 || held != (heldTabs{count: 2, title: "Shop"}) { + t.Fatalf("first window id=%v held=%+v ok=%v, want its own tabs", id, held, ok) + } + if _, id, held, ok := c.agentChromeTabWindow(float64(42), nil); !ok || id != 13 || held.count != 1 { + t.Fatalf("no window id=%v held=%+v ok=%v, want the latest", id, held, ok) + } + if _, _, _, ok := c.agentChromeTabWindow(float64(42), float64(5)); ok { + t.Fatal("a window the task did not open is held") + } + // A browser started again runs as another pid: no earlier window is held. + c.holdAgentChromeWindow(77, 3, 1, "Home") + if _, _, _, ok := c.agentChromeTabWindow(float64(42), float64(9)); ok { + t.Fatal("a window of the earlier browser is still held") + } +} diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 2c563153..2eddf2a9 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -196,9 +196,7 @@ func (c *cuaSession) launchAgentChrome(ctx context.Context, b agentBrowser, urls m.agentChromePID, m.agentChromeChecked = pid, time.Now() m.mu.Unlock() tabs, title, _ := c.agentChromeTabsAndTitle(ctx, pid, window.WindowID, 0) - c.op.mu.Lock() - c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = tabs, title, pid, window.WindowID - c.op.mu.Unlock() + c.holdAgentChromeWindow(pid, window.WindowID, tabs, title) opened := "No URL was opened, so no tab is this task's." if len(urls) > 0 { opened = "The last one is this task's and in front; pass tabs to a Skill script that checks it." @@ -623,15 +621,10 @@ func (c *cuaSession) agentChromeTabsAndTitle(ctx context.Context, pid int, windo return count, state.Structured.Title, close } -// countTabs counts the tab rows of a browser window's tree: a radio button -// outside the page that holds a button, its Close button. -func countTabs(tree string) int { - count, _ := tabStrip(tree) - return count -} - -// tabStrip is countTabs with the label of a tab's Close button, which follows -// the user's language; it is empty for a Safari tab, which has none. +// tabStrip counts the tab rows of a browser window's tree, each a radio +// button outside the page that holds a button, its Close button, and returns +// the label of that button, which follows the user's language; it is empty +// for a Safari tab, which has none. func tabStrip(tree string) (count int, close string) { rows := strings.Split(tree, "\n") indent := func(row string) int { return len(row) - len(strings.TrimLeft(row, " ")) } @@ -696,22 +689,45 @@ func (c *cuaSession) agentChromeWindowTitle(ctx context.Context, pid int, window return "" } -// agentChromeTabWindow names the window whose tabs hold the task: the one -// it opened with launch_app, when an input goes to that browser and names -// no other window. An input to another app or window, such as one of the -// user's own Safari windows, is not held. -func (c *cuaSession) agentChromeTabWindow(pid, window any) (int, float64, bool) { +// heldTabs is what a window the task opened with launch_app showed after +// the task's last input there: its tab count and its front tab's title. A +// count of 0 holds nothing, as for a strip that could not be read. +type heldTabs struct { + count int + title string +} + +// holdAgentChromeWindow makes window, in the browser that runs as pid, the +// task's latest window, held to tabs and title. A window the task opened +// before stays held to its own tabs, so going back to it keeps its check; +// a browser that runs as another pid holds none of them. +func (c *cuaSession) holdAgentChromeWindow(pid int, window float64, tabs int, title string) { c.op.mu.Lock() - own, ownWindow := c.op.chromePID, c.op.chromeWindow - c.op.mu.Unlock() + defer c.op.mu.Unlock() + if c.op.chromePID != pid || c.op.chromeHeld == nil { + c.op.chromeHeld = map[float64]heldTabs{} + } + c.op.chromeHeld[window] = heldTabs{count: tabs, title: title} + c.op.chromePID, c.op.chromeWindow = pid, window +} + +// agentChromeTabWindow names the window whose tabs hold the task: one it +// opened with launch_app, when an input goes to that browser, or its latest +// such window when the input names none. An input to another app or window, +// such as one of the user's own Safari windows, is not held. +func (c *cuaSession) agentChromeTabWindow(pid, window any) (int, float64, heldTabs, bool) { + c.op.mu.Lock() + defer c.op.mu.Unlock() + own, id := c.op.chromePID, c.op.chromeWindow number, ok := pid.(float64) if !ok || own == 0 || int(number) != own { - return 0, 0, false + return 0, 0, heldTabs{}, false } - if id, ok := window.(float64); ok && id != ownWindow { - return 0, 0, false + if named, ok := window.(float64); ok { + id = named } - return own, ownWindow, true + held, ok := c.op.chromeHeld[id] + return own, id, held, ok } // checkAgentChromeTabs holds the task to the tabs it left: the Computer's @@ -722,14 +738,9 @@ func (c *cuaSession) agentChromeTabWindow(pid, window any) (int, float64, bool) // held. The strip is read only when the window's title, which list_windows // gives without a read, differs from the front tab's title after that input. func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) error { - c.op.mu.Lock() - want, title := c.op.chromeTabs, c.op.chromeTitle - c.op.mu.Unlock() - if want == 0 { - return nil - } - number, id, ok := c.agentChromeTabWindow(pid, window) - if !ok { + number, id, held, ok := c.agentChromeTabWindow(pid, window) + want, title := held.count, held.title + if !ok || want == 0 { return nil } if title != "" && c.agentChromeWindowTitle(ctx, number, id) == title { @@ -742,7 +753,7 @@ func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) if got == 0 || got == want { if got > 0 { c.op.mu.Lock() - c.op.chromeTitle = front + c.op.chromeHeld[id] = heldTabs{count: want, title: front} c.op.mu.Unlock() } return nil @@ -763,18 +774,17 @@ func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) // keeps the read whole: after a script whose reads the Agent may act on, a // bounded read would drop their rows from Cua's snapshot. func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any, full bool) string { - c.op.mu.Lock() - before, title := c.op.chromeTabs, c.op.chromeTitle - c.op.mu.Unlock() - if before == 0 { + number, id, held, ok := c.agentChromeTabWindow(pid, window) + before, title := held.count, held.title + if !ok || before == 0 { return "" } - got, front, close := c.settledAgentChromeTabs(ctx, pid, window, full, before) + got, front, close := c.settledAgentChromeTabs(ctx, number, id, full, before) if got == 0 { return "" } c.op.mu.Lock() - c.op.chromeTabs, c.op.chromeTitle = got, front + c.op.chromeHeld[id] = heldTabs{count: got, title: front} c.op.mu.Unlock() switch { case got != before && front == title: @@ -801,11 +811,7 @@ func (c *cuaSession) recordAgentChromeTabs(ctx context.Context, pid, window any, // a moment after the input that opened or closed it returns, and the next // input's pause already covers that wait. The count is 0 when the strip // cannot be read. -func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, pid, window any, full bool, before int) (int, string, string) { - number, id, ok := c.agentChromeTabWindow(pid, window) - if !ok { - return 0, "", "" - } +func (c *cuaSession) settledAgentChromeTabs(ctx context.Context, number int, id float64, full bool, before int) (int, string, string) { depth := agentChromeTabDepth if full { depth = 0 diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go index d6090661..40583afa 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script.go @@ -69,7 +69,10 @@ var scriptForbidden = []string{"delivery_mode", "foreground", "screenshot_out_fi // could not verify in time. A script reads text, so getState takes no // screenshot unless the script asks for one: Cua's default captures one per // read, which took seconds on a busy page. The guard returns the window reads -// made after the script's last input beside its value, under scriptReadsKey, +// made after the script's last input beside its value, under scriptReadsKey: +// a read that started before an input finished, such as one beside an input +// the script did not await, is dropped, since it may show the page before +// that input, // for scriptResultReads to take out. It keeps a read by the pid and window_id // that Cua puts in every getState and get_window_state result, as in a tool // read, so a script read lands on the same window key as the Computer's own. A positive inputLimit paces the script for the @@ -93,7 +96,7 @@ const keys = Object.keys, names = Object.getOwnPropertyNames, proto = Object.get const apply = Reflect.apply, stringify = JSON.stringify, parse = JSON.parse, test = RegExp.prototype.test, then = Promise.prototype.then, slice = Array.prototype.slice; const plainProtos = [Object.prototype, Array.prototype, null], real = globalThis.cua; const bad = /`+strings.Join(pattern, "|")+`/; -const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0, reads = [], readChars = 0; +const limit = `+strconv.Itoa(inputLimit)+`, chrome = `+strconv.Itoa(chrome)+`, rest = real.sleep, random = Math.random, floor = Math.floor; let sent = 0, reads = [], readChars = 0, inputs = 0; const reading = (k, c) => k === "getState" || k === "call" && c[0] === "get_window_state"; const keep = r => { if (r === null || typeof r !== "object" || typeof r.snapshot_id !== "string" || typeof r.pid !== "number" || typeof r.window_id !== "number") return; const read = {snapshot_id: r.snapshot_id, pid: r.pid, window_id: r.window_id}; for (const f of ["tree_markdown", "tree_diff", "since", "since_status"]) if (typeof r[f] === "string") read[f] = r[f]; const size = (read.tree_markdown || "").length + (read.tree_diff || "").length; reads[reads.length] = read; readChars += size; while (reads.length > `+strconv.Itoa(scriptReadsKept)+` || readChars > `+strconv.Itoa(scriptReadCharacters)+` && reads.length > 0) { const first = reads[0]; readChars -= (first.tree_markdown || "").length + (first.tree_diff || "").length; reads = apply(slice, reads, [1]); } }; const keyOps = {__proto__: null, hotkey: true, type_text: true, press_key: true}, keyMethods = {__proto__: null, hotkey: true, typeText: true, pressKey: true}; @@ -116,11 +119,11 @@ define(w, k, {enumerable: true, get() { const v = o[k]; if (typeof v !== "functi if (limit > 0 && k === "typeText") { const given = c.length > 2 && c[2] !== null && typeof c[2] === "object" ? c[2] : {}; const t = {length: 3}; define(t, 0, {value: c[0]}); define(t, 1, {value: c[1]}); define(t, 2, {value: {delay_ms: `+strconv.Itoa(agentChromeTypeDelay)+`, ...given}}); c = t; } if (k === "getState" && (c.length === 0 || c[0] === undefined || c[0] !== null && typeof c[0] === "object" && c[0].include_screenshot === undefined)) { const t = {length: 1}; define(t, 0, {value: {include_screenshot: false, ...(c.length > 0 && c[0] !== undefined ? c[0] : {})}}); c = t; } if (limit > 0 && k === "sleep" && o === real && typeof c[0] === "number" && c[0] > `+strconv.Itoa(scriptSleepLimit)+`) { const t = {length: 1}; define(t, 0, {value: `+strconv.Itoa(scriptSleepLimit)+`}); c = t; } -const run = () => { const r = apply(v, o, c); const seen = x => { if (reading(k, c)) keep(x); return out(x); }; return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [seen]) : seen(r); }; +const sends = input(k, c), fence = () => { reads = []; readChars = 0; inputs++; }; +const run = () => { if (sends) fence(); const at = inputs, r = apply(v, o, c); const seen = x => { if (sends) fence(); else if (reading(k, c) && at === inputs) keep(x); return out(x); }; const missed = e => { if (sends) fence(); throw e; }; return r !== null && typeof r === "object" && typeof r.then === "function" ? apply(then, r, [seen, missed]) : seen(r); }; if (launches(k, c)) throw new Error("COMPUTER_SCRIPT_REFUSED: a script opens no app or URL, so the site checks of launch_app hold and the key rule knows every app the script reaches; call launch_app as its own tool"); if (keyed(k, c, o)) throw new Error(`+strconv.Quote(everydayChromeKeysRefused)+`); -if (input(k, c)) { reads = []; readChars = 0; } -if (limit <= 0 || !input(k, c)) return run(); +if (limit <= 0 || !sends) return run(); if (++sent > limit) throw new Error("COMPUTER_SCRIPT_INPUT_LIMIT: a script sends at most " + limit + " inputs to the Computer's own browser; return what you have, look at the window, then continue in a new script"); return apply(then, apply(rest, real, [`+strconv.Itoa(scriptInputPauseMin)+` + floor(random() * `+strconv.Itoa(scriptInputPauseSpan)+`)]), [run]); }; }}); } } return freeze(w); }; diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go index caff3f53..7cbbc31b 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua_script_guard_test.go @@ -102,6 +102,37 @@ func TestScriptGuardPacesInputsForTheComputersChrome(t *testing.T) { } } +// A paced input reaches the driver after its pause, so a read the script +// makes during that pause, beside an input it did not await, shows the page +// before the input and is not returned; a read after the input is. +func TestScriptGuardDropsAReadThatAnInputOverlaps(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node is not installed") + } + dir := t.TempDir() + harness := filepath.Join(dir, "harness.mjs") + if err := os.WriteFile(harness, []byte(guardHarness), 0o600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ body, want string }{ + {`const app = await cua.getApp({pid: 4}); const click = app.click(1); await app.getState({query: "rows"}); await click; return "done";`, `"result":"done"`}, + {`const app = await cua.getApp({pid: 4}); await app.click(1); await app.getState({query: "rows"}); return "done";`, `"result":{"__computer_reads":[`}, + } { + script := filepath.Join(dir, "script.js") + if err := os.WriteFile(script, []byte(guardedScript(`{}`, tc.body, 42, 2, 0)), 0o600); err != nil { + t.Fatal(err) + } + out, err := exec.Command(node, harness, script).CombinedOutput() + if err != nil { + t.Fatalf("node: %v\n%s", err, out) + } + if got := string(out); !strings.Contains(got, tc.want) || !strings.Contains(got, "sleep:paced") { + t.Fatalf("script %s: got %s, want %s", tc.body, got, tc.want) + } + } +} + // In the Computer's own browser a script types with a short gap between // characters unless it names one, and one fixed wait lasts at most // scriptSleepLimit: pages are waited for with app.waitFor. diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 42d23889..4370c2d2 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -2,15 +2,10 @@ package localtools import ( "bufio" - "bytes" "context" - "encoding/base64" "encoding/json" "errors" "fmt" - "image" - "image/color" - "image/png" "io" "net" "net/http" @@ -124,35 +119,6 @@ func runToolFixture(mode string) { case "tools/list": result = map[string]any{"tools": []tool{{Name: "click", InputSchema: json.RawMessage(`{"additionalProperties":false,"properties":{"session":{"type":"string"}},"required":["session"],"type":"object"}`)}, {Name: "browser_snapshot", InputSchema: json.RawMessage(`{"additionalProperties":false,"properties":{},"type":"object"}`)}}} case "tools/call": - if strings.HasPrefix(mode, "pages") && request.Params.Name != "start_session" && request.Params.Name != "end_session" && request.Params.Name != "check_permissions" { - // A picture viewer: page k shows one white pixel at (10+2k, 6), - // and keys past page 2 change nothing. Mode pages-refused - // refuses the second key. - marker := os.Getenv("LINGER_LOCAL_TOOL_MARKER") - data, _ := os.ReadFile(marker) - presses := strings.Count(string(data), `"name":"press_key"`) - result = map[string]any{"content": []any{}} - switch request.Params.Name { - case "press_key": - f, _ := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) - _ = json.NewEncoder(f).Encode(request.Params) - _ = f.Close() - if mode == "pages-hang" { - // The key never completes, so its outcome stays unknown. - continue - } - if mode == "pages-refused" && presses == 1 { - result = map[string]any{"isError": true, "content": []any{map[string]string{"type": "text", "text": "off_space_or_ax_unresolved"}}} - } - case "get_window_state": - picture := image.NewRGBA(image.Rect(0, 0, 40, 30)) - picture.Set(10+2*min(presses, 2), 6, color.White) - var encoded bytes.Buffer - _ = png.Encode(&encoded, picture) - result = map[string]any{"content": []any{map[string]string{"type": "image", "mimeType": "image/png", "data": base64.StdEncoding.EncodeToString(encoded.Bytes())}}} - } - break - } if strings.HasPrefix(mode, "tabs") && request.Params.Name != "start_session" && request.Params.Name != "end_session" && request.Params.Name != "check_permissions" { // The Computer's Chrome: one window, where cmd+t adds a tab and // cmd+w closes one unless mode tabs-refused refuses it, and each @@ -1102,13 +1068,13 @@ func TestLocalToolCuaMacAgentChromeStopsWhenSomeoneElseChangesTheTabs(t *testing agentChromePID = func(context.Context, string) (int, string, error) { return 42, "--mute-audio", nil } t.Cleanup(func() { agentChromePID = original }) c.op.manager.agentChromeOwner = c.op - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = 1, 42, 9 + c.holdAgentChromeWindow(42, 9, 1, "") // The task's own cmd+t opens a tab that is the task's. if _, err := c.call(t.Context(), "hotkey", json.RawMessage(`{"pid":42,"window_id":9,"keys":["cmd","t"]}`)); err != nil { t.Fatal(err) } - if c.op.chromeTabs != 2 { - t.Fatalf("tabs after the task's own tab=%d, want 2", c.op.chromeTabs) + if c.op.chromeHeld[9].count != 2 { + t.Fatalf("tabs after the task's own tab=%d, want 2", c.op.chromeHeld[9].count) } // A tab someone else opens stops the task before its next input. f, err := os.OpenFile(marker, os.O_APPEND|os.O_WRONLY, 0o600) @@ -1147,7 +1113,7 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsOwnInputChangesLate(t *testing.T) agentChromePID = func(context.Context, string) (int, string, error) { return 42, "--mute-audio", nil } t.Cleanup(func() { agentChromePID = original }) c.op.manager.agentChromeOwner = c.op - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = 1, 42, 9 + c.holdAgentChromeWindow(42, 9, 1, "") // The click below presses a row of page text, which lists no press. c.snapshots = newSnapshotRows() c.snapshots.record(windowRead{Snapshot: "s1", PID: 42, Window: 9, Tree: `- [1] AXStaticText = "Sort by time"`}) @@ -1163,8 +1129,8 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsOwnInputChangesLate(t *testing.T) if err != nil { t.Fatal(err) } - if c.op.chromeTabs != step.want { - t.Fatalf("tabs after the task's own %s=%d, want %d", step.keys, c.op.chromeTabs, step.want) + if c.op.chromeHeld[9].count != step.want { + t.Fatalf("tabs after the task's own %s=%d, want %d", step.keys, c.op.chromeHeld[9].count, step.want) } if !strings.Contains(string(result), step.note) { t.Fatalf("result after the task's own %s=%s, want note %s", step.keys, result, step.note) @@ -1200,7 +1166,7 @@ func TestLocalToolCuaMacAgentChromeReadsTheTabStripOnlyAsFarAsItNeeds(t *testing agentChromePID = func(context.Context, string) (int, string, error) { return 42, "--mute-audio", nil } t.Cleanup(func() { agentChromePID = original }) c.op.manager.agentChromeOwner = c.op - c.op.chromeTabs, c.op.chromeTitle, c.op.chromePID, c.op.chromeWindow = 1, "Tab 0", 42, 9 + c.holdAgentChromeWindow(42, 9, 1, "Tab 0") reads := func() []string { var got []string for _, call := range fixtureCalls(t, marker) { @@ -1245,7 +1211,7 @@ func TestLocalToolCuaMacAgentChromeReadsTheTabStripOnlyAsFarAsItNeeds(t *testing // result reports the count without saying the input closed it. Fewer // tabs in a bounded read are proven by whole reads, since a strip split // across the bound would read as fewer too. - c.op.chromeTabs = 2 + c.op.chromeHeld[9] = heldTabs{count: 2, title: "Tab 0"} before = len(reads()) result, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)) if err != nil { @@ -1258,7 +1224,7 @@ func TestLocalToolCuaMacAgentChromeReadsTheTabStripOnlyAsFarAsItNeeds(t *testing t.Fatalf("result=%s, want the neutral note %s", result, note) } // A front tab that changed since the last input is read whole first. - c.op.chromeTitle = "Elsewhere" + c.op.chromeHeld[9] = heldTabs{count: c.op.chromeHeld[9].count, title: "Elsewhere"} if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); err != nil { t.Fatal(err) } @@ -1285,7 +1251,7 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsScriptsChange(t *testing.T) { agentChromePID = func(context.Context, string) (int, string, error) { return 42, "--mute-audio", nil } t.Cleanup(func() { agentChromePID = original }) c.op.manager.agentChromeOwner = c.op - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = 1, 42, 9 + c.holdAgentChromeWindow(42, 9, 1, "") // change writes a tab change to the fixture's record; shown says whether // the next read already shows it, as for a change made between inputs. change := func(keys string, shown bool) { @@ -1313,8 +1279,8 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsScriptsChange(t *testing.T) { if _, err := c.call(t.Context(), "run_script", json.RawMessage(step.script)); err != nil { t.Fatalf("script that sent %s: %v", step.keys, err) } - if c.op.chromeTabs != step.want { - t.Fatalf("tabs after the script that sent %s=%d, want %d", step.keys, c.op.chromeTabs, step.want) + if c.op.chromeHeld[9].count != step.want { + t.Fatalf("tabs after the script that sent %s=%d, want %d", step.keys, c.op.chromeHeld[9].count, step.want) } if _, err := c.call(t.Context(), "click", click); err != nil { t.Fatalf("input after the script that sent %s: %v", step.keys, err) @@ -1351,7 +1317,7 @@ func TestLocalToolCuaMacAgentChromeHoldsOnlyItsOwnWindow(t *testing.T) { agentChromePID = func(context.Context, string) (int, string, error) { return 42, "--mute-audio", nil } t.Cleanup(func() { agentChromePID = original }) c.op.manager.agentChromeOwner = c.op - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = 2, 42, 9 + c.holdAgentChromeWindow(42, 9, 2, "") // A script to another app while that browser runs is neither paced nor // held to its tabs, though the browser shows another count. if _, err := c.call(t.Context(), "run_script", json.RawMessage(`{"script":"return 1","args":{"pid":99}}`)); err != nil { @@ -1360,8 +1326,8 @@ func TestLocalToolCuaMacAgentChromeHoldsOnlyItsOwnWindow(t *testing.T) { if calls := fixtureCalls(t, marker); len(calls) == 0 || !strings.Contains(fmt.Sprint(calls[len(calls)-1]), "const limit = 0,") { t.Fatalf("calls=%v, want an unpaced script", calls) } - if c.op.chromeTabs != 2 { - t.Fatalf("tabs=%d after a script to another app, want 2", c.op.chromeTabs) + if c.op.chromeHeld[9].count != 2 { + t.Fatalf("tabs=%d after a script to another app, want 2", c.op.chromeHeld[9].count) } // A task stopped by a check opens no URL there either. c.op.chromeBlocked = "captcha" @@ -2104,8 +2070,8 @@ func TestLocalToolCuaMacEverydayChromeOpensTheComputersWindowAndTakesNoKeys(t *t } // launch_app keeps the front tab's title from list_windows, so the // first input, which changes no tab, reports no change. - if c.op.chromeTitle != "Tab 0" { - t.Fatalf("front title after launch_app=%q, want Tab 0", c.op.chromeTitle) + if got := c.op.chromeHeld[12].title; got != "Tab 0" { + t.Fatalf("front title after launch_app=%q, want Tab 0", got) } if result, err := c.call(t.Context(), "set_value", json.RawMessage(`{"pid":42,"window_id":12,"value":"kettle","element_token":"s1:1"}`)); err != nil || strings.Contains(string(result), "tab in front") { t.Fatalf("set_value result=%s err=%v, want no tab note", result, err) @@ -2212,7 +2178,7 @@ func TestLocalToolCuaMacAgentChromeOpensAFreshWindowForAnUnreadableOne(t *testin t.Fatalf("result=%s, want the note for a started browser", result) } // Typing there is paced at the default speed unless the Agent sets one. - c.op.chromeTabs, c.op.chromePID, c.op.chromeWindow = 1, 42, 9 + c.holdAgentChromeWindow(42, 9, 1, "") if _, err := c.call(t.Context(), "type_text", json.RawMessage(`{"pid":42,"window_id":9,"text":"kettle","element_token":"s1:1"}`)); err != nil { t.Fatal(err) } diff --git a/apps/computer/local-daemon/internal/linger/localtools/manager.go b/apps/computer/local-daemon/internal/linger/localtools/manager.go index 26393aee..1ac282b4 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/manager.go +++ b/apps/computer/local-daemon/internal/linger/localtools/manager.go @@ -229,13 +229,11 @@ type operation struct { chromeInputs int chromeBlocked string chromeSites map[string]bool - // chromeTabs is the tab count of the Computer's own browser after this - // task's last input there, and chromePID and chromeWindow the browser - // and window this task opened with launch_app; see checkAgentChromeTabs. - // chromeTitle is the front tab's title read with chromeTabs, from - // launch_app on. Guarded by mu. - chromeTabs int - chromeTitle string + // chromeHeld holds each window this task opened with launch_app to + // what it showed after the task's last input there, chromePID is the + // browser they belong to and chromeWindow the latest of them; see + // checkAgentChromeTabs. Guarded by mu. + chromeHeld map[float64]heldTabs chromePID int chromeWindow float64 closeOnce sync.Once From a3fadde4f84eb9130af041c12d9714c972bf3132 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 21:57:49 +0800 Subject: [PATCH 27/30] fix(computer): refuse one input on a tab change instead of ending the task A tab that the task's own input opened after the reads that followed it looks the same as one someone else opened, and ending the task on it threw away a task that was working. The check before an input now refuses only that input, holds the window to the tabs it shows now, and asks the Agent to read the window and go on only when the front tab is its page. The humanlike-browse Skill says so and ships as 1.0.8, since an environment sealed 1.0.7. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 7 +++++-- .../internal/linger/localtools/agent_chrome.go | 13 ++++++++----- .../linger/localtools/localtools_test.go | 16 ++++++++++------ skills/builtin/catalog.yaml | 8 ++++---- .../humanlike-browse/{1.0.7 => 1.0.8}/SKILL.md | 3 ++- 5 files changed, 29 insertions(+), 18 deletions(-) rename skills/builtin/humanlike-browse/{1.0.7 => 1.0.8}/SKILL.md (96%) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 43658b50..15d64b3f 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -376,8 +376,11 @@ the task; `rate_limited` also puts the site (its registrable domain) in a Once a task opens a site there, the browser tools refuse that site for the rest of the task, and `launch_app` refuses to open it in another browser. A change of the front tab between the task's inputs that also changed the -number of tabs stops the task with `COMPUTER_TAB_CHANGED`, because the next -input could reach another page. The Daemon reads the tab strip before an input +number of tabs refuses that one input with `COMPUTER_TAB_CHANGED`, because it +could reach another page: the Agent reads the window and goes on only when +the front tab is its page. A tab its own input opened late, after the reads +that followed the input, looks the same, so the task is not stopped, and the +next input is held to the tabs the window shows then. The Daemon reads the tab strip before an input only when the window's title in `list_windows` differs from the front tab's title after the last input, so a tab opened or closed behind the front tab does not stop the task: the input still reaches the task's page, and its diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index 2eddf2a9..a784b454 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -732,9 +732,8 @@ func (c *cuaSession) agentChromeTabWindow(pid, window any) (int, float64, heldTa // checkAgentChromeTabs holds the task to the tabs it left: the Computer's // own Chrome keeps the tab count that followed this task's last input, so a -// different count means someone else opened or closed a tab, and the tab in -// front may not be this task's. The task then stops, as after a reported -// check. A strip that cannot be read, or a task that has no count yet, is not +// different count means a tab opened or closed since, and the tab in front +// may not be this task's. The input is then refused, once. A strip that cannot be read, or a task that has no count yet, is not // held. The strip is read only when the window's title, which list_windows // gives without a read, differs from the front tab's title after that input. func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) error { @@ -758,10 +757,14 @@ func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) } return nil } + // A tab that the task's own input opened after the reads that followed + // it looks the same as one someone else opened, so this input is + // refused, not the task: the Agent reads the window and judges the + // front tab, and the next input is held to what the window shows now. c.op.mu.Lock() - c.op.chromeBlocked = "a change of tabs" + c.op.chromeHeld[id] = heldTabs{count: got, title: front} c.op.mu.Unlock() - return fmt.Errorf("COMPUTER_TAB_CHANGED: the Computer's own browser had %d tab(s) after this task's last input and has %d now, so someone else opened or closed a tab and the tab in front may not be this task's; nothing was sent. Tell the user and end the task: %w", want, got, errNotSent) + return fmt.Errorf("COMPUTER_TAB_CHANGED: the Computer's own browser had %d tab(s) after this task's last input and has %d now, with %q in front; a tab may have opened late after that input, or someone else changed the tabs. Nothing was sent. Read the window: go on only if the tab in front is this task's page, otherwise report what was done and end the task: %w", want, got, front, errNotSent) } // recordAgentChromeTabs keeps the tab count after this task's own input or diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 4370c2d2..99769440 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1076,7 +1076,8 @@ func TestLocalToolCuaMacAgentChromeStopsWhenSomeoneElseChangesTheTabs(t *testing if c.op.chromeHeld[9].count != 2 { t.Fatalf("tabs after the task's own tab=%d, want 2", c.op.chromeHeld[9].count) } - // A tab someone else opens stops the task before its next input. + // A tab someone else opens refuses the next input once, with nothing + // sent; the input after it is held to the tabs the window shows now. f, err := os.OpenFile(marker, os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { t.Fatal(err) @@ -1087,12 +1088,15 @@ func TestLocalToolCuaMacAgentChromeStopsWhenSomeoneElseChangesTheTabs(t *testing if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_TAB_CHANGED") { t.Fatalf("err=%v, want the tab change refused", err) } - if _, err := c.call(t.Context(), "press_key", json.RawMessage(`{"pid":42,"window_id":9,"key":"escape"}`)); !strings.Contains(fmt.Sprint(err), "COMPUTER_SITE_BLOCKED") { - t.Fatalf("err=%v, want the task held", err) - } if calls := fixtureCalls(t, marker); len(calls) != sent { t.Fatalf("calls after the change=%v, want none", calls[sent:]) } + if c.op.chromeHeld[9].count != 3 { + t.Fatalf("tabs held after the refusal=%d, want 3", c.op.chromeHeld[9].count) + } + if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); err != nil { + t.Fatalf("input after the refusal: %v", err) + } if err := c.drain(); err != nil { t.Fatal(err) } @@ -1286,8 +1290,8 @@ func TestLocalToolCuaMacAgentChromeKeepsTabsItsScriptsChange(t *testing.T) { t.Fatalf("input after the script that sent %s: %v", step.keys, err) } } - // A tab someone else opens between inputs stops the next script before - // it runs. + // A tab someone else opens between inputs refuses the next script + // before it runs. change(`["cmd","t"]`, true) before, _ := os.ReadFile(marker) if _, err := c.call(t.Context(), "run_script", json.RawMessage(`{"script":"return 1"}`)); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_TAB_CHANGED") { diff --git a/skills/builtin/catalog.yaml b/skills/builtin/catalog.yaml index c9da046f..142dd755 100644 --- a/skills/builtin/catalog.yaml +++ b/skills/builtin/catalog.yaml @@ -219,14 +219,14 @@ skills: version: 1.0.0 package_path: builtin/personal-profile/1.0.0 - skill_id: simplexai_humanlike_browse - bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_7 + bootstrap_manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_8 default_agent_types: - anotherme - general manifests: - - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_7 - version: 1.0.7 - package_path: builtin/humanlike-browse/1.0.7 + - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_8 + version: 1.0.8 + package_path: builtin/humanlike-browse/1.0.8 - manifest_id: skill_manifest_simplexai_humanlike_browse_1_0_4 version: 1.0.4 package_path: builtin/humanlike-browse/1.0.4 diff --git a/skills/builtin/humanlike-browse/1.0.7/SKILL.md b/skills/builtin/humanlike-browse/1.0.8/SKILL.md similarity index 96% rename from skills/builtin/humanlike-browse/1.0.7/SKILL.md rename to skills/builtin/humanlike-browse/1.0.8/SKILL.md index 9d2e0515..9c5bfc64 100644 --- a/skills/builtin/humanlike-browse/1.0.7/SKILL.md +++ b/skills/builtin/humanlike-browse/1.0.8/SKILL.md @@ -76,4 +76,5 @@ If the user takes over the window, read the page and the account again before th - When a page shows a captcha, a code or QR check, a sign-in wall or a too-many-requests message, call `report_blocker` with that kind and the page's URL at once. Then tell the user in the Room what the page asks for and that they can deal with it in that browser window, report what was done, and end the task. - When only one item cannot be viewed, call `report_blocker` with `content_unavailable` and its URL, and skip it. - Never pass a check yourself, switch accounts or retry around a limit. -- When the Computer refuses input with `COMPUTER_INPUT_BUDGET`, `COMPUTER_SITE_COOLDOWN`, `COMPUTER_SITE_BLOCKED` or `COMPUTER_TAB_CHANGED`, report what was done and end the task. +- When the Computer refuses input with `COMPUTER_INPUT_BUDGET`, `COMPUTER_SITE_COOLDOWN` or `COMPUTER_SITE_BLOCKED`, report what was done and end the task. +- When it refuses input with `COMPUTER_TAB_CHANGED`, read the window. Go on only if the tab in front is the page this task was working on; otherwise report what was done and end the task. From a0f0f0562cf11ed8b26bab6608f4cecf0b588180 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 22:00:55 +0800 Subject: [PATCH 28/30] test(computer): prove the window settle wait counts from its candidate The settle wait of the everyday Chrome window search is its own type, so a test shows that a window that showed briefly lends no time to the window that takes its place. Co-Authored-By: Claude Opus 5.5 --- .../linger/localtools/agent_browser.go | 38 ++++++++++++------- .../linger/localtools/agent_browser_test.go | 21 ++++++++++ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go index d19ae54f..afa9f6a7 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser.go @@ -431,8 +431,7 @@ func (c *cuaSession) launchEverydayChrome(ctx context.Context, b agentBrowser, u // so a home page that moves to another site, such as a sign-in page, still // finds its window. Windows that stay ambiguous are refused. func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, pid int, before map[float64]bool, site string) (listedWindow, int, error) { - var seen time.Time - var candidate float64 + var wait windowSettle for deadline := time.Now().Add(sharedBrowserWindowWait); ; { windows, err := c.appWindows(ctx, pid) if err != nil { @@ -448,17 +447,7 @@ func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, p matching = append(matching, w) } } - // The wait counts from when the lone candidate showed: one that - // left, or another in its place, starts it again. - switch { - case len(fresh) == 0: - seen = time.Time{} - case len(fresh) == 1 && fresh[0].WindowID != candidate: - seen, candidate = time.Now(), fresh[0].WindowID - case seen.IsZero(): - seen = time.Now() - } - settled := !seen.IsZero() && time.Since(seen) >= everydayChromeSettle + settled := wait.settled(fresh, time.Now()) switch { case len(matching) == 1: return matching[0], 1, nil @@ -476,6 +465,29 @@ func (c *cuaSession) everydayChromeWindow(ctx context.Context, b agentBrowser, p } } +// windowSettle times how long the new windows of a launch have stood. The +// wait counts from when the lone candidate showed: one that left, or another +// in its place, starts it again, so a window that showed only briefly cannot +// lend its time to the window that follows it. +type windowSettle struct { + seen time.Time + candidate float64 +} + +// settled reports whether fresh, the new windows seen at now, have stood for +// everydayChromeSettle. +func (w *windowSettle) settled(fresh []listedWindow, now time.Time) bool { + switch { + case len(fresh) == 0: + w.seen = time.Time{} + case len(fresh) == 1 && fresh[0].WindowID != w.candidate: + w.seen, w.candidate = now, fresh[0].WindowID + case w.seen.IsZero(): + w.seen = now + } + return !w.seen.IsZero() && now.Sub(w.seen) >= everydayChromeSettle +} + // everydayChromeSettle is how long the Computer lets the user's Chrome // settle before it trusts what it sees: the window list must stay the same // that long after a start, since Chrome reopens restored windows one after diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go index 7d56ca16..b5da6b0f 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_browser_test.go @@ -3,6 +3,7 @@ package localtools import ( "strings" "testing" + "time" ) func TestCountTabsReadsChromeEdgeAndFirefoxStrips(t *testing.T) { @@ -106,3 +107,23 @@ func TestAgentChromeHoldsEachWindowTheTaskOpened(t *testing.T) { t.Fatal("a window of the earlier browser is still held") } } + +// A window that showed briefly lends no time to the one that takes its +// place: the settle wait counts from when the lone candidate showed. +func TestWindowSettleCountsFromTheCandidate(t *testing.T) { + start := time.Unix(1000, 0) + a, b := []listedWindow{{WindowID: 21}}, []listedWindow{{WindowID: 22}} + var wait windowSettle + if wait.settled(a, start) { + t.Fatal("window A settled when it showed") + } + if wait.settled(b, start.Add(everydayChromeSettle)) { + t.Fatal("window B took the time window A stood") + } + if !wait.settled(b, start.Add(2*everydayChromeSettle)) { + t.Fatal("window B did not settle after standing the settle time") + } + if wait.settled(nil, start.Add(3*everydayChromeSettle)) || wait.settled(b, start.Add(3*everydayChromeSettle)) { + t.Fatal("window B settled again at once after it left and came back") + } +} From 9c602e41f206154258ebbc1b55d628afb4c76378 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 22:19:18 +0800 Subject: [PATCH 29/30] fix(computer): confirm a Codex stop when the process left on its own A cancelled Start whose Codex process exited before the stop closed its stdin was reported unconfirmed: the stop then counted the exit as its own, and only an outside exit excused the cleanup error the gone process left. A reaped leader and an empty group are the evidence that nothing runs the turn, so a cleanup that failed because the process had gone is excused whoever ended it. Co-Authored-By: Claude Opus 5.5 --- .../internal/provider/native/codex_process.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/computer/local-daemon/internal/provider/native/codex_process.go b/apps/computer/local-daemon/internal/provider/native/codex_process.go index 1110edd5..0c8624ed 100644 --- a/apps/computer/local-daemon/internal/provider/native/codex_process.go +++ b/apps/computer/local-daemon/internal/provider/native/codex_process.go @@ -306,10 +306,12 @@ func (p *codexProcess) stopTree() bool { p.mu.Unlock() p.reapWithGrace() gone := p.groupGone() - p.mu.Lock() - external := p.external - p.mu.Unlock() - excused := cleanupErr == nil || (external && errors.Is(cleanupErr, errCodexChildGone)) + // A cleanup that failed because the process had gone needs no excuse + // beyond the reaped leader and the empty group: nothing is left to run + // the turn or its terminals. Who ended the process does not matter, and + // cannot be told reliably: the leader can exit just before the stop + // above closes its stdin, which then counts the exit as the stop's. + excused := cleanupErr == nil || errors.Is(cleanupErr, errCodexChildGone) return p.confirmIf(p.isReaped() && gone && excused, logger) } From 7c54a58fda029adbab1fc195e741666c47506295 Mon Sep 17 00:00:00 2001 From: jeffrey Date: Thu, 12 Nov 2026 22:46:37 +0800 Subject: [PATCH 30/30] fix(computer): hold inputs after a tab change until the Agent reads the window After a change of tabs refused an input, the next input went through even when the Agent had not looked, since the Computer's own strip read does not stale the Agent's token. The window now stays refused until the Agent's own get_window_state of it. The Codex stop's cleanup excuse is its own function with a deterministic test, and the tab-change test is named for what it proves. Co-Authored-By: Claude Opus 5.5 --- apps/computer/local-daemon/README.md | 11 ++--- .../linger/localtools/agent_chrome.go | 44 ++++++++++++++----- .../internal/linger/localtools/cua.go | 1 + .../linger/localtools/localtools_test.go | 12 ++++- .../internal/provider/native/codex_process.go | 18 +++++--- .../provider/native/codex_units_test.go | 20 +++++++++ 6 files changed, 81 insertions(+), 25 deletions(-) diff --git a/apps/computer/local-daemon/README.md b/apps/computer/local-daemon/README.md index 15d64b3f..8ea8d4e2 100644 --- a/apps/computer/local-daemon/README.md +++ b/apps/computer/local-daemon/README.md @@ -376,11 +376,12 @@ the task; `rate_limited` also puts the site (its registrable domain) in a Once a task opens a site there, the browser tools refuse that site for the rest of the task, and `launch_app` refuses to open it in another browser. A change of the front tab between the task's inputs that also changed the -number of tabs refuses that one input with `COMPUTER_TAB_CHANGED`, because it -could reach another page: the Agent reads the window and goes on only when -the front tab is its page. A tab its own input opened late, after the reads -that followed the input, looks the same, so the task is not stopped, and the -next input is held to the tabs the window shows then. The Daemon reads the tab strip before an input +number of tabs refuses inputs there with `COMPUTER_TAB_CHANGED`, because they +could reach another page, until the Agent reads the window with +`get_window_state`; it goes on only when the front tab is its page. A tab its +own input opened late, after the reads that followed the input, looks the +same, so the task is not stopped, and the next input is held to the tabs the +window shows then. The Daemon reads the tab strip before an input only when the window's title in `list_windows` differs from the front tab's title after the last input, so a tab opened or closed behind the front tab does not stop the task: the input still reaches the task's page, and its diff --git a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go index a784b454..7f4e7151 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go +++ b/apps/computer/local-daemon/internal/linger/localtools/agent_chrome.go @@ -691,10 +691,13 @@ func (c *cuaSession) agentChromeWindowTitle(ctx context.Context, pid int, window // heldTabs is what a window the task opened with launch_app showed after // the task's last input there: its tab count and its front tab's title. A -// count of 0 holds nothing, as for a strip that could not be read. +// count of 0 holds nothing, as for a strip that could not be read. unread +// marks a window whose change of tabs refused an input and that the Agent +// has not read since: until it does, every input there is refused. type heldTabs struct { - count int - title string + count int + title string + unread bool } // holdAgentChromeWindow makes window, in the browser that runs as pid, the @@ -733,15 +736,20 @@ func (c *cuaSession) agentChromeTabWindow(pid, window any) (int, float64, heldTa // checkAgentChromeTabs holds the task to the tabs it left: the Computer's // own Chrome keeps the tab count that followed this task's last input, so a // different count means a tab opened or closed since, and the tab in front -// may not be this task's. The input is then refused, once. A strip that cannot be read, or a task that has no count yet, is not -// held. The strip is read only when the window's title, which list_windows -// gives without a read, differs from the front tab's title after that input. +// may not be this task's. Inputs are then refused until the Agent reads the +// window. A strip that cannot be read, or a task that has no count yet, is +// not held. The strip is read only when the window's title, which +// list_windows gives without a read, differs from the front tab's title +// after that input. func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) error { number, id, held, ok := c.agentChromeTabWindow(pid, window) want, title := held.count, held.title if !ok || want == 0 { return nil } + if held.unread { + return fmt.Errorf("COMPUTER_TAB_CHANGED: the tabs of the Computer's own browser changed and this task has not read the window since, so the tab in front may not be this task's. Nothing was sent. Read the window with get_window_state: go on only if the tab in front is this task's page, otherwise report what was done and end the task: %w", errNotSent) + } if title != "" && c.agentChromeWindowTitle(ctx, number, id) == title { // The front tab still shows what this task's last input left. A // tab someone opened behind it changes nothing this input reaches, @@ -758,13 +766,27 @@ func (c *cuaSession) checkAgentChromeTabs(ctx context.Context, pid, window any) return nil } // A tab that the task's own input opened after the reads that followed - // it looks the same as one someone else opened, so this input is - // refused, not the task: the Agent reads the window and judges the - // front tab, and the next input is held to what the window shows now. + // it looks the same as one someone else opened, so inputs are refused, + // not the task: once the Agent has read the window and judged the front + // tab, its next input is held to what the window shows now. c.op.mu.Lock() - c.op.chromeHeld[id] = heldTabs{count: got, title: front} + c.op.chromeHeld[id] = heldTabs{count: got, title: front, unread: true} + c.op.mu.Unlock() + return fmt.Errorf("COMPUTER_TAB_CHANGED: the Computer's own browser had %d tab(s) after this task's last input and has %d now, with %q in front; a tab may have opened late after that input, or someone else changed the tabs. Nothing was sent. Read the window with get_window_state: go on only if the tab in front is this task's page, otherwise report what was done and end the task: %w", want, got, front, errNotSent) +} + +// readAgentChromeWindow takes the Agent's own read of a window it opened +// with launch_app as its look at the tab in front after a change of tabs, +// so inputs there go through again. +func (c *cuaSession) readAgentChromeWindow(pid, window any) { + _, id, held, ok := c.agentChromeTabWindow(pid, window) + if !ok || !held.unread { + return + } + held.unread = false + c.op.mu.Lock() + c.op.chromeHeld[id] = held c.op.mu.Unlock() - return fmt.Errorf("COMPUTER_TAB_CHANGED: the Computer's own browser had %d tab(s) after this task's last input and has %d now, with %q in front; a tab may have opened late after that input, or someone else changed the tabs. Nothing was sent. Read the window: go on only if the tab in front is this task's page, otherwise report what was done and end the task: %w", want, got, front, errNotSent) } // recordAgentChromeTabs keeps the tab count after this task's own input or diff --git a/apps/computer/local-daemon/internal/linger/localtools/cua.go b/apps/computer/local-daemon/internal/linger/localtools/cua.go index 5eb31dfd..e1da5c94 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/cua.go +++ b/apps/computer/local-daemon/internal/linger/localtools/cua.go @@ -946,6 +946,7 @@ func (c *cuaSession) finishInput(ctx context.Context, name string, arguments map } if err == nil && name == "get_window_state" && arguments["include_accessibility_tree"] != false { c.op.reads.observe(result) + c.readAgentChromeWindow(arguments["pid"], arguments["window_id"]) } if err == nil && inputTools[name] && ctx.Value(cuaInternalInput{}) == nil && runtime.GOOS != "darwin" { result = c.withWindowState(ctx, arguments, result) diff --git a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go index 99769440..0df6e7e9 100644 --- a/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go +++ b/apps/computer/local-daemon/internal/linger/localtools/localtools_test.go @@ -1053,7 +1053,7 @@ func TestLocalToolCuaMacAgentChromeOpensAURLOnlyInANewTab(t *testing.T) { } } -func TestLocalToolCuaMacAgentChromeStopsWhenSomeoneElseChangesTheTabs(t *testing.T) { +func TestLocalToolCuaMacAgentChromeRefusesInputsWhenSomeoneElseChangesTheTabs(t *testing.T) { if runtime.GOOS != "darwin" { t.Skip("the Computer's own Chrome is macOS only") } @@ -1094,8 +1094,16 @@ func TestLocalToolCuaMacAgentChromeStopsWhenSomeoneElseChangesTheTabs(t *testing if c.op.chromeHeld[9].count != 3 { t.Fatalf("tabs held after the refusal=%d, want 3", c.op.chromeHeld[9].count) } + // The Agent's old token still names a row, so only its own read of the + // window lets inputs through again. + if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); !errors.Is(err, errNotSent) || !strings.Contains(err.Error(), "COMPUTER_TAB_CHANGED") { + t.Fatalf("input before a read err=%v, want refused", err) + } + if _, err := c.call(t.Context(), "get_window_state", json.RawMessage(`{"pid":42,"window_id":9}`)); err != nil { + t.Fatal(err) + } if _, err := c.call(t.Context(), "click", json.RawMessage(`{"pid":42,"window_id":9,"element_token":"s1:1"}`)); err != nil { - t.Fatalf("input after the refusal: %v", err) + t.Fatalf("input after the read: %v", err) } if err := c.drain(); err != nil { t.Fatal(err) diff --git a/apps/computer/local-daemon/internal/provider/native/codex_process.go b/apps/computer/local-daemon/internal/provider/native/codex_process.go index 0c8624ed..cfb37cd1 100644 --- a/apps/computer/local-daemon/internal/provider/native/codex_process.go +++ b/apps/computer/local-daemon/internal/provider/native/codex_process.go @@ -306,13 +306,17 @@ func (p *codexProcess) stopTree() bool { p.mu.Unlock() p.reapWithGrace() gone := p.groupGone() - // A cleanup that failed because the process had gone needs no excuse - // beyond the reaped leader and the empty group: nothing is left to run - // the turn or its terminals. Who ended the process does not matter, and - // cannot be told reliably: the leader can exit just before the stop - // above closes its stdin, which then counts the exit as the stop's. - excused := cleanupErr == nil || errors.Is(cleanupErr, errCodexChildGone) - return p.confirmIf(p.isReaped() && gone && excused, logger) + return p.confirmIf(p.isReaped() && gone && codexCleanupExcused(cleanupErr), logger) +} + +// codexCleanupExcused reports whether a turn cleanup leaves the stop +// confirmable once the leader is reaped and its group is empty. A cleanup +// that failed because the process had gone needs no other excuse: nothing is +// left to run the turn or its terminals. Who ended the process does not +// matter, and cannot be told reliably: the leader can exit just before the +// stop closes its stdin, which then counts the exit as the stop's. +func codexCleanupExcused(cleanupErr error) bool { + return cleanupErr == nil || errors.Is(cleanupErr, errCodexChildGone) } func (p *codexProcess) reapWithGrace() { diff --git a/apps/computer/local-daemon/internal/provider/native/codex_units_test.go b/apps/computer/local-daemon/internal/provider/native/codex_units_test.go index 626b78c3..a15f4602 100644 --- a/apps/computer/local-daemon/internal/provider/native/codex_units_test.go +++ b/apps/computer/local-daemon/internal/provider/native/codex_units_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "log/slog" "os" @@ -493,3 +494,22 @@ func TestCodexToolOutputTokenLimitFollowsContextWindow(t *testing.T) { t.Fatal("windows that keep the global limit must keep the thread") } } + +// A cleanup that failed only because the process had gone, which the stop +// may have counted as its own exit, still lets a reaped, empty group confirm +// the stop; any other cleanup failure does not. +func TestCodexCleanupExcusedWhenTheProcessHadGone(t *testing.T) { + gone := fmt.Errorf("turn/start outcome unconfirmed: %w", errCodexChildGone) + for _, tc := range []struct { + err error + want bool + }{ + {nil, true}, + {gone, true}, + {fmt.Errorf("interrupt Codex turn: %w", context.DeadlineExceeded), false}, + } { + if got := codexCleanupExcused(tc.err); got != tc.want { + t.Fatalf("codexCleanupExcused(%v) = %v, want %v", tc.err, got, tc.want) + } + } +}