diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 04927c2a..b1a464fb 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -75,7 +75,9 @@ To move it to another cua-driver version: on what it finds; its release's tarball, checked by sha256, is unpacked into the driver's `bin/` by `install.sh`. Agents get what its own MCP server, `agent-desktop mcp`, serves, as `mcp.json` pins it: its instructions and its 67 `desktop_*` tools, each call forwarded to a server of its own, and its skill, its references and its guide for macOS -leading the computer's skill. The `computer_*` tools, gate 3's probe, `computer-ready` and recording run on +leading the computer's skill. The agent doesn't share the computer's files, so each screenshot a call saves to an +`output_path`, in a batch too, comes back beside the answer (up to 8 MB in all, a line naming the rest), and the +instructions add a line saying other files it writes stay on the computer. The `computer_*` tools, gate 3's probe, `computer-ready` and recording run on `MacOSDriver` (`agentenv_protocol.computer._macos`): ScreenCaptureKit captures the display and Quartz events, posted where the hardware's are, send the input, from the driver's own process. `[install.macos]` grants Accessibility, Post Event and Screen Recording to the driver's Python, its venv's own copy of the pinned one, and the LaunchAgent starts @@ -95,8 +97,8 @@ chaining the actions it's sure of and the read after them with `&&`. Agents get the harness on a list of command lines in order, the first that fails stopping the rest, with no shell between, and a line that doesn't start with one of its commands running nothing; instructions that say how its shell lines map to the tool and send the agent to its skill, `skill.md` as pinned, which the computer's skill also leads with (Claude Code -cuts a server's instructions at 4,096 characters, so the skill itself isn't in them); and a read's screenshot as an -image beside its text. Its `computer_*` tools and its permissions are agent-desktop's, from +cuts a server's instructions at 4,096 characters, so the skill itself isn't in them); and a read's screenshot, a +`--keep` copy too, as an image beside its text, the instructions saying the files it names stay on the computer. Its `computer_*` tools and its permissions are agent-desktop's, from `MacOSDriver` and a grant to its own Python. The harness gets none of the driver's environment, and its telemetry is off (`DO_NOT_TRACK`). `computer-ready` checks its version, that it prints the pinned skill, and the TCC permissions. Name it as `macos-harness`. diff --git a/src/agent_env/drivers/envs/agent-desktop/driver.py b/src/agent_env/drivers/envs/agent-desktop/driver.py index 145edb20..aa077df5 100644 --- a/src/agent_env/drivers/envs/agent-desktop/driver.py +++ b/src/agent_env/drivers/envs/agent-desktop/driver.py @@ -1,7 +1,8 @@ """Vercel's agent-desktop as a computer driver on macOS. agent-desktop reads apps' accessibility trees and acts on what it finds; its own MCP server, ``agent-desktop mcp``, serves each of its commands as a tool, and agents get what that server serves at its pinned version (``mcp.json``): its instructions and its tools, each call forwarded to a server of -its own and answered as the server answers it, with agent-desktop's skill leading the computer's. The computer/v1 +its own and answered as the server answers it, with each screenshot the call saved to a file beside the answer, since +the agent can't read the computer's files; agent-desktop's skill leads the computer's. The computer/v1 operations are MacOSDriver's. This process's Python is granted Accessibility, Post Event and Screen Recording, and agent-desktop, which it starts, has them as its own. A tool can't pass agent-desktop's global flags, so the server's come from the driver's ``headed`` setting: off, agent-desktop's default, it acts through the accessibility API alone and @@ -11,6 +12,7 @@ from __future__ import annotations import asyncio +import base64 import contextlib import functools import hashlib @@ -21,7 +23,7 @@ from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client -from mcp.types import CallToolResult, TextContent +from mcp.types import CallToolResult, ImageContent, TextContent from agentenv_protocol import EnvironmentTool, ForwardedTool, server_instructions from agentenv_protocol.computer import ComputerSettings, MacOSDriver, ReadyCheck, computer_driver @@ -40,6 +42,12 @@ PATH = "/usr/bin:/bin:/usr/sbin:/sbin" #: How long one of its commands that reads its skills may take. SKILL_TIMEOUT_S = 60 +#: The most bytes of saved screenshots one answer carries. +MAX_IMAGE_BYTES = 8 * 1024 * 1024 +#: What the instructions add to agent-desktop's: the computer's files stay on it. +FILES = ("Files these tools write stay on this computer, which you don't share: a screenshot saved to an output_path, " + "in a batch too, comes back with its answer, up to 8 MB of them an answer, and other files, such as a trace " + "export, can't be read from where you run.") @computer_driver(name="agent-desktop", supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}]) @@ -59,7 +67,7 @@ def forwarded_tools(self) -> list[ForwardedTool]: @server_instructions def agent_desktop_instructions(self) -> str: - return _manifest()["instructions"] + return f"{_manifest()['instructions']}\n\n{FILES}" async def ready_checks(self) -> list[ReadyCheck]: """agent-desktop's executables and the tools its server lists, as pinned, checked until they pass once, and the @@ -88,10 +96,11 @@ async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: that fails before answering may have run the command.""" try: async with self._server() as session: - return await session.call_tool(tool, arguments) + result = await session.call_tool(tool, arguments) except Exception as e: return CallToolResult(isError=True, content=[TextContent(type="text", text=( f"agent-desktop's server failed before it answered, so {tool} may have run: {type(e).__name__}: {e}"))]) + return _with_images(result) @contextlib.asynccontextmanager async def _server(self) -> AsyncIterator[ClientSession]: @@ -122,6 +131,47 @@ async def _pinned_checks(self) -> list[ReadyCheck]: else f"these tools differ from {MANIFEST.name}: {', '.join(changed)}")] +def _with_images(result: CallToolResult) -> CallToolResult: + """``result`` with each screenshot its answer saved to a file, up to ``MAX_IMAGE_BYTES`` in all, and a line naming + the saved ones it doesn't carry.""" + images, total, past, unread = [], 0, [], [] + for path in dict.fromkeys(_saved(result)): + try: + if total + path.stat().st_size > MAX_IMAGE_BYTES: + past.append(path) + continue + data = path.read_bytes() + except OSError: + unread.append(path) + continue + total += len(data) + images.append(ImageContent(type="image", data=base64.b64encode(data).decode(), mimeType="image/png")) + notes = [f"{label}: {', '.join(map(str, paths))}" for label, paths in ( + ("Saved but not attached, past the most one answer carries", past), ("Saved but not readable", unread)) if paths] + content = [*images, *(TextContent(type="text", text=note) for note in notes)] + return result.model_copy(update={"content": [*result.content, *content]}) if content else result + + +def _saved(result: CallToolResult) -> list[Path]: + """The files ``result``'s screenshots were saved to, a batch's included: each one's ``data.path``, a relative one + from the server's working directory, which is this process's.""" + found: list[Path] = [] + + def walk(answer: Any) -> None: + if not isinstance(answer, dict) or not isinstance(data := answer.get("data"), dict): + return + if answer.get("command") == "screenshot" and data.get("path"): + found.append(Path(data["path"])) + for each in data.get("results") or []: + walk(each) + + for block in result.content: + if isinstance(block, TextContent): + with contextlib.suppress(ValueError): + walk(json.loads(block.text)) + return found + + @functools.cache def _manifest() -> dict: return json.loads(MANIFEST.read_text()) diff --git a/src/agent_env/drivers/envs/macos-harness/driver.py b/src/agent_env/drivers/envs/macos-harness/driver.py index e816824e..254ce6ca 100644 --- a/src/agent_env/drivers/envs/macos-harness/driver.py +++ b/src/agent_env/drivers/envs/macos-harness/driver.py @@ -45,8 +45,9 @@ "guide is its skill: before your first command, run [[\"skill\"]] and follow it. Where the skill shows " "`macos-harness ...` in a shell, call macos_harness with the words after `macos-harness`; a chain joined " "with `&&` is a list of command lines in one call. Text needs no quoting or heredoc: pass it as one word, " - "such as [\"paste\", \"--app\", \"TextEdit\", \"--text\", \"line one\\nline two\"]. A read's screenshot " - "comes back as an image.") + "such as [\"paste\", \"--app\", \"TextEdit\", \"--text\", \"line one\\nline two\"]. A read's screenshot, " + "a --keep copy too, comes back as an image; the files the harness names stay on this computer, which you don't " + "share, so open none from where you run.") @computer_driver(name="macos-harness", supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}]) diff --git a/tst/unit/drivers/agent_desktop_driver_test.py b/tst/unit/drivers/agent_desktop_driver_test.py index e552c17a..df773014 100644 --- a/tst/unit/drivers/agent_desktop_driver_test.py +++ b/tst/unit/drivers/agent_desktop_driver_test.py @@ -1,9 +1,10 @@ """The agent-desktop driver against a fake agent-desktop: its instructions and tools are its server's as pinned, each call -reaches a server of its own, ``--headed`` when the driver's setting says, with none of the driver's environment; a -server that fails answers an error; readiness checks the pinned executables and the tools the server lists; and its +reaches a server of its own, ``--headed`` when the driver's setting says, with none of the driver's environment, and +the screenshots it saves come back beside its answer; a server that fails answers an error; readiness checks the pinned executables and the tools the server lists; and its skill is agent-desktop's own, with its references and its guide for macOS.""" from __future__ import annotations +import base64 import hashlib import importlib.util import json @@ -40,9 +41,22 @@ async def tools(): return [types.Tool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]) for t in manifest["tools"]] + def screenshot(args): + if not Path(args["output_path"]).is_dir(): + Path(args["output_path"]).write_bytes(b"\\x89PNG fake") + return {"ok": True, "command": "screenshot", "data": {"format": "png", "path": args["output_path"]}} + @server.call_tool(validate_input=False) async def call(name, arguments): - return [types.TextContent(type="text", text=json.dumps({"ok": True, "command": name, "data": arguments}))] + if name == "desktop_screenshot": + answer = screenshot(arguments) + elif name == "desktop_batch": + answer = {"ok": True, "command": "batch", "data": {"results": [ + screenshot(c["args"]) if c["command"] == "screenshot" else {"ok": True, "command": c["command"]} + for c in json.loads(arguments["commands_json"])]}} + else: + answer = {"ok": True, "command": name, "data": arguments} + return [types.TextContent(type="text", text=json.dumps(answer))] async def main(): async with stdio_server() as (read, write): @@ -88,7 +102,8 @@ def test_its_instructions_and_tools_are_its_servers_at_the_pinned_version(module assert [t.tool.name for t in driver.forwarded_tools()] == [t["name"] for t in manifest["tools"]] assert len(manifest["tools"]) == 67 and "desktop_skills_get" in manifest["instructions"] - assert driver.agent_desktop_instructions() == manifest["instructions"] and not driver.settings.headed + assert driver.agent_desktop_instructions() == f"{manifest['instructions']}\n\n{module.FILES}" + assert not driver.settings.headed @pytest.mark.asyncio @@ -107,6 +122,35 @@ async def test_a_call_reaches_a_server_of_its_own_headed_when_the_setting_says_w assert "AGENTENV_CONTROL_TOKEN" not in first["env"] and "HOME" in first["env"] +@pytest.mark.asyncio +async def test_each_screenshot_a_call_saves_comes_back_beside_its_answer_up_to_the_cap_and_no_other_file_does( + module, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + named, unreadable = tmp_path / "named.png", tmp_path / "folder.png" + named.write_bytes(b"\x89PNG named") + unreadable.mkdir() + driver = module.AgentDesktop() + + saved = await driver._forward("desktop_screenshot", output_path=str(tmp_path / "a.png")) + relative = await driver._forward("desktop_screenshot", output_path="rel.png") + echoed = await driver._forward("desktop_find", name=str(named)) + unread = await driver._forward("desktop_screenshot", output_path=str(unreadable)) + monkeypatch.setattr(module, "MAX_IMAGE_BYTES", 2 * len(b"\x89PNG fake")) + shots = [{"command": "screenshot", "args": {"output_path": str(tmp_path / f"b{n}.png")}} for n in range(3)] + batch = await driver._forward("desktop_batch", commands_json=json.dumps( + [*shots, {"command": "trace export", "args": {"out": str(tmp_path / "trace.html")}}, shots[0]])) + + text, image = saved.content + assert json.loads(text.text)["data"]["path"] == str(tmp_path / "a.png") + assert (image.type, image.mimeType, base64.b64decode(image.data)) == ("image", "image/png", b"\x89PNG fake") + assert [block.type for block in relative.content] == ["text", "image"] + assert [block.type for block in echoed.content] == ["text"] + assert [block.type for block in unread.content] == ["text", "text"] + assert unread.content[-1].text == f"Saved but not readable: {unreadable}" + assert [block.type for block in batch.content] == ["text", "image", "image", "text"] + assert batch.content[-1].text == f"Saved but not attached, past the most one answer carries: {tmp_path / 'b2.png'}" + + @pytest.mark.asyncio async def test_a_server_that_fails_before_it_answers_is_an_error_saying_the_command_may_have_run(module, tmp_path): (tmp_path / "fake" / "die").write_text("") diff --git a/tst/unit/drivers/macos_harness_driver_test.py b/tst/unit/drivers/macos_harness_driver_test.py index ca907ae6..081d411e 100644 --- a/tst/unit/drivers/macos_harness_driver_test.py +++ b/tst/unit/drivers/macos_harness_driver_test.py @@ -120,6 +120,7 @@ async def test_its_instructions_send_the_agent_to_its_skill_and_readiness_checks description, body, files = await driver.bundled_skill() assert instructions == module.PREFACE and len(instructions) < 4096 and 'run [["skill"]]' in instructions + assert "the files the harness names stay on this computer" in instructions assert "chain joined with `&&` is a list" in instructions assert [(check.name, check.ok) for check in ready] == [("macos-harness", True), ("macOS permissions", True)] assert not old.ok and old.detail == "version '0.3.0', not the pinned 0.2.1"