From f8957c799d99b7b8224f113cc0725fe470288689 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 11:45:11 -0800 Subject: [PATCH 1/9] feat(browser): browser task steps for cookies, local storage and tabs (FD-4943) A task_steps/browser package: a BrowserStep base that finds the website browser in the deployed env and calls browser/v1 with the control token, and browser_set_cookies, browser_delete_cookies, browser_set_local_storage, browser_clear_local_storage, browser_open_tab, browser_close_tab and browser_activate_tab on it. Preflight refuses an env with no browser, a browser it doesn't have, or none named among several; each step checks its operation on the deployed card, and tab steps pick the one tab at their URL as the browser shows it. tabs/open takes background, opening the tab behind the active one, which stays active. browser_connect_cdp moves into the package on the same base. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 5 +- .../website_browser/control.py | 15 +- .../website_browser/environment.py | 4 +- .../tests/test_website_browser.py | 28 +- src/agent_env/browsers/README.md | 13 + src/agent_env/task_step/registry.py | 19 +- .../task_step/task_steps/browser/__init__.py | 2 + .../task_step/task_steps/browser/base.py | 118 +++++++++ .../connect_cdp.py} | 59 +---- .../task_step/task_steps/browser/cookies.py | 94 +++++++ .../task_steps/browser/local_storage.py | 92 +++++++ .../task_step/task_steps/browser/tabs.py | 131 ++++++++++ .../cli/run_bundle_browser_use_test.py | 2 +- .../computer/browser_computer_test.py | 25 +- tst/integration/computer/browser_steps.py | 73 ++++++ tst/integration/computer/chromium_app_test.py | 22 +- .../task_step/test_browser_connect_cdp.py | 2 +- tst/unit/task_step/test_browser_steps.py | 241 ++++++++++++++++++ 18 files changed, 879 insertions(+), 66 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/browser/__init__.py create mode 100644 src/agent_env/task_step/task_steps/browser/base.py rename src/agent_env/task_step/task_steps/{browser_connect_cdp.py => browser/connect_cdp.py} (69%) create mode 100644 src/agent_env/task_step/task_steps/browser/cookies.py create mode 100644 src/agent_env/task_step/task_steps/browser/local_storage.py create mode 100644 src/agent_env/task_step/task_steps/browser/tabs.py create mode 100644 tst/integration/computer/browser_steps.py create mode 100644 tst/unit/task_step/test_browser_steps.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 0582a36ca..0c569ee12 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -935,8 +935,9 @@ are the profile's template. The base class adds: `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. Also framework-only, for page drivers and agentenv's own steps alike: - `navigate {url}` and `history {direction}` (`back`, `forward` or `reload`), in the active tab; - - `tabs/list`, `tabs/open {url?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's own tab - ids. The active tab is the one a dialog is open on, else the visible one whose page has focus (a hidden + - `tabs/list`, `tabs/open {url?, background?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's + own tab ids. A tab opened with `background` opens behind the active tab, which stays active; otherwise it comes + to the front. The active tab is the one a dialog is open on, else the visible one whose page has focus (a hidden tab's page that had input still says it has focus), else the one agentenv last brought to the front. Headless, a tab opened at a URL opens blank and goes there once it's held to the window's size; - `dialog/answer {accept, text?}`, the agent's answer to the dialog open on the active tab; diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index 8add4f5b9..f3026a390 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -120,17 +120,22 @@ async def tab_list(self) -> list[dict]: active = await self.active_tab() return [OpenTab(**tab, active=tab["id"] == active).model_dump() for tab in self.tabs.values()] - async def open_tab(self, url: Optional[str] = None) -> str: - """A new tab at ``url``, else blank, brought to the front; its id. Headless, it opens blank and goes to ``url`` - once it's held to the window's size, so its page loads at that size.""" + async def open_tab(self, url: Optional[str] = None, background: bool = False) -> str: + """A new tab at ``url``, else blank, brought to the front, or with ``background`` behind the active tab, which + stays active; its id. Headless, it opens blank and goes to ``url`` once it's held to the window's size, so its + page loads at that size.""" + active = await self.active_tab() if background else None held = self.viewport is not None and url start = "about:blank" if held else url or "about:blank" - target = (await self.devtools.call("Target.createTarget", {"url": start}))["targetId"] + target = (await self.devtools.call("Target.createTarget", {"url": start, "background": background}))["targetId"] if held: await asyncio.wait_for(self._pinned.setdefault(target, asyncio.Event()).wait(), PIN_TIMEOUT_S) session = next(s for s, t in self._sessions.items() if t == target) await self.devtools.call("Page.navigate", {"url": url}, session_id=session) - await self._bring_to_front(target) + if not background: + await self._bring_to_front(target) + elif active is not None and active in self.tabs and await self.active_tab() != active: + await self._bring_to_front(active) return target async def reopen_start_tabs(self, urls: list[str]) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index 10f87dd17..51ad3109e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -376,8 +376,8 @@ async def browser_tabs_list(self) -> dict: return {"tabs": await self.control.tab_list()} @extension(BROWSER, op="tabs/open") - async def browser_tabs_open(self, url: Optional[str] = None) -> dict: - return {"tab": await self.control.open_tab(url)} + async def browser_tabs_open(self, url: Optional[str] = None, background: bool = False) -> dict: + return {"tab": await self.control.open_tab(url, background)} @extension(BROWSER, op="tabs/activate") async def browser_tabs_activate(self, tab: str) -> dict: diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 07f906d93..413073428 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -654,7 +654,8 @@ async def test_a_tab_it_opens_loads_once_held_to_the_window_comes_to_the_front_a devtools.send("Target.targetDestroyed", {"targetId": "b"}) on_b = [name for name, _, session in devtools.calls if session == "s-b"] - assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank"}, None)] + assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank", "background": False}, + None)] assert ({"url": "https://mail.test/b"}, "s-b") in _sent(devtools, "Page.navigate") assert on_b.index("Emulation.setDeviceMetricsOverride") < on_b.index("Page.navigate") assert [(tab["id"], tab["active"]) for tab in after_open] == [("a", False), ("b", True)] @@ -663,6 +664,31 @@ async def test_a_tab_it_opens_loads_once_held_to_the_window_comes_to_the_front_a assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs] == [("b", "agentenv")] +@pytest.mark.asyncio +@pytest.mark.parametrize("focus_moves", [False, True]) +async def test_a_tab_opened_in_the_background_opens_behind_the_active_one_which_stays_in_front(focus_moves): + focus = {"s-a": True} + devtools = Tabbed({"Runtime.evaluate": lambda params, session: { + "result": {"value": ["visible", focus.get(session, False)]}}}) + control = await _tabs(devtools, "a") + + opening = asyncio.ensure_future(control.open_tab("https://mail.test/b", background=True)) + await asyncio.sleep(0) + devtools.send("Target.targetCreated", _page("b", "https://mail.test/b", "Mail b")) + devtools.send("Target.attachedToTarget", {"sessionId": "s-b", **_page("b")}) + if focus_moves: + focus.update({"s-a": False, "s-b": True}) + opened = await opening + + assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank", "background": True}, + None)] + assert ({"url": "https://mail.test/b"}, "s-b") in _sent(devtools, "Page.navigate") + assert ({}, "s-b") not in _sent(devtools, "Page.bringToFront") + assert _sent(devtools, "Target.activateTarget") == ([({"targetId": "a"}, None)] if focus_moves else []) + if not focus_moves: + assert [(tab["id"], tab["active"]) for tab in await control.tab_list()] == [("a", True), ("b", False)] + + @pytest.mark.asyncio async def test_a_hidden_tab_whose_page_still_says_it_has_focus_isnt_the_active_one(): states = {"s-a": ["hidden", True], "s-b": ["visible", True]} diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 3e81a3cdb..8631cb011 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -15,3 +15,16 @@ under `[settings]`: `start_urls`, opened at boot, the first tab active; `window` them (`env_params.app_settings = {chromium = {start_urls = [...]}}`). Its data is its profile, `~/.config/chromium`, which the image has none of, so each deploy and each reset starts with a clean one. Name it from another bundle as `chromium`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/browsers/chromium`. + +A task changes a browser's state with the steps in `agent_env.task_step.task_steps.browser`, each a call to one +`browser/v1` operation on the browser in the env `env_id`, the one `browser` names when the env has several: +`browser_set_cookies {cookies}`, in the shape `cookies/set` takes; `browser_delete_cookies {name?, domain?, path?}`, +every cookie matching all of those given, a domain as the cookie stores it (`.shop.test` for a domain cookie), every +cookie with none, the count deleted recorded; `browser_set_local_storage {origin, items}`, `items` a map of keys to +values, the origin's other keys left as they are; `browser_clear_local_storage {origin?}`, every origin seen with none; +`browser_open_tab {url, background?}`, in front, or with `background` behind the active tab, which stays active; and +`browser_close_tab {url}` and `browser_activate_tab {url}`, on the one open tab at that URL as the browser shows it +(scheme and host lowercase, no default port, `/` for an empty path), none or several failing the step with the tabs +open listed. Preflight refuses a step on an env with no browser, none of the name given, or several and none named. +`browser_connect_cdp` hands an agent a CDP address to the same browser. A whole state, Playwright's `storageState` +included, loads through `load_artifact` or `data/add` instead. diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 0bdaf23b3..1b9a3f298 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -27,7 +27,17 @@ def get_task_step_registry() -> dict[str, type["TaskStep"]]: def _builtin_registry() -> dict[str, type["TaskStep"]]: - from agent_env.task_step.task_steps.browser_connect_cdp import BrowserConnectCdpTaskStep + from agent_env.task_step.task_steps.browser.connect_cdp import BrowserConnectCdpTaskStep + from agent_env.task_step.task_steps.browser.cookies import BrowserDeleteCookiesTaskStep, BrowserSetCookiesTaskStep + from agent_env.task_step.task_steps.browser.local_storage import ( + BrowserClearLocalStorageTaskStep, + BrowserSetLocalStorageTaskStep, + ) + from agent_env.task_step.task_steps.browser.tabs import ( + BrowserActivateTabTaskStep, + BrowserCloseTabTaskStep, + BrowserOpenTabTaskStep, + ) from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep from agent_env.task_step.task_steps.computer_recording import ( EndComputerRecordingTaskStep, @@ -89,6 +99,13 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(ReviewTaskStep): ReviewTaskStep, _get_type(BuildMcpCliTaskStep): BuildMcpCliTaskStep, _get_type(BrowserConnectCdpTaskStep): BrowserConnectCdpTaskStep, + _get_type(BrowserSetCookiesTaskStep): BrowserSetCookiesTaskStep, + _get_type(BrowserDeleteCookiesTaskStep): BrowserDeleteCookiesTaskStep, + _get_type(BrowserSetLocalStorageTaskStep): BrowserSetLocalStorageTaskStep, + _get_type(BrowserClearLocalStorageTaskStep): BrowserClearLocalStorageTaskStep, + _get_type(BrowserOpenTabTaskStep): BrowserOpenTabTaskStep, + _get_type(BrowserCloseTabTaskStep): BrowserCloseTabTaskStep, + _get_type(BrowserActivateTabTaskStep): BrowserActivateTabTaskStep, _get_type(CollectArtifactsTaskStep): CollectArtifactsTaskStep, _get_type(StartComputerRecordingTaskStep): StartComputerRecordingTaskStep, _get_type(EndComputerRecordingTaskStep): EndComputerRecordingTaskStep, diff --git a/src/agent_env/task_step/task_steps/browser/__init__.py b/src/agent_env/task_step/task_steps/browser/__init__.py new file mode 100644 index 000000000..85b5d7646 --- /dev/null +++ b/src/agent_env/task_step/task_steps/browser/__init__.py @@ -0,0 +1,2 @@ +"""Task steps on a deployed env's website browser, through browser/v1: its cookies, local storage and tabs set or changed, +and a CDP address to it handed to an agent.""" diff --git a/src/agent_env/task_step/task_steps/browser/base.py b/src/agent_env/task_step/task_steps/browser/base.py new file mode 100644 index 000000000..b292c3f5f --- /dev/null +++ b/src/agent_env/task_step/task_steps/browser/base.py @@ -0,0 +1,118 @@ +"""BrowserStep: a task step on the website browser in a deployed env, found among the env's children by its card, and +called through browser/v1 with the env's control token.""" + +from __future__ import annotations + +from typing import Any, Optional + +from agentenv_protocol.client import find_extension, find_extension_method, invoke_extension +from agentenv_protocol.website_browser import BROWSER + +from agent_env.env.env import DeployedEnv, Env +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.store.base import NotFoundError +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency + +#: How long a browser/v1 operation may take to answer. +CALL_TIMEOUT_S = 60 +#: Where the steps record what each found or did, by step id. +RECORDS_KEY = "browser_steps" + + +class BrowserStep(TaskStep): + """A step on the website browser in the env ``env_id``: the one ``browser`` names, else the env's only one. + Preflight refuses an env that has no browser, or none of that name, or several when it names none.""" + + def __init__(self, id: str, version: Optional[int], env_id: str, browser: Optional[str] = None, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.browser = browser + + def config_problems(self) -> list[str]: + problems = [] + if not isinstance(self.env_id, str) or not self.env_id: + problems.append(f"step {self.id!r}: env_id must be a non-empty string") + if self.browser is not None and (not isinstance(self.browser, str) or not self.browser): + problems.append(f"step {self.id!r}: browser, when given, must be a non-empty string") + return problems + + def preflight(self) -> list[str]: + problems = super().preflight() + if not isinstance(self.env_id, str) or not self.env_id: + return problems + try: + env = Env.get(self.env_id) + except NotFoundError: # what the run reports + return problems + names = stored_browsers(env) + if not names: + problems.append(f"step {self.id!r}: env {self.env_id!r} has no website browser: browser steps act on a " + "computer env's browser app, or a browser deployed alone") + elif self.browser is not None and self.browser not in names: + problems.append(f"step {self.id!r}: env {self.env_id!r} has no website browser named {self.browser!r} " + f"(it has {names})") + elif self.browser is None and len(names) > 1: + problems.append(f"step {self.id!r}: env {self.env_id!r} has {len(names)} website browsers, {names}: " + "name one as browser") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "env_id": self.env_id, "browser": self.browser} + + @classmethod + def _browser_from_dict(cls, data: dict) -> dict: + return {**cls._base_from_dict(data), "env_id": data["env_id"], "browser": data.get("browser")} + + def _find(self, context: TaskStepContext) -> tuple[DeployedEnv, Optional[str], dict]: + """The deployed env, and its browser's name, None for the env itself, and card.""" + deployed = deployed_env(context, self.env_id) + return (deployed, *find_browser(deployed, self.browser)) + + async def _call(self, context: TaskStepContext, op: str, params: dict, + timeout_s: float = CALL_TIMEOUT_S) -> Any: + """``op``'s answer from the browser, refused when its card serves no such operation.""" + deployed, name, card = self._find(context) + if find_extension_method(card, BROWSER, op) is None: + raise RuntimeError(f"the website browser {name or deployed.env_id!r} in env {deployed.env_id!r} serves no " + f"browser/v1 {op}: rebuild it on a newer agentenv-framework-protocol") + return await invoke_extension(deployed.environment_url, card, BROWSER, params, timeout_s, method=op, + headers=deployed.control_headers) + + def _record(self, context: TaskStepContext, **fields: Any) -> None: + context.metadata.setdefault(RECORDS_KEY, {})[self.id] = {"env_id": self.env_id, "browser": self.browser, + **fields} + + +def stored_browsers(env: Env) -> list[str]: + """The names of the website browsers a stored env deploys: a computer env's browser apps, the app itself for one + deployed alone; none for any other env.""" + if not isinstance(env, ComputerEnv): + return [] + return [app.environment_name for app in env.composed_apps() if app.declaration.browser] + + +def deployed_env(context: TaskStepContext, env_id: str) -> DeployedEnv: + deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) + if deployed is None: + raise RuntimeError(f"env {env_id!r} isn't in context.deployed_envs") + return deployed + + +def find_browser(deployed: DeployedEnv, named: Optional[str]) -> tuple[Optional[str], dict]: + """The website browser in ``deployed``, the one ``named`` else its only one: its name among the env's children, + or None for the env itself, and its card.""" + card = deployed.environment_card or {} + browsers = [(None, card)] if find_extension(card, BROWSER) else [] + browsers += [(child.get("name"), child) for child in card.get("children_environments") or [] + if find_extension(child, BROWSER)] + if named is not None: + found = next((browser for browser in browsers if browser[0] == named), None) + if found is None: + raise RuntimeError(f"env {deployed.env_id!r} has no website browser named {named!r}") + return found + if len(browsers) != 1: + raise RuntimeError(f"env {deployed.env_id!r} has {len(browsers) or 'no'} website browsers" + + (": name one as browser" if browsers else "")) + return browsers[0] diff --git a/src/agent_env/task_step/task_steps/browser_connect_cdp.py b/src/agent_env/task_step/task_steps/browser/connect_cdp.py similarity index 69% rename from src/agent_env/task_step/task_steps/browser_connect_cdp.py rename to src/agent_env/task_step/task_steps/browser/connect_cdp.py index 86b73ddc5..423323053 100644 --- a/src/agent_env/task_step/task_steps/browser_connect_cdp.py +++ b/src/agent_env/task_step/task_steps/browser/connect_cdp.py @@ -8,10 +8,8 @@ from typing import ClassVar, Optional import httpx -from agentenv_protocol.client import find_extension_method, invoke_extension from agentenv_protocol.tool_access import DISABLE, DISABLE_TOOL_EXTENSION_URI, WILDCARD from agentenv_protocol.tool_context import DEFAULT_ROLE -from agentenv_protocol.website_browser import BROWSER from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.entity.refs import EntityRef @@ -20,6 +18,7 @@ from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.browser.base import BrowserStep logger = logging.getLogger(__name__) @@ -33,7 +32,7 @@ ADD_TIMEOUT_S = 60 -class BrowserConnectCdpTaskStep(TaskStep): +class BrowserConnectCdpTaskStep(BrowserStep): """Hand the agent ``agent_name`` a CDP address to the website browser ``browser`` in the env ``env_id``, the env's only one when unnamed: added to its cdp-config/v1 as ``name``, with every tool the env serves hidden from its role unless ``keep_tools``. The address is the browser's own for that agent, as its sandbox reaches it.""" @@ -53,17 +52,13 @@ def __init__( depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) - self.env_id = env_id + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) self.agent_name = agent_name or TaskStep.DEFAULT_AGENT_NAME - self.browser = browser self.name = name self.keep_tools = keep_tools def config_problems(self) -> list[str]: - problems = [] - if not isinstance(self.env_id, str) or not self.env_id: - problems.append(f"step {self.id!r}: env_id must be a non-empty string") + problems = super().config_problems() if not isinstance(self.name, str) or not self.name: problems.append(f"step {self.id!r}: name must be a non-empty string") if not isinstance(self.keep_tools, bool): @@ -71,32 +66,21 @@ def config_problems(self) -> list[str]: return problems def to_dict(self) -> dict: - base = super().to_dict() - base.update(env_id=self.env_id, agent_name=self.agent_name, browser=self.browser, name=self.name, - keep_tools=self.keep_tools) - return base + return {**super().to_dict(), "agent_name": self.agent_name, "name": self.name, "keep_tools": self.keep_tools} @classmethod def from_dict(cls, data: dict) -> BrowserConnectCdpTaskStep: - return cls( - **cls._base_from_dict(data), - env_id=data["env_id"], - agent_name=data.get("agent_name"), - browser=data.get("browser"), - name=data.get("name") or DEFAULT_NAME, - keep_tools=data.get("keep_tools", False), - ) + return cls(**cls._browser_from_dict(data), agent_name=data.get("agent_name"), + name=data.get("name") or DEFAULT_NAME, keep_tools=data.get("keep_tools", False)) async def execute(self, context: TaskStepContext) -> TaskStepContext: - deployed = _deployed_env(context, self.env_id) + deployed, browser, _ = self._find(context) agent = _deployed_agent(context, self.agent_name) cdp_config = A2AAgent.find_extension(agent.a2a_card or {}, A2AAgent.EXT_CDP_CONFIG) if cdp_config is None or not agent.a2a_url: raise RuntimeError(f"agent {agent.agent_name!r} can't take a CDP address: its card has no " f"{A2AAgent.EXT_CDP_CONFIG}") - browser, card = _browser(deployed, self.browser) - address = await invoke_extension(deployed.environment_url, card, BROWSER, {"user": agent.agent_name}, - CDP_TIMEOUT_S, method="cdp", headers=deployed.control_headers) + address = await self._call(context, "cdp", {"user": agent.agent_name}, CDP_TIMEOUT_S) http_url, ws_url = _as_agent_reaches(address, deployed, agent) async with httpx.AsyncClient(timeout=ADD_TIMEOUT_S) as client: added = await client.post(agent.a2a_url + cdp_config["params"]["endpoint"], json={ @@ -118,13 +102,6 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context -def _deployed_env(context: TaskStepContext, env_id: str) -> DeployedEnv: - deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) - if deployed is None: - raise RuntimeError(f"env {env_id!r} isn't in context.deployed_envs") - return deployed - - def _deployed_agent(context: TaskStepContext, agent_name: str) -> DeployedAgent: agent = next((a for a in context.deployed_agents if a.agent_name == agent_name), None) if agent is None: @@ -132,24 +109,6 @@ def _deployed_agent(context: TaskStepContext, agent_name: str) -> DeployedAgent: return agent -def _browser(deployed: DeployedEnv, named: Optional[str]) -> tuple[Optional[str], dict]: - """The website browser to connect to, the one named else the env's only one: its name among the env's children, - or None for the env itself, and its card.""" - card = deployed.environment_card or {} - browsers = [(None, card)] if find_extension_method(card, BROWSER, "cdp") else [] - browsers += [(child.get("name"), child) for child in card.get("children_environments") or [] - if find_extension_method(child, BROWSER, "cdp")] - if named is not None: - found = next((browser for browser in browsers if browser[0] == named), None) - if found is None: - raise RuntimeError(f"env {deployed.env_id!r} has no website browser named {named!r}") - return found - if len(browsers) != 1: - raise RuntimeError(f"env {deployed.env_id!r} has {len(browsers) or 'no'} website browsers" - + (": name one as browser" if browsers else "")) - return browsers[0] - - def _as_agent_reaches(address: dict, deployed: DeployedEnv, agent: DeployedAgent) -> tuple[str, str]: """``address``'s URLs as the agent's sandbox reaches them: under the env's URL as the agent reaches that, as deploy_agent gives it MCP URLs. A URL anywhere else, at a gateway's public address say, is kept.""" diff --git a/src/agent_env/task_step/task_steps/browser/cookies.py b/src/agent_env/task_step/task_steps/browser/cookies.py new file mode 100644 index 000000000..93e440d80 --- /dev/null +++ b/src/agent_env/task_step/task_steps/browser/cookies.py @@ -0,0 +1,94 @@ +"""browser_set_cookies and browser_delete_cookies: cookies set in, or deleted from, an env's website browser.""" + +from __future__ import annotations + +import logging +from typing import ClassVar, Optional + +from agentenv_protocol.website_browser import CookieToSet +from pydantic import ValidationError + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStepDependency +from agent_env.task_step.task_steps.browser.base import BrowserStep + +logger = logging.getLogger(__name__) + + +class BrowserSetCookiesTaskStep(BrowserStep): + """Set ``cookies`` in the browser, all or none, each in the shape cookies/set takes: a ``url``, or a ``domain`` and + ``path``, and ``max_age`` in place of ``expires`` for a lifetime from when it's set.""" + + type: ClassVar[str] = "browser_set_cookies" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, cookies: list[dict], + browser: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True): + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.cookies = cookies + + def config_problems(self) -> list[str]: + problems = super().config_problems() + if not isinstance(self.cookies, list) or not self.cookies: + return [*problems, f"step {self.id!r}: cookies must be a non-empty list"] + for index, cookie in enumerate(self.cookies): + try: + CookieToSet.model_validate(cookie) + except ValidationError as e: + problems.append(f"step {self.id!r}: cookies[{index}]: " + "; ".join( + f"{'.'.join(map(str, error['loc'])) or 'it'}: {error['msg']}" + for error in e.errors(include_input=False, include_url=False))) + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "cookies": self.cookies} + + @classmethod + def from_dict(cls, data: dict) -> BrowserSetCookiesTaskStep: + return cls(**cls._browser_from_dict(data), cookies=data["cookies"]) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + await self._call(context, "cookies/set", {"cookies": self.cookies}) + logger.info(f"{self.id}: set {len(self.cookies)} cookie(s) in the browser of env {self.env_id!r}") + return context + + +class BrowserDeleteCookiesTaskStep(BrowserStep): + """Delete every cookie in the browser matching all of ``name``, ``domain`` (exactly as the cookie stores it, a + domain cookie's with its leading dot) and ``path`` given, or every cookie when none is; deleting none is fine, + and the count is recorded.""" + + type: ClassVar[str] = "browser_delete_cookies" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, name: Optional[str] = None, + domain: Optional[str] = None, path: Optional[str] = None, browser: Optional[str] = None, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.name, self.domain, self.path = name, domain, path + + def config_problems(self) -> list[str]: + problems = super().config_problems() + for field in ("name", "domain", "path"): + value = getattr(self, field) + if value is not None and (not isinstance(value, str) or not value): + problems.append(f"step {self.id!r}: {field}, when given, must be a non-empty string") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "name": self.name, "domain": self.domain, "path": self.path} + + @classmethod + def from_dict(cls, data: dict) -> BrowserDeleteCookiesTaskStep: + return cls(**cls._browser_from_dict(data), name=data.get("name"), domain=data.get("domain"), + path=data.get("path")) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + matching = {field: value for field in ("name", "domain", "path") if (value := getattr(self, field)) is not None} + deleted = (await self._call(context, "cookies/delete", matching))["deleted"] + logger.info(f"{self.id}: deleted {deleted} cookie(s) matching {matching or 'anything'} in the browser of env " + f"{self.env_id!r}") + self._record(context, deleted=deleted) + return context diff --git a/src/agent_env/task_step/task_steps/browser/local_storage.py b/src/agent_env/task_step/task_steps/browser/local_storage.py new file mode 100644 index 000000000..5c5462cd7 --- /dev/null +++ b/src/agent_env/task_step/task_steps/browser/local_storage.py @@ -0,0 +1,92 @@ +"""browser_set_local_storage and browser_clear_local_storage: an origin's local storage set or cleared in an env's +website browser.""" + +from __future__ import annotations + +import logging +from typing import ClassVar, Optional + +from agentenv_protocol.website_browser import origin_of + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStepDependency +from agent_env.task_step.task_steps.browser.base import BrowserStep + +logger = logging.getLogger(__name__) + + +class BrowserSetLocalStorageTaskStep(BrowserStep): + """Set each of ``items``, a map of keys to values, in ``origin``'s local storage, leaving the origin's other keys + as they are.""" + + type: ClassVar[str] = "browser_set_local_storage" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, origin: str, items: dict[str, str], + browser: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True): + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.origin, self.items = origin, items + + def config_problems(self) -> list[str]: + problems = super().config_problems() + _origin_problems(self.id, self.origin, required=True) + if (not isinstance(self.items, dict) or not self.items + or not all(isinstance(key, str) and isinstance(value, str) for key, value in self.items.items())): + problems.append(f"step {self.id!r}: items must be a non-empty map of keys to string values") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "origin": self.origin, "items": self.items} + + @classmethod + def from_dict(cls, data: dict) -> BrowserSetLocalStorageTaskStep: + return cls(**cls._browser_from_dict(data), origin=data["origin"], items=data["items"]) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + await self._call(context, "local_storage/set", {"origins": [{ + "origin": self.origin, "localStorage": [{"name": key, "value": value} for key, value in self.items.items()]}]}) + logger.info(f"{self.id}: set {sorted(self.items)} in {self.origin}'s local storage in the browser of env " + f"{self.env_id!r}") + return context + + +class BrowserClearLocalStorageTaskStep(BrowserStep): + """Clear ``origin``'s local storage in the browser, or every origin's it has seen when it names none.""" + + type: ClassVar[str] = "browser_clear_local_storage" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, origin: Optional[str] = None, + browser: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True): + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.origin = origin + + def config_problems(self) -> list[str]: + return super().config_problems() + _origin_problems(self.id, self.origin, required=False) + + def to_dict(self) -> dict: + return {**super().to_dict(), "origin": self.origin} + + @classmethod + def from_dict(cls, data: dict) -> BrowserClearLocalStorageTaskStep: + return cls(**cls._browser_from_dict(data), origin=data.get("origin")) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + cleared = (await self._call(context, "local_storage/clear", + {} if self.origin is None else {"origins": [self.origin]}))["origins"] + logger.info(f"{self.id}: cleared the local storage of {cleared} in the browser of env {self.env_id!r}") + return context + + +def _origin_problems(step_id: str, origin: object, *, required: bool) -> list[str]: + if origin is None and not required: + return [] + if not isinstance(origin, str): + return [f"step {step_id!r}: origin must be an http or https origin such as https://shop.test"] + try: + origin_of(origin, strict=True) + except ValueError as e: + return [f"step {step_id!r}: origin: {e}"] + return [] diff --git a/src/agent_env/task_step/task_steps/browser/tabs.py b/src/agent_env/task_step/task_steps/browser/tabs.py new file mode 100644 index 000000000..60d868830 --- /dev/null +++ b/src/agent_env/task_step/task_steps/browser/tabs.py @@ -0,0 +1,131 @@ +"""browser_open_tab, browser_close_tab and browser_activate_tab: a tab opened in an env's website browser, or one open +there, picked by its URL, closed or brought to the front.""" + +from __future__ import annotations + +import logging +from typing import ClassVar, Optional +from urllib.parse import urlsplit, urlunsplit + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStepDependency +from agent_env.task_step.task_steps.browser.base import BrowserStep + +logger = logging.getLogger(__name__) + +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +class BrowserOpenTabTaskStep(BrowserStep): + """Open a tab at ``url``, in front, or with ``background`` behind the active tab, which stays active; the new tab's + id is recorded.""" + + type: ClassVar[str] = "browser_open_tab" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, url: str, background: bool = False, + browser: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True): + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.url, self.background = url, background + + def config_problems(self) -> list[str]: + problems = super().config_problems() + _url_problems(self.id, self.url) + if not isinstance(self.background, bool): + problems.append(f"step {self.id!r}: background must be true or false") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "url": self.url, "background": self.background} + + @classmethod + def from_dict(cls, data: dict) -> BrowserOpenTabTaskStep: + return cls(**cls._browser_from_dict(data), url=data["url"], background=data.get("background", False)) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + tab = (await self._call(context, "tabs/open", {"url": self.url, "background": self.background}))["tab"] + logger.info(f"{self.id}: opened tab {tab} at {self.url}{' in the background' if self.background else ''} in " + f"the browser of env {self.env_id!r}") + self._record(context, tab=tab) + return context + + +class _TabAtUrlStep(BrowserStep): + """A step on the open tab whose URL is ``url``, as the browser shows it now: the same after both are normalized as + the browser normalizes a URL. No tab, or several, at that URL fails the step, naming the tabs open.""" + + #: The browser/v1 operation called on the tab, and what the step says it did. + op: ClassVar[str] + did: ClassVar[str] + + def __init__(self, id: str, version: Optional[int], env_id: str, url: str, browser: Optional[str] = None, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.url = url + + def config_problems(self) -> list[str]: + return super().config_problems() + _url_problems(self.id, self.url) + + def to_dict(self) -> dict: + return {**super().to_dict(), "url": self.url} + + @classmethod + def from_dict(cls, data: dict) -> _TabAtUrlStep: + return cls(**cls._browser_from_dict(data), url=data["url"]) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + tabs = (await self._call(context, "tabs/list", {}))["tabs"] + wanted = normalized(self.url) + matching = [tab for tab in tabs if normalized(tab["url"]) == wanted] + if len(matching) != 1: + raise RuntimeError(f"{self.type} {self.id!r}: {len(matching) or 'no'} tabs are at {self.url} in the browser " + f"of env {self.env_id!r}; the tabs open are at {[tab['url'] for tab in tabs]}") + await self._call(context, self.op, {"tab": matching[0]["id"]}) + logger.info(f"{self.id}: {self.did} tab {matching[0]['id']} at {self.url} in the browser of env {self.env_id!r}") + self._record(context, tab=matching[0]["id"]) + return context + + +class BrowserCloseTabTaskStep(_TabAtUrlStep): + """Close the open tab at ``url``.""" + + type: ClassVar[str] = "browser_close_tab" + entity_refs = (EntityRef.env("env_id"),) + op, did = "tabs/close", "closed" + + +class BrowserActivateTabTaskStep(_TabAtUrlStep): + """Bring the open tab at ``url`` to the front, the active tab.""" + + type: ClassVar[str] = "browser_activate_tab" + entity_refs = (EntityRef.env("env_id"),) + op, did = "tabs/activate", "activated" + + +def normalized(url: str) -> str: + """``url`` as the browser shows it: its scheme and host lowercase, its default port dropped, and an empty path as + ``/``; its query and fragment as they are. A URL with no host, such as ``about:blank``, is left as it is.""" + parts = urlsplit(url) + if not parts.netloc: + return url + scheme, host = parts.scheme.lower(), (parts.hostname or "") + host = f"[{host}]" if ":" in host else host + port = parts.port + netloc = host if port is None or port == _DEFAULT_PORTS.get(scheme) else f"{host}:{port}" + if parts.username is not None: + netloc = f"{parts.username}{':' + parts.password if parts.password is not None else ''}@{netloc}" + return urlunsplit((scheme, netloc, parts.path or "/", parts.query, parts.fragment)) + + +def _url_problems(step_id: str, url: object) -> list[str]: + if not isinstance(url, str) or not url: + return [f"step {step_id!r}: url must be a non-empty string"] + try: + parts = urlsplit(url) + _ = parts.port + except ValueError as e: + return [f"step {step_id!r}: url {url!r} isn't a URL: {e}"] + if not parts.scheme or (parts.scheme.lower() in _DEFAULT_PORTS and not parts.netloc): + return [f"step {step_id!r}: url {url!r} isn't absolute: give its scheme and host, such as https://shop.test/"] + return [] diff --git a/tst/integration/cli/run_bundle_browser_use_test.py b/tst/integration/cli/run_bundle_browser_use_test.py index 82d071941..e28c56c1d 100644 --- a/tst/integration/cli/run_bundle_browser_use_test.py +++ b/tst/integration/cli/run_bundle_browser_use_test.py @@ -24,7 +24,7 @@ from agent_env.store.routing import namespace_routing from agent_env.task.teardown import teardown_run from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_steps.browser_connect_cdp import BrowserConnectCdpTaskStep +from agent_env.task_step.task_steps.browser.connect_cdp import BrowserConnectCdpTaskStep from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep from tst.util.fake_openai import FakeOpenAI, Text, ToolCall diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 57de7fa04..0a38a470f 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -12,7 +12,8 @@ fresh one its next turn opens the site through. Through an app server's restart, which relaunches the browser, and a driver's, the agent's address keeps working, and two agents connected at once each drive the browser through their own. browser/v1 reads and sets its cookies, local storage and downloads, and its data plane signs it in, snapshots it, resets it, loads a -snapshot back and holds up under chaos. Needs Docker and the local registry on +snapshot back and holds up under chaos; and the browser task steps set its cookies and storage and open, activate and +close its tabs by URL. Needs Docker and the local registry on localhost:5000, and Node 20 or later and npm for the CDP clients.""" import asyncio @@ -43,11 +44,12 @@ from agent_env.store.routing import namespace_routing from agent_env.task.teardown import teardown_run from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_steps.browser_connect_cdp import BrowserConnectCdpTaskStep +from agent_env.task_step.task_steps.browser.connect_cdp import BrowserConnectCdpTaskStep from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data, check_browser_data_under_chaos +from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state, check_browser_state_under_chaos from tst.util.a2a_test_agent import put_test_agent @@ -347,6 +349,25 @@ async def act(): assert not report.failed +@pytest.mark.asyncio +async def test_its_task_steps_set_cookies_and_storage_open_a_tab_behind_and_activate_and_close_it_by_its_url( + browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [DATA_SITE]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + await _ready(deployed) + card = (await _get(deployed, "/.well-known/agent-env.json")).json() + app = next(child for child in card["children_environments"] if child["name"] == "chromium") + await check_browser_steps(deployed, browser_computer.id, lambda op: deployed.environment_url + + protocol_v1.find_extension_method(app, BROWSER, op)["endpoint"]) + finally: + report = await teardown_run(context) + + assert not report.failed + + @pytest.mark.asyncio async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_gone_they_fail_at_once( browser_computer): diff --git a/tst/integration/computer/browser_steps.py b/tst/integration/computer/browser_steps.py new file mode 100644 index 000000000..ddb903c0c --- /dev/null +++ b/tst/integration/computer/browser_steps.py @@ -0,0 +1,73 @@ +"""The browser task steps on a deployed chromium, inside a computer or alone, each made from a task's JSON as the step +registry reads it: each passes preflight on its env; a cookie and local storage set by steps sign in a tab opened in +the background, behind the active tab, which stays active; steps bring that tab to the front and close it by its URL, +delete the cookie and clear the storage; and a tab step naming a URL no tab shows fails, listing the tabs open.""" + +import asyncio +import time +from typing import Callable + +from agent_env.env.envs.computer.utils import inside +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_step import TaskStep +from tst.integration.computer.browser_data import ORIGIN, POLL_S, SERVE_S, SERVE_SITE, SITE, WAIT_S, _client + +SIGNED_IN = "signed in as ada theme dark" + + +async def check_browser_steps(deployed, env_id: str, op_url: Callable[[str], str]) -> None: + """The steps' effects, as the module says, read back through browser/v1 at ``op_url(op)``.""" + call, titled = _client(deployed, op_url) + context = TaskStepContext(deployed_envs=[deployed]) + + def step(type: str, **fields) -> TaskStep: + made = get_task_step_registry()[type].from_dict({"id": type, "version": None, "type": type, "env_id": env_id, + **fields}) + assert made.config_problems() + made.preflight() == [], type + return made + + async def tab_titled(tab: str, title: str) -> dict: + last, deadline = None, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + last = next((t for t in (await call("tabs/list", method="GET"))["tabs"] if t["id"] == tab), None) + if last is not None and last["title"] == title: + return last + await asyncio.sleep(POLL_S) + raise AssertionError(f"tab {tab} wasn't titled {title!r} within {WAIT_S:g} s: {last!r}") + + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_SITE, str(SERVE_S)], + timeout_s=SERVE_S + 60)) + try: + await asyncio.sleep(3) + await call("navigate", {"url": SITE}) + start = await titled("login") + context = await step("browser_set_cookies", cookies=[ + {"name": "sid", "value": "ada", "url": SITE, "max_age": 3600}]).execute(context) + context = await step("browser_set_local_storage", origin=ORIGIN, items={"theme": "dark"}).execute(context) + context = await step("browser_open_tab", url=SITE, background=True).execute(context) + opened = context.metadata["browser_steps"]["browser_open_tab"]["tab"] + behind = await tab_titled(opened, SIGNED_IN) + after_open = [(tab["id"], tab["active"]) for tab in (await call("tabs/list", method="GET"))["tabs"]] + context = await step("browser_activate_tab", url=SITE).execute(context) + in_front = await titled(SIGNED_IN) + context = await step("browser_close_tab", url=SITE).execute(context) + after_close = [tab["url"] for tab in (await call("tabs/list", method="GET"))["tabs"]] + try: + await step("browser_close_tab", url=f"{SITE}nowhere").execute(context) + missing = None + except RuntimeError as e: + missing = str(e) + context = await step("browser_delete_cookies", name="sid").execute(context) + context = await step("browser_clear_local_storage", origin=ORIGIN).execute(context) + cookies = (await call("cookies/get", {"urls": [SITE]}))["cookies"] + storage = (await call("local_storage/get", {"origins": [ORIGIN]}))["origins"] + finally: + serving.cancel() + + assert start["url"] == f"{SITE}login" and behind["url"] == SITE + assert after_open == [(start["id"], True), (opened, False)], after_open + assert in_front["id"] == opened and after_close == [f"{SITE}login"], after_close + assert missing is not None and f"no tabs are at {SITE}nowhere" in missing and f"'{SITE}login'" in missing, missing + assert context.metadata["browser_steps"]["browser_delete_cookies"]["deleted"] == 1 + assert cookies == [] and storage == [{"origin": ORIGIN, "localStorage": []}], (cookies, storage) diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 2ba341f8b..fb964624f 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -9,7 +9,8 @@ its control connection keeps lists a tab an agent opened and closed, a download with its size and sha256, a confirm() accepted for it, and each launch, exit and stop with its cause, across a kill and a restart of its server; and browser/v1 reads and sets its cookies, local storage and downloads; and its data plane signs it in, snapshots it, resets it and loads a -snapshot back. Needs Docker +snapshot back; and the browser task steps set its cookies and storage and open, activate and close its tabs by URL. +Needs Docker and the local registry on localhost:5000.""" import asyncio @@ -29,6 +30,7 @@ from agentenv_protocol.types import WELL_KNOWN_PATH from agentenv_protocol.website_browser import BROWSER from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data +from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state from tst.integration.computer.website_browser_test import _app, _browser_process, _keys, _ready from agent_env.env.envs.computer.utils import inside @@ -350,6 +352,24 @@ async def act(): assert not report.failed +@pytest.mark.asyncio +async def test_its_task_steps_set_cookies_and_storage_open_a_tab_behind_and_activate_and_close_it_by_its_url( + chromium_desktop): + with namespace_routing(): + deployed = await Env.get(chromium_desktop.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [DATA_SITE]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + card = await _app(deployed, WELL_KNOWN_PATH, "chromium") + await _ready(deployed, card, "chromium") + await check_browser_steps(deployed, chromium_desktop.id, lambda op: f"{deployed.environment_url}/svc/chromium" + f"{protocol_v1.find_extension_method(card, BROWSER, op)['endpoint']}") + finally: + report = await teardown_run(context) + + assert not report.failed + + def _report(n: int) -> bytes: """The content SERVE_RECORDED serves at its nth request for report.bin.""" return b"agentenv report %d\n" % n * 64 diff --git a/tst/unit/task_step/test_browser_connect_cdp.py b/tst/unit/task_step/test_browser_connect_cdp.py index 9dbc645f0..7e1616137 100644 --- a/tst/unit/task_step/test_browser_connect_cdp.py +++ b/tst/unit/task_step/test_browser_connect_cdp.py @@ -12,7 +12,7 @@ from agent_env.env.env import DeployedSandboxEnv from agent_env.task_step.context import DeployedAgent, TaskStepContext -from agent_env.task_step.task_steps.browser_connect_cdp import BrowserConnectCdpTaskStep +from agent_env.task_step.task_steps.browser.connect_cdp import BrowserConnectCdpTaskStep _ENV = "http://localhost:41000" _AGENT = "http://localhost:42000" diff --git a/tst/unit/task_step/test_browser_steps.py b/tst/unit/task_step/test_browser_steps.py new file mode 100644 index 000000000..601b1381c --- /dev/null +++ b/tst/unit/task_step/test_browser_steps.py @@ -0,0 +1,241 @@ +"""The browser task steps: each calls its browser/v1 operation on the env's browser with the control token, the tab +steps pick the tab by its URL as the browser shows it, a browser whose card lacks the operation is refused before any +call, preflight refuses an env with no browser or none of the name given, and fields that don't fit are config +problems naming no cookie's value. Only the HTTP boundary is mocked.""" + +from __future__ import annotations + +import json +import re +from types import SimpleNamespace + +import httpx +import pytest + +from agent_env.env.env import DeployedSandboxEnv +from agent_env.env.envs.computer.app_env import ComputerAppEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.store.base import NotFoundError +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.browser import base +from agent_env.task_step.task_steps.browser.cookies import BrowserDeleteCookiesTaskStep, BrowserSetCookiesTaskStep +from agent_env.task_step.task_steps.browser.local_storage import (BrowserClearLocalStorageTaskStep, + BrowserSetLocalStorageTaskStep) +from agent_env.task_step.task_steps.browser.tabs import (BrowserActivateTabTaskStep, BrowserCloseTabTaskStep, + BrowserOpenTabTaskStep, normalized) + +_ENV = "http://localhost:41000" +_PREFIX = "/svc/chromium/agentenv/ext/browser" +_OPS = ("cookies/set", "cookies/delete", "local_storage/set", "local_storage/clear", "tabs/open", "tabs/list", + "tabs/close", "tabs/activate") +_TABS = [{"id": "A", "url": "https://shop.test/", "active": True}, {"id": "B", "url": "https://shop.test/orders?page=2", + "active": False}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("step", "op", "body", "recorded"), [ + (BrowserSetCookiesTaskStep(id="s", version=None, env_id="computer", cookies=[ + {"name": "sid", "value": "ada", "url": "https://shop.test/", "max_age": 3600}]), + "cookies/set", {"cookies": [{"name": "sid", "value": "ada", "url": "https://shop.test/", "max_age": 3600}]}, None), + (BrowserDeleteCookiesTaskStep(id="s", version=None, env_id="computer", name="sid", domain=".shop.test"), + "cookies/delete", {"name": "sid", "domain": ".shop.test"}, {"deleted": 2}), + (BrowserDeleteCookiesTaskStep(id="s", version=None, env_id="computer"), "cookies/delete", {}, {"deleted": 2}), + (BrowserSetLocalStorageTaskStep(id="s", version=None, env_id="computer", origin="https://shop.test", + items={"cart": "3", "theme": "dark"}), + "local_storage/set", {"origins": [{"origin": "https://shop.test", "localStorage": [ + {"name": "cart", "value": "3"}, {"name": "theme", "value": "dark"}]}]}, None), + (BrowserClearLocalStorageTaskStep(id="s", version=None, env_id="computer", origin="https://shop.test"), + "local_storage/clear", {"origins": ["https://shop.test"]}, None), + (BrowserClearLocalStorageTaskStep(id="s", version=None, env_id="computer"), "local_storage/clear", {}, None), + (BrowserOpenTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/help", background=True), + "tabs/open", {"url": "https://shop.test/help", "background": True}, {"tab": "C"}), +]) +async def test_each_step_calls_its_operation_on_the_browser_with_the_control_token(monkeypatch, step, op, body, + recorded): + sent = _mock(monkeypatch) + context = TaskStepContext(deployed_envs=[_env()], metadata={}) + + await step.execute(context) + + (request,) = sent + assert (str(request.url), json.loads(request.content)) == (f"{_ENV}{_PREFIX}/{op}", body) + assert request.headers["authorization"] == "Bearer control" + assert context.metadata.get("browser_steps", {}).get("s") == ( + None if recorded is None else {"env_id": "computer", "browser": None, **recorded}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("step_class", "op"), [(BrowserCloseTabTaskStep, "tabs/close"), + (BrowserActivateTabTaskStep, "tabs/activate")]) +@pytest.mark.parametrize(("url", "tab"), [("HTTPS://Shop.test:443", "A"), ("https://shop.test/orders?page=2", "B")]) +async def test_a_tab_step_acts_on_the_one_tab_at_its_url_as_the_browser_shows_it(monkeypatch, step_class, op, url, + tab): + sent = _mock(monkeypatch) + context = TaskStepContext(deployed_envs=[_env()], metadata={}) + + await step_class(id="s", version=None, env_id="computer", url=url).execute(context) + + listed, acted = sent + assert str(listed.url) == f"{_ENV}{_PREFIX}/tabs/list" + assert (str(acted.url), json.loads(acted.content)) == (f"{_ENV}{_PREFIX}/{op}", {"tab": tab}) + assert context.metadata["browser_steps"]["s"]["tab"] == tab + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("tabs", "url", "says"), [ + (_TABS, "https://shop.test/orders", "no tabs are at https://shop.test/orders in the browser of env 'computer'; " + "the tabs open are at ['https://shop.test/', 'https://shop.test/orders?page=2']"), + ([_TABS[0], {**_TABS[0], "id": "C"}], "https://shop.test/", "2 tabs are at https://shop.test/ in the browser"), +]) +async def test_no_tab_or_several_at_a_tab_steps_url_fail_it_naming_the_tabs_open(monkeypatch, tabs, url, says): + sent = _mock(monkeypatch, tabs=tabs) + + with pytest.raises(RuntimeError, match=re.escape(says)): + await BrowserCloseTabTaskStep(id="s", version=None, env_id="computer", url=url).execute( + TaskStepContext(deployed_envs=[_env()], metadata={})) + assert [str(request.url) for request in sent] == [f"{_ENV}{_PREFIX}/tabs/list"] + + +@pytest.mark.asyncio +async def test_a_browser_whose_card_lacks_the_operation_or_isnt_there_is_refused_before_any_call(monkeypatch): + sent = _mock(monkeypatch) + old = _env(ops=[op for op in _OPS if op != "tabs/open"]) + step = BrowserOpenTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/") + + with pytest.raises(RuntimeError, match="'chromium' in env 'computer' serves no browser/v1 tabs/open"): + await step.execute(TaskStepContext(deployed_envs=[old], metadata={})) + with pytest.raises(RuntimeError, match="has no website browser named 'firefox'"): + await BrowserOpenTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/", + browser="firefox").execute(TaskStepContext(deployed_envs=[_env()], metadata={})) + with pytest.raises(RuntimeError, match="isn't in context.deployed_envs"): + await step.execute(TaskStepContext(deployed_envs=[], metadata={})) + assert sent == [] + + +_NO_BROWSER = "env 'computer' has no website browser: browser steps act on a computer env's browser app, or a browser deployed alone" + + +@pytest.mark.parametrize(("stored", "browser", "problem"), [ + ("desktop", None, None), + ("alone", None, None), + ("alone", "chromium", None), + ("two", "brave", None), + ("missing", None, None), + ("plain", None, _NO_BROWSER), + ("website", None, _NO_BROWSER), + ("desktop", "firefox", "env 'computer' has no website browser named 'firefox' (it has ['chromium'])"), + ("two", None, "env 'computer' has 2 website browsers, ['chromium', 'brave']: name one as browser"), +]) +def test_preflight_refuses_an_env_with_no_browser_none_of_its_name_or_several_unnamed(monkeypatch, stored, browser, + problem): + def get(env_id): + if stored == "missing": + raise NotFoundError(env_id) + return _stored(stored) + + monkeypatch.setattr(base, "Env", SimpleNamespace(get=get)) + step = BrowserActivateTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/", + browser=browser) + + assert step.preflight() == ([] if problem is None else [f"step 's': {problem}"]) + + +@pytest.mark.parametrize(("step", "problems"), [ + (BrowserSetCookiesTaskStep(id="s", version=None, env_id="", cookies=[{"name": "sid", "value": "hunter2", + "colour": "red"}]), + ["step 's': env_id must be a non-empty string", "step 's': cookies[0]: colour: Extra inputs are not permitted"]), + (BrowserSetCookiesTaskStep(id="s", version=None, env_id="computer", cookies=[]), + ["step 's': cookies must be a non-empty list"]), + (BrowserDeleteCookiesTaskStep(id="s", version=None, env_id="computer", domain=""), + ["step 's': domain, when given, must be a non-empty string"]), + (BrowserSetLocalStorageTaskStep(id="s", version=None, env_id="computer", origin="https://shop.test/cart", + items={"cart": 3}), + ["step 's': origin: 'https://shop.test/cart' is a URL, not an origin: give only its scheme, host and port", + "step 's': items must be a non-empty map of keys to string values"]), + (BrowserClearLocalStorageTaskStep(id="s", version=None, env_id="computer", origin="shop.test"), + ["step 's': origin: 'shop.test' isn't an http or https origin"]), + (BrowserOpenTabTaskStep(id="s", version=None, env_id="computer", url="/orders", background="yes"), + ["step 's': url '/orders' isn't absolute: give its scheme and host, such as https://shop.test/", + "step 's': background must be true or false"]), + (BrowserCloseTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test:port/", browser=""), + ["step 's': browser, when given, must be a non-empty string", + "step 's': url 'https://shop.test:port/' isn't a URL: Port could not be cast to integer value as 'port'"]), +]) +def test_fields_that_dont_fit_are_config_problems_and_a_cookies_value_never_shows(step, problems): + found = step.config_problems() + + assert found == problems + assert "hunter2" not in " ".join(found) + + +@pytest.mark.parametrize("step", [ + BrowserSetCookiesTaskStep(id="s", version=None, env_id="computer", browser="chromium", + cookies=[{"name": "sid", "value": "ada", "url": "https://shop.test/"}]), + BrowserDeleteCookiesTaskStep(id="s", version=None, env_id="computer", name="sid", domain="shop.test", path="/a"), + BrowserSetLocalStorageTaskStep(id="s", version=None, env_id="computer", origin="https://shop.test", + items={"cart": "3"}), + BrowserClearLocalStorageTaskStep(id="s", version=None, env_id="computer"), + BrowserOpenTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/", background=True), + BrowserCloseTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/"), + BrowserActivateTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/"), +]) +def test_each_step_is_registered_and_round_trips(step): + registered = get_task_step_registry()[step.type] + + assert registered is type(step) and registered.from_dict(step.to_dict()).to_dict() == step.to_dict() + + +@pytest.mark.parametrize(("url", "shown"), [ + ("HTTPS://Shop.TEST", "https://shop.test/"), + ("http://shop.test:80/a?b=1#c", "http://shop.test/a?b=1#c"), + ("https://shop.test:8443", "https://shop.test:8443/"), + ("http://[::1]:8080", "http://[::1]:8080/"), + ("about:blank", "about:blank"), +]) +def test_a_url_is_normalized_as_the_browser_shows_it(url, shown): + assert normalized(url) == shown + + +def _env(ops=_OPS) -> DeployedSandboxEnv: + """A browser computer's deployment, its browser app chromium among its children, serving ``ops``.""" + browser = {"uri": "urn:agentenv:browser/v1", "params": {"endpoint": _PREFIX, "methods": { + op: {"endpoint": f"{_PREFIX}/{op}", **({"method": "GET"} if op == "tabs/list" else {})} for op in ops}}} + return DeployedSandboxEnv( + env_id="computer", env_version=1, sandbox_id="sb", sandbox_type="local", control_token="control", + environment_card_url=f"{_ENV}/.well-known/agent-env.json", + environment_card={"name": "chromium-driver", "capabilities": {"operations": ["reset"]}, + "children_environments": [{"name": "chromium", + "capabilities": {"extensions": [browser]}}]}) + + +def _stored(kind: str): + """A stored env without its constructor: a desktop with chromium, the browser alone, a desktop with two browsers + or none, or a website env.""" + def app(name, browser=True): + made = ComputerAppEnv.__new__(ComputerAppEnv) + made.environment_name, made.declaration = name, SimpleNamespace(browser=browser) + return made + + if kind == "alone": + return app("chromium") + if kind == "website": + return SimpleNamespace(type="website") + desktop = ComputerEnv.__new__(ComputerEnv) + desktop.apps = {"desktop": [app("chromium"), app("notes", browser=None)], "two": [app("chromium"), app("brave")], + "plain": [app("notes", browser=None)]}[kind] + return desktop + + +def _mock(monkeypatch, *, tabs=_TABS) -> list[httpx.Request]: + """Answer the browser's operations as a browser would, ``tabs`` open; return the requests.""" + sent, real = [], httpx.AsyncClient + + def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + op = request.url.path.removeprefix(f"{_PREFIX}/") + return httpx.Response(200, json={"cookies/delete": {"deleted": 2}, "local_storage/clear": {"origins": []}, + "tabs/open": {"tab": "C"}, "tabs/list": {"tabs": tabs}}.get(op, {})) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + return sent From 9ce68d5408d5a7bed54e469227fd7523c3f82c04 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 12:25:50 -0800 Subject: [PATCH 2/9] fix(website-browser): a background open brings the old tab back only if the new one took its place So a tab another call switched to while the new tab loaded stays in front. Co-Authored-By: Claude Opus 5.5 --- .../website_browser/control.py | 6 +++--- .../tests/test_website_browser.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index f3026a390..b8da02816 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -122,8 +122,8 @@ async def tab_list(self) -> list[dict]: async def open_tab(self, url: Optional[str] = None, background: bool = False) -> str: """A new tab at ``url``, else blank, brought to the front, or with ``background`` behind the active tab, which - stays active; its id. Headless, it opens blank and goes to ``url`` once it's held to the window's size, so its - page loads at that size.""" + stays active, brought back only if the new tab took its place; its id. Headless, it opens blank and goes to + ``url`` once it's held to the window's size, so its page loads at that size.""" active = await self.active_tab() if background else None held = self.viewport is not None and url start = "about:blank" if held else url or "about:blank" @@ -134,7 +134,7 @@ async def open_tab(self, url: Optional[str] = None, background: bool = False) -> await self.devtools.call("Page.navigate", {"url": url}, session_id=session) if not background: await self._bring_to_front(target) - elif active is not None and active in self.tabs and await self.active_tab() != active: + elif active is not None and active in self.tabs and await self.active_tab() == target: await self._bring_to_front(active) return target diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 413073428..810ec6ff7 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -689,6 +689,24 @@ async def test_a_tab_opened_in_the_background_opens_behind_the_active_one_which_ assert [(tab["id"], tab["active"]) for tab in await control.tab_list()] == [("a", True), ("b", False)] +@pytest.mark.asyncio +async def test_a_background_open_leaves_a_tab_another_call_switched_to_meanwhile_in_front(): + focus = {"s-a": True} + devtools = Tabbed({"Runtime.evaluate": lambda params, session: { + "result": {"value": ["visible", focus.get(session, False)]}}}) + control = await _tabs(devtools, "a", "c") + + opening = asyncio.ensure_future(control.open_tab("https://mail.test/b", background=True)) + await asyncio.sleep(0) + focus.update({"s-a": False, "s-c": True}) + devtools.send("Target.targetCreated", _page("b", "https://mail.test/b", "Mail b")) + devtools.send("Target.attachedToTarget", {"sessionId": "s-b", **_page("b")}) + await opening + + assert _sent(devtools, "Target.activateTarget") == [] + assert [(tab["id"], tab["active"]) for tab in await control.tab_list()] == [("a", False), ("c", True), ("b", False)] + + @pytest.mark.asyncio async def test_a_hidden_tab_whose_page_still_says_it_has_focus_isnt_the_active_one(): states = {"s-a": ["hidden", True], "s-b": ["visible", True]} From 62b83ff48d7afb4467dab49f6feabac4819cf902 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 12:45:19 -0800 Subject: [PATCH 3/9] fix(episode): the browser task steps name no agent, so an episode with outside agents runs them Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/rules.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 586202194..a2e9efc50 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -149,7 +149,9 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s UNAFFECTED = frozenset({ # They name no agent. - "aggregate_verifiers", "apply_server_config", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks", + "aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage", + "browser_close_tab", "browser_delete_cookies", "browser_open_tab", "browser_set_cookies", + "browser_set_local_storage", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_computer_recording", "start_computer_recording", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", "run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "snapshot_computer_env", "sync_env_clock", From c24f034bb3075820c7d85894b0b05285ebb47c45 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 13:13:08 -0800 Subject: [PATCH 4/9] fix(task): a step's preflight reads its env at the version the task's deploy_env pins A task's preflight, and a bundle's, give each step the env versions the deploy_env steps before it pin, the first for each env, and the browser steps, load_artifact and snapshot_computer_env read their env at that version through stored_env, else at its latest. A step reading an env its run deploys at another version no longer passes or fails preflight on the wrong one. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/materialize.py | 9 +- src/agent_env/task/task.py | 7 +- src/agent_env/task_step/preflight.py | 33 +++++++ src/agent_env/task_step/task_step.py | 5 ++ .../task_step/task_steps/browser/base.py | 7 +- .../computer/snapshot_computer_env.py | 7 +- .../task_step/task_steps/deploy_env.py | 3 + .../task_step/task_steps/load_artifact.py | 7 +- tst/unit/task_step/test_browser_steps.py | 6 +- .../test_preflight_pinned_versions.py | 85 +++++++++++++++++++ 10 files changed, 153 insertions(+), 16 deletions(-) create mode 100644 src/agent_env/task_step/preflight.py create mode 100644 tst/unit/task_step/test_preflight_pinned_versions.py diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index 840fbf553..60cd26daf 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -37,6 +37,7 @@ from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing_enabled from agent_env.task.task import Task +from agent_env.task_step.preflight import pinned from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep from agent_env.utils.docker.build import DockerBuildError @@ -450,8 +451,11 @@ def _preflight(plan: Plan, write: Write, task: Task, unwritten: set[tuple[str, s a step that deploys on a provider for the run's ``sandbox``, which overrides it.""" outputs = {output.id for output in write.source.outputs} solver = plan.agent.steps.get(write.id, ()) if plan.agent is not None else () - problems, unchecked = [], [] + problems, unchecked, versions = [], [], {} for config, step in zip(write.source.config, task.steps): + before = dict(versions) + if (deployed := step.deploys_env()) is not None: + versions.setdefault(*deployed) overrides = {key: sandbox for key in _SANDBOX_FIELDS if sandbox is not None and hasattr(step, key)} if step.id in solver: overrides |= {"a2a_agent_id": plan.agent.agent_id, "a2a_agent_version": plan.agent.version} @@ -466,7 +470,8 @@ def _preflight(plan: Plan, write: Write, task: Task, unwritten: set[tuple[str, s if type(step).preflight is not TaskStep.preflight: unchecked.append(step) continue - problems.extend(step.preflight()) + with pinned(before): + problems.extend(step.preflight()) return problems, unchecked diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index 2d217b2de..69d002d74 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -23,6 +23,7 @@ from agent_env.plugins._requirements import unmet_problems from agent_env.store.document_store import UnknownDocumentTypeError from agent_env.store.routing import run_scope +from agent_env.task_step.preflight import pinned from agent_env.task.teardown import env_record_ids from agent_env.task_step.context import TaskStepContext, regraft_redacted_keys from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops @@ -405,9 +406,13 @@ def preflight(self) -> list[str]: Covers what needs the stores to answer; `_validate_dag` already covers ids and edges. """ problems: list[str] = [] + versions: dict[str, Optional[int]] = {} for step in self.steps: problems.extend(step.config_problems()) - problems.extend(step.preflight()) + with pinned(versions): + problems.extend(step.preflight()) + if (deployed := step.deploys_env()) is not None: + versions.setdefault(*deployed) return problems def to_dict(self) -> dict: diff --git a/src/agent_env/task_step/preflight.py b/src/agent_env/task_step/preflight.py new file mode 100644 index 000000000..067a85395 --- /dev/null +++ b/src/agent_env/task_step/preflight.py @@ -0,0 +1,33 @@ +"""What a step's preflight knows of the task it's in: the env version each deploy_env step before it pins, so a step +reads the env its run will deploy rather than the latest one stored.""" + +from __future__ import annotations + +import contextlib +from collections.abc import Iterator, Mapping +from contextvars import ContextVar +from typing import Optional + +from agent_env.env.env import Env +from agent_env.store.base import NotFoundError + +_PINNED: ContextVar[Mapping[str, Optional[int]]] = ContextVar("pinned_env_versions", default={}) + + +@contextlib.contextmanager +def pinned(versions: Mapping[str, Optional[int]]) -> Iterator[None]: + """``versions``, by env id, the ones a preflight in the block reads its envs at.""" + token = _PINNED.set(dict(versions)) + try: + yield + finally: + _PINNED.reset(token) + + +def stored_env(env_id: str) -> Optional[Env]: + """The env ``env_id`` at the version a deploy_env step before this one pins, else its latest; None when the store + has no such env, which the run reports.""" + try: + return Env.get(env_id, _PINNED.get().get(env_id)) + except NotFoundError: + return None diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index 44c56c723..ddf6239b7 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -122,6 +122,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext | None: """Execute this task step.""" pass + def deploys_env(self) -> Optional[tuple[str, Optional[int]]]: + """The env id this step deploys and the version it pins, None for its latest, which the preflights of the + steps after it read that env at; None for a step that deploys no env.""" + return None + def preflight(self) -> list[str]: """Config problems detectable before a run, one message each; empty when fine. diff --git a/src/agent_env/task_step/task_steps/browser/base.py b/src/agent_env/task_step/task_steps/browser/base.py index b292c3f5f..7b25f21be 100644 --- a/src/agent_env/task_step/task_steps/browser/base.py +++ b/src/agent_env/task_step/task_steps/browser/base.py @@ -10,8 +10,8 @@ from agent_env.env.env import DeployedEnv, Env from agent_env.env.envs.computer.env import ComputerEnv -from agent_env.store.base import NotFoundError from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.preflight import stored_env from agent_env.task_step.task_step import TaskStep, TaskStepDependency #: How long a browser/v1 operation may take to answer. @@ -42,9 +42,8 @@ def preflight(self) -> list[str]: problems = super().preflight() if not isinstance(self.env_id, str) or not self.env_id: return problems - try: - env = Env.get(self.env_id) - except NotFoundError: # what the run reports + env = stored_env(self.env_id) + if env is None: return problems names = stored_browsers(env) if not names: diff --git a/src/agent_env/task_step/task_steps/computer/snapshot_computer_env.py b/src/agent_env/task_step/task_steps/computer/snapshot_computer_env.py index 4f5f35ba1..cebe70606 100644 --- a/src/agent_env/task_step/task_steps/computer/snapshot_computer_env.py +++ b/src/agent_env/task_step/task_steps/computer/snapshot_computer_env.py @@ -12,9 +12,9 @@ from agent_env.entity.refs import EntityRef, RefRole from agent_env.env.env import Env from agent_env.env.envs.computer.env import ComputerEnv -from agent_env.store.base import NotFoundError from agent_env.store.ids import is_local_id, validate_local_id from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.preflight import stored_env from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.snapshot_env import DEFAULT_EXPORT_TIMEOUT_SECONDS, SnapshotEnvTaskStep @@ -81,9 +81,8 @@ def preflight(self) -> list[str]: problems = super().preflight() if self.env_id is None: return problems - try: - env = Env.get(self.env_id) - except NotFoundError: # what the run reports + env = stored_env(self.env_id) + if env is None: # what the run reports return problems return problems if isinstance(env, ComputerEnv) else [*problems, _not_a_computer(env)] diff --git a/src/agent_env/task_step/task_steps/deploy_env.py b/src/agent_env/task_step/task_steps/deploy_env.py index 1f8047b01..d8b55bec6 100644 --- a/src/agent_env/task_step/task_steps/deploy_env.py +++ b/src/agent_env/task_step/task_steps/deploy_env.py @@ -111,6 +111,9 @@ def from_dict(cls, data: dict) -> DeployEnvTaskStep: env_params=data.get("env_params"), ) + def deploys_env(self) -> Optional[tuple[str, Optional[int]]]: + return (self.env_id, self.env_version) if isinstance(self.env_id, str) and self.env_id else None + def preflight(self) -> list[str]: """An invalid gateway_mode, an env this step can't load, or an option it would refuse at deploy, reported at save instead.""" from agent_env.env.env import Env diff --git a/src/agent_env/task_step/task_steps/load_artifact.py b/src/agent_env/task_step/task_steps/load_artifact.py index fdecb9e75..e2f4a8c55 100644 --- a/src/agent_env/task_step/task_steps/load_artifact.py +++ b/src/agent_env/task_step/task_steps/load_artifact.py @@ -19,6 +19,7 @@ from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.preflight import stored_env from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope from agent_env.store.base import NotFoundError @@ -225,10 +226,12 @@ def preflight(self) -> list[str]: """An environment artifact this step names for an env whose type loads none.""" if self.env_id is None: return [] + env = stored_env(self.env_id) + if env is None: # what the run reports + return [] try: - env = Env.get(self.env_id) artifacts = [Artifact.get(ref["id"], ref.get("version")) for ref in self.artifacts] - except NotFoundError: # what the run reports + except NotFoundError: return [] return [refusal for artifact in artifacts if (refusal := self._unloadable(env, self.env_id, artifact))] diff --git a/tst/unit/task_step/test_browser_steps.py b/tst/unit/task_step/test_browser_steps.py index 601b1381c..f5003d6e2 100644 --- a/tst/unit/task_step/test_browser_steps.py +++ b/tst/unit/task_step/test_browser_steps.py @@ -18,7 +18,7 @@ from agent_env.store.base import NotFoundError from agent_env.task_step.context import TaskStepContext from agent_env.task_step.registry import get_task_step_registry -from agent_env.task_step.task_steps.browser import base +from agent_env.task_step import preflight from agent_env.task_step.task_steps.browser.cookies import BrowserDeleteCookiesTaskStep, BrowserSetCookiesTaskStep from agent_env.task_step.task_steps.browser.local_storage import (BrowserClearLocalStorageTaskStep, BrowserSetLocalStorageTaskStep) @@ -129,12 +129,12 @@ async def test_a_browser_whose_card_lacks_the_operation_or_isnt_there_is_refused ]) def test_preflight_refuses_an_env_with_no_browser_none_of_its_name_or_several_unnamed(monkeypatch, stored, browser, problem): - def get(env_id): + def get(env_id, version=None): if stored == "missing": raise NotFoundError(env_id) return _stored(stored) - monkeypatch.setattr(base, "Env", SimpleNamespace(get=get)) + monkeypatch.setattr(preflight, "Env", SimpleNamespace(get=get)) step = BrowserActivateTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/", browser=browser) diff --git a/tst/unit/task_step/test_preflight_pinned_versions.py b/tst/unit/task_step/test_preflight_pinned_versions.py new file mode 100644 index 000000000..6bfd5a87a --- /dev/null +++ b/tst/unit/task_step/test_preflight_pinned_versions.py @@ -0,0 +1,85 @@ +"""A step's preflight reads its env at the version the first deploy_env before it in the task pins, else the latest: +through a task's preflight and a bundle's, for the browser steps, load_artifact and snapshot_computer_env.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from agent_env.bundle import materialize +from agent_env.env.envs.computer.app_env import ComputerAppEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.task.task import Task +from agent_env.task_step import preflight +from agent_env.task_step.task_steps import load_artifact +from agent_env.task_step.task_steps.browser.tabs import BrowserActivateTabTaskStep +from agent_env.task_step.task_steps.computer.snapshot_computer_env import SnapshotComputerEnvTaskStep +from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep +from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep + +_NO_BROWSER = ("step 'early': env 'desk' has no website browser: browser steps act on a computer env's browser app, " + "or a browser deployed alone") + + +@pytest.fixture +def asked(monkeypatch) -> list: + """The (env id, version) each preflight reads, from a store whose desk has chromium at version 1 only.""" + read = [] + + def get(env_id, version=None): + read.append((env_id, version)) + return _desk(browser=version == 1) + + monkeypatch.setattr(preflight, "Env", SimpleNamespace(get=get)) + monkeypatch.setattr(DeployEnvTaskStep, "preflight", lambda self: []) + return read + + +def _steps() -> list: + return [_activate("early"), DeployEnvTaskStep(id="deploy", version=None, env_id="desk", env_version=1), + _activate("pinned"), + DeployEnvTaskStep(id="again", version=None, env_id="desk", env_version=2), _activate("still")] + + +def test_a_tasks_preflight_reads_a_steps_env_at_the_version_the_first_deploy_before_it_pins(asked): + problems = Task(id="t", version=None, steps=_steps()).preflight() + + assert problems == [_NO_BROWSER] + assert asked == [("desk", None), ("desk", 1), ("desk", 1)] + + +def test_a_bundles_preflight_reads_them_so_too(asked): + steps = _steps() + write = SimpleNamespace(id="t", source=SimpleNamespace(config=[step.to_dict() for step in steps], outputs=[])) + + problems, unchecked = materialize._preflight(SimpleNamespace(agent=None), write, Task(id="t", version=None, + steps=steps), set()) + + assert (problems, unchecked) == ([_NO_BROWSER], []) + assert asked == [("desk", None), ("desk", 1), ("desk", 1)] + + +def test_load_artifact_and_snapshot_computer_env_read_the_pinned_version_too(asked, monkeypatch): + monkeypatch.setattr(load_artifact.Artifact, "get", lambda id, version=None: SimpleNamespace(id=id)) + loading = LoadArtifactTaskStep(id="load", version=None, env_id="desk", artifact_id="seed") + snapshot = SnapshotComputerEnvTaskStep(id="snap", version=None, env_id="desk") + + with preflight.pinned({"desk": 3}): + assert loading.preflight() == [] and snapshot.preflight() == [] + assert snapshot.preflight() == [] + + assert asked == [("desk", 3), ("desk", 3), ("desk", None)] + + +def _activate(step_id: str) -> BrowserActivateTabTaskStep: + return BrowserActivateTabTaskStep(id=step_id, version=None, env_id="desk", url="https://shop.test/") + + +def _desk(browser: bool) -> ComputerEnv: + """A stored desktop, without its constructor: chromium among its apps or not.""" + app = ComputerAppEnv.__new__(ComputerAppEnv) + app.environment_name, app.declaration = "chromium", SimpleNamespace(browser=browser or None) + desk = ComputerEnv.__new__(ComputerEnv) + desk.apps = [app] + return desk From 1eaae4090bb9d4ed8a9657cd93c465d9a76c5443 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 14:50:15 -0800 Subject: [PATCH 5/9] fix(browser): browser_open_tab sends background only when set, and tab URLs match a host in another script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A browser built before tabs/open took background refuses an unknown parameter, so a plain open sends none and works on it. A tab step's URL and a tab's are both normalized with the host as punycode, as the browser shows it, so https://bücher.test/ matches its tab. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/browser/tabs.py | 17 ++++++++++++----- tst/unit/task_step/test_browser_steps.py | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/browser/tabs.py b/src/agent_env/task_step/task_steps/browser/tabs.py index 60d868830..f4680bdc9 100644 --- a/src/agent_env/task_step/task_steps/browser/tabs.py +++ b/src/agent_env/task_step/task_steps/browser/tabs.py @@ -3,6 +3,7 @@ from __future__ import annotations +import contextlib import logging from typing import ClassVar, Optional from urllib.parse import urlsplit, urlunsplit @@ -19,7 +20,7 @@ class BrowserOpenTabTaskStep(BrowserStep): """Open a tab at ``url``, in front, or with ``background`` behind the active tab, which stays active; the new tab's - id is recorded.""" + id is recorded. ``background`` is sent only when set, so a browser built before it opens tabs in front.""" type: ClassVar[str] = "browser_open_tab" entity_refs = (EntityRef.env("env_id"),) @@ -44,7 +45,8 @@ def from_dict(cls, data: dict) -> BrowserOpenTabTaskStep: return cls(**cls._browser_from_dict(data), url=data["url"], background=data.get("background", False)) async def execute(self, context: TaskStepContext) -> TaskStepContext: - tab = (await self._call(context, "tabs/open", {"url": self.url, "background": self.background}))["tab"] + params = {"url": self.url, **({"background": True} if self.background else {})} + tab = (await self._call(context, "tabs/open", params))["tab"] logger.info(f"{self.id}: opened tab {tab} at {self.url}{' in the background' if self.background else ''} in " f"the browser of env {self.env_id!r}") self._record(context, tab=tab) @@ -104,13 +106,18 @@ class BrowserActivateTabTaskStep(_TabAtUrlStep): def normalized(url: str) -> str: - """``url`` as the browser shows it: its scheme and host lowercase, its default port dropped, and an empty path as - ``/``; its query and fragment as they are. A URL with no host, such as ``about:blank``, is left as it is.""" + """``url`` as the browser shows it: its scheme and host lowercase, a host in another script as punycode, its default + port dropped, and an empty path as ``/``; its query and fragment as they are. A URL with no host, such as + ``about:blank``, is left as it is.""" parts = urlsplit(url) if not parts.netloc: return url scheme, host = parts.scheme.lower(), (parts.hostname or "") - host = f"[{host}]" if ":" in host else host + if ":" in host: + host = f"[{host}]" + elif not host.isascii(): + with contextlib.suppress(UnicodeError): + host = host.encode("idna").decode("ascii") port = parts.port netloc = host if port is None or port == _DEFAULT_PORTS.get(scheme) else f"{host}:{port}" if parts.username is not None: diff --git a/tst/unit/task_step/test_browser_steps.py b/tst/unit/task_step/test_browser_steps.py index f5003d6e2..7c7fa8882 100644 --- a/tst/unit/task_step/test_browser_steps.py +++ b/tst/unit/task_step/test_browser_steps.py @@ -50,6 +50,8 @@ (BrowserClearLocalStorageTaskStep(id="s", version=None, env_id="computer"), "local_storage/clear", {}, None), (BrowserOpenTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/help", background=True), "tabs/open", {"url": "https://shop.test/help", "background": True}, {"tab": "C"}), + (BrowserOpenTabTaskStep(id="s", version=None, env_id="computer", url="https://shop.test/help"), + "tabs/open", {"url": "https://shop.test/help"}, {"tab": "C"}), ]) async def test_each_step_calls_its_operation_on_the_browser_with_the_control_token(monkeypatch, step, op, body, recorded): @@ -191,6 +193,7 @@ def test_each_step_is_registered_and_round_trips(step): ("http://shop.test:80/a?b=1#c", "http://shop.test/a?b=1#c"), ("https://shop.test:8443", "https://shop.test:8443/"), ("http://[::1]:8080", "http://[::1]:8080/"), + ("https://Bücher.test/a", "https://xn--bcher-kva.test/a"), ("about:blank", "about:blank"), ]) def test_a_url_is_normalized_as_the_browser_shows_it(url, shown): From 444ae71f05add882c3deeb18534438685608e78e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 15:09:35 -0800 Subject: [PATCH 6/9] fix(browser): tab URLs spell a host in another script as Chromium does, by UTS 46 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python's idna codec maps faß.test to fass.test, a different site, where Chromium keeps them apart, so a tab step naming one could act on the other's tab. The host is now encoded with the idna package's UTS 46 mapping without transitional processing, as Chromium does. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + src/agent_env/task_step/task_steps/browser/tabs.py | 13 ++++++++----- tst/unit/task_step/test_browser_steps.py | 2 ++ uv.lock | 2 ++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c420f0975..f35d066fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "cryptography>=42", # the local transfer CA "httpx>=0.27.0", "huggingface_hub>=1.0", # agent-env run hf:// resolves and downloads a Hub bundle; checked on 1.0, 1.33 and 2.4 + "idna>=3.0", # a browser tab step matches a URL's host as Chromium spells it, by UTS 46 "jsonschema>=4.20", # checks an app's settings against its Settings' schema before a deploy "litellm>=1.96.0,<1.97", "mcp>=1.25,<2", # same floor as env/gateway/requirements.txt and agentenv-protocol; 2.0.0 removed mcp.server.fastmcp, imported at module scope by env/gateway diff --git a/src/agent_env/task_step/task_steps/browser/tabs.py b/src/agent_env/task_step/task_steps/browser/tabs.py index f4680bdc9..69f756e17 100644 --- a/src/agent_env/task_step/task_steps/browser/tabs.py +++ b/src/agent_env/task_step/task_steps/browser/tabs.py @@ -8,6 +8,8 @@ from typing import ClassVar, Optional from urllib.parse import urlsplit, urlunsplit +import idna + from agent_env.entity.refs import EntityRef from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_step import TaskStepDependency @@ -106,9 +108,10 @@ class BrowserActivateTabTaskStep(_TabAtUrlStep): def normalized(url: str) -> str: - """``url`` as the browser shows it: its scheme and host lowercase, a host in another script as punycode, its default - port dropped, and an empty path as ``/``; its query and fragment as they are. A URL with no host, such as - ``about:blank``, is left as it is.""" + """``url`` as the browser shows it: its scheme and host lowercase, a host in another script as punycode by UTS 46 + without transitional mapping, as Chromium spells it (``faß`` stays apart from ``fass``), its default port dropped, + and an empty path as ``/``; its query and fragment as they are. A URL with no host, such as ``about:blank``, is left + as it is.""" parts = urlsplit(url) if not parts.netloc: return url @@ -116,8 +119,8 @@ def normalized(url: str) -> str: if ":" in host: host = f"[{host}]" elif not host.isascii(): - with contextlib.suppress(UnicodeError): - host = host.encode("idna").decode("ascii") + with contextlib.suppress(idna.IDNAError, UnicodeError): + host = idna.encode(host, uts46=True, transitional=False).decode("ascii") port = parts.port netloc = host if port is None or port == _DEFAULT_PORTS.get(scheme) else f"{host}:{port}" if parts.username is not None: diff --git a/tst/unit/task_step/test_browser_steps.py b/tst/unit/task_step/test_browser_steps.py index 7c7fa8882..b2c270cbc 100644 --- a/tst/unit/task_step/test_browser_steps.py +++ b/tst/unit/task_step/test_browser_steps.py @@ -194,6 +194,8 @@ def test_each_step_is_registered_and_round_trips(step): ("https://shop.test:8443", "https://shop.test:8443/"), ("http://[::1]:8080", "http://[::1]:8080/"), ("https://Bücher.test/a", "https://xn--bcher-kva.test/a"), + ("https://faß.test/", "https://xn--fa-hia.test/"), + ("https://fass.test/", "https://fass.test/"), ("about:blank", "about:blank"), ]) def test_a_url_is_normalized_as_the_browser_shows_it(url, shown): diff --git a/uv.lock b/uv.lock index 245ae44c4..fbb94a1f5 100644 --- a/uv.lock +++ b/uv.lock @@ -58,6 +58,7 @@ dependencies = [ { name = "fastapi" }, { name = "httpx" }, { name = "huggingface-hub" }, + { name = "idna" }, { name = "jsonschema" }, { name = "litellm" }, { name = "mcp" }, @@ -230,6 +231,7 @@ requires-dist = [ { name = "httpx2", marker = "extra == 'vercel'", specifier = ">=2.13.1,<3" }, { name = "huggingface-hub", specifier = ">=1.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.100" }, + { name = "idna", specifier = ">=3.0" }, { name = "jsonschema", specifier = ">=4.20" }, { name = "litellm", specifier = ">=1.96.0,<1.97" }, { name = "mcp", specifier = ">=1.25,<2" }, From e8041d84a0119087246f8f670a01d69caa58599a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 15:23:20 -0800 Subject: [PATCH 7/9] ci: retrigger the PR's checks on the latest commit Co-Authored-By: Claude Opus 5.5 From fcddd637d9b9393c7d629106a2297864901cd460 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 16:18:51 -0800 Subject: [PATCH 8/9] test(browser): tab steps find a host in another script by its URL as written, and a task pinning the browser's version runs them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser steps check opens a tab at faß.localhost, sees chromium spell it xn--fa-hia, and closes it by the URL as written, in a desktop and alone. A new browser computer test runs a whole task through task.run: its deploy_env pins a version of the browser whose latest version has none, its preflight passes on the pinned one, the same task unpinned reports every browser step, and its tab and cookie steps run on the version deployed. Co-Authored-By: Claude Opus 5.5 --- .../computer/browser_computer_test.py | 48 ++++++++++++++++++- tst/integration/computer/browser_steps.py | 22 +++++++-- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 0a38a470f..da4144c84 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -12,8 +12,9 @@ fresh one its next turn opens the site through. Through an app server's restart, which relaunches the browser, and a driver's, the agent's address keeps working, and two agents connected at once each drive the browser through their own. browser/v1 reads and sets its cookies, local storage and downloads, and its data plane signs it in, snapshots it, resets it, loads a -snapshot back and holds up under chaos; and the browser task steps set its cookies and storage and open, activate and -close its tabs by URL. Needs Docker and the local registry on +snapshot back and holds up under chaos; the browser task steps set its cookies and storage and open, activate and +close its tabs by URL; and a task whose deploy_env pins a version of the browser, its latest one with none, passes +preflight on the version pinned and runs its browser steps through task.run. Needs Docker and the local registry on localhost:5000, and Node 20 or later and npm for the CDP clients.""" import asyncio @@ -41,7 +42,9 @@ from agent_env.artifact.artifacts.file import FileArtifact from agent_env.env import Env from agent_env.env.envs.computer.utils import inside +from agent_env.env.store import get_env_store from agent_env.store.routing import namespace_routing +from agent_env.task import Task from agent_env.task.teardown import teardown_run from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.browser.connect_cdp import BrowserConnectCdpTaskStep @@ -121,6 +124,8 @@ def log_message(self, *args): SERVE_START_S = 3 #: A page that needs no server, so killing the driver, whose child a served site is, leaves it to open. DATA_PAGE = "data:text/html,Home

Home

" +#: A page a task's tab steps open, bring to the front and close by its URL. +PINNED_PAGE = "data:text/html,pinned" # Every TCP port something in the computer listens on, and whether any process is an X server. LISTENING = """import glob ports = set() @@ -368,6 +373,45 @@ async def test_its_task_steps_set_cookies_and_storage_open_a_tab_behind_and_acti assert not report.failed +@pytest.mark.asyncio +async def test_a_task_pinning_a_version_of_the_browser_passes_preflight_on_it_and_runs_its_steps_through_task_run( + browser_computer): + suffix = uuid.uuid4().hex[:8] + with namespace_routing(): + env = Env.get(browser_computer.id) + stored, env_id = env.to_dict(), f"{browser_computer.id}-pinned-{suffix}" + pinned = get_env_store().put_document(type(env).from_dict({**stored, "id": env_id, "version": None})) + no_browser = {key: value for key, value in stored["declaration"].items() if key != "browser"} + get_env_store().put_document(type(env).from_dict({**stored, "id": env_id, "version": None, + "declaration": no_browser})) + steps = [{"id": "env", "type": "deploy_env", "env_id": env_id, "env_version": pinned.version}, + {"id": "sign-in", "type": "browser_set_cookies", "env_id": env_id, + "cookies": [{"name": "sid", "value": "ada", "url": DATA_SITE, "max_age": 3600}]}, + {"id": "open", "type": "browser_open_tab", "env_id": env_id, "url": PINNED_PAGE, "background": True}, + {"id": "front", "type": "browser_activate_tab", "env_id": env_id, "url": PINNED_PAGE}, + {"id": "close", "type": "browser_close_tab", "env_id": env_id, "url": PINNED_PAGE}, + {"id": "sign-out", "type": "browser_delete_cookies", "env_id": env_id, "name": "sid"}] + task_id = f"@local/agentenv-framework/browsers/pinned-steps-{suffix}" + task = Task.put(id=task_id, steps=Task.from_dict({"id": task_id, "steps": steps}).steps) + unpinned = Task.from_dict({"id": task_id, "steps": [{**steps[0], "env_version": None}, *steps[1:]]}) + problems, unpinned_problems = task.preflight(), unpinned.preflight() + context = TaskStepContext() + try: + await task.run(context=context) + finally: + report = await teardown_run(context) + + assert problems == [], problems + assert [problem for problem in unpinned_problems if "has no website browser" in problem] == [ + f"step {step!r}: env {env_id!r} has no website browser: browser steps act on a computer env's browser app, " + "or a browser deployed alone" for step in ("sign-in", "open", "front", "close", "sign-out")], unpinned_problems + assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") + assert [(deployed.env_id, deployed.env_version) for deployed in context.deployed_envs] == [(env_id, pinned.version)] + records = context.metadata["browser_steps"] + assert records["open"]["tab"] == records["front"]["tab"] == records["close"]["tab"], records + assert records["sign-out"]["deleted"] == 1 and not report.failed, (records, report) + + @pytest.mark.asyncio async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_gone_they_fail_at_once( browser_computer): diff --git a/tst/integration/computer/browser_steps.py b/tst/integration/computer/browser_steps.py index ddb903c0c..d5771e684 100644 --- a/tst/integration/computer/browser_steps.py +++ b/tst/integration/computer/browser_steps.py @@ -1,7 +1,8 @@ """The browser task steps on a deployed chromium, inside a computer or alone, each made from a task's JSON as the step registry reads it: each passes preflight on its env; a cookie and local storage set by steps sign in a tab opened in the background, behind the active tab, which stays active; steps bring that tab to the front and close it by its URL, -delete the cookie and clear the storage; and a tab step naming a URL no tab shows fails, listing the tabs open.""" +delete the cookie and clear the storage; a tab step naming a URL no tab shows fails, listing the tabs open; and a tab at +a host in another script, which chromium spells by UTS 46, is closed by the URL as written.""" import asyncio import time @@ -14,6 +15,9 @@ from tst.integration.computer.browser_data import ORIGIN, POLL_S, SERVE_S, SERVE_SITE, SITE, WAIT_S, _client SIGNED_IN = "signed in as ada theme dark" +#: A page at a host in another script, on loopback as chromium sends any .localhost host: what's checked is how chromium +#: spells its URL, which holds whether or not the page loads. +IDN_PAGE = SITE.replace("127.0.0.1", "faß.localhost") + "other" async def check_browser_steps(deployed, env_id: str, op_url: Callable[[str], str]) -> None: @@ -27,14 +31,14 @@ def step(type: str, **fields) -> TaskStep: assert made.config_problems() + made.preflight() == [], type return made - async def tab_titled(tab: str, title: str) -> dict: + async def tab_until(tab: str, holds: Callable[[dict], bool], what: str) -> dict: last, deadline = None, time.monotonic() + WAIT_S while time.monotonic() < deadline: last = next((t for t in (await call("tabs/list", method="GET"))["tabs"] if t["id"] == tab), None) - if last is not None and last["title"] == title: + if last is not None and holds(last): return last await asyncio.sleep(POLL_S) - raise AssertionError(f"tab {tab} wasn't titled {title!r} within {WAIT_S:g} s: {last!r}") + raise AssertionError(f"tab {tab} wasn't {what} within {WAIT_S:g} s: {last!r}") serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_SITE, str(SERVE_S)], timeout_s=SERVE_S + 60)) @@ -47,7 +51,7 @@ async def tab_titled(tab: str, title: str) -> dict: context = await step("browser_set_local_storage", origin=ORIGIN, items={"theme": "dark"}).execute(context) context = await step("browser_open_tab", url=SITE, background=True).execute(context) opened = context.metadata["browser_steps"]["browser_open_tab"]["tab"] - behind = await tab_titled(opened, SIGNED_IN) + behind = await tab_until(opened, lambda tab: tab["title"] == SIGNED_IN, f"titled {SIGNED_IN!r}") after_open = [(tab["id"], tab["active"]) for tab in (await call("tabs/list", method="GET"))["tabs"]] context = await step("browser_activate_tab", url=SITE).execute(context) in_front = await titled(SIGNED_IN) @@ -58,6 +62,12 @@ async def tab_titled(tab: str, title: str) -> dict: missing = None except RuntimeError as e: missing = str(e) + context = await step("browser_open_tab", url=IDN_PAGE, background=True).execute(context) + idn_tab = context.metadata["browser_steps"]["browser_open_tab"]["tab"] + idn_shown = await tab_until(idn_tab, lambda tab: tab["url"] not in ("", "about:blank"), "past about:blank") + context = await step("browser_close_tab", url=IDN_PAGE).execute(context) + idn_closed = context.metadata["browser_steps"]["browser_close_tab"]["tab"] + after_idn = [tab["id"] for tab in (await call("tabs/list", method="GET"))["tabs"]] context = await step("browser_delete_cookies", name="sid").execute(context) context = await step("browser_clear_local_storage", origin=ORIGIN).execute(context) cookies = (await call("cookies/get", {"urls": [SITE]}))["cookies"] @@ -69,5 +79,7 @@ async def tab_titled(tab: str, title: str) -> dict: assert after_open == [(start["id"], True), (opened, False)], after_open assert in_front["id"] == opened and after_close == [f"{SITE}login"], after_close assert missing is not None and f"no tabs are at {SITE}nowhere" in missing and f"'{SITE}login'" in missing, missing + assert idn_shown["url"] == SITE.replace("127.0.0.1", "xn--fa-hia.localhost") + "other", idn_shown + assert idn_closed == idn_tab and after_idn == [start["id"]], (idn_closed, after_idn) assert context.metadata["browser_steps"]["browser_delete_cookies"]["deleted"] == 1 assert cookies == [] and storage == [{"origin": ORIGIN, "localStorage": []}], (cookies, storage) From d784e3b0a38800a17b6c2af3bdf8b0cf5131493f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 16:37:14 -0800 Subject: [PATCH 9/9] fix(browser): tab URLs keep a host IDNA 2008 refuses, mapping it by UTS 46 as Chromium does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit idna.encode applies IDNA 2008's code point rules, so a host Chromium takes, such as ☃.net, was left in Unicode and no tab matched it. Hosts are now mapped by UTS 46 without transitional processing and each label put in punycode. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/browser/tabs.py | 11 ++++++----- tst/unit/task_step/test_browser_steps.py | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/browser/tabs.py b/src/agent_env/task_step/task_steps/browser/tabs.py index 69f756e17..6e855ccbf 100644 --- a/src/agent_env/task_step/task_steps/browser/tabs.py +++ b/src/agent_env/task_step/task_steps/browser/tabs.py @@ -108,10 +108,10 @@ class BrowserActivateTabTaskStep(_TabAtUrlStep): def normalized(url: str) -> str: - """``url`` as the browser shows it: its scheme and host lowercase, a host in another script as punycode by UTS 46 - without transitional mapping, as Chromium spells it (``faß`` stays apart from ``fass``), its default port dropped, - and an empty path as ``/``; its query and fragment as they are. A URL with no host, such as ``about:blank``, is left - as it is.""" + """``url`` as the browser shows it: its scheme and host lowercase, a host in another script mapped by UTS 46 without + transitional mapping and its labels in punycode, as Chromium spells it (``faß`` stays apart from ``fass``, and ``☃`` + is kept though IDNA 2008 refuses it), its default port dropped, and an empty path as ``/``; its query and fragment as + they are. A URL with no host, such as ``about:blank``, is left as it is.""" parts = urlsplit(url) if not parts.netloc: return url @@ -120,7 +120,8 @@ def normalized(url: str) -> str: host = f"[{host}]" elif not host.isascii(): with contextlib.suppress(idna.IDNAError, UnicodeError): - host = idna.encode(host, uts46=True, transitional=False).decode("ascii") + host = ".".join(label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii") + for label in idna.uts46_remap(host, std3_rules=False, transitional=False).split(".")) port = parts.port netloc = host if port is None or port == _DEFAULT_PORTS.get(scheme) else f"{host}:{port}" if parts.username is not None: diff --git a/tst/unit/task_step/test_browser_steps.py b/tst/unit/task_step/test_browser_steps.py index b2c270cbc..eae52b143 100644 --- a/tst/unit/task_step/test_browser_steps.py +++ b/tst/unit/task_step/test_browser_steps.py @@ -196,6 +196,8 @@ def test_each_step_is_registered_and_round_trips(step): ("https://Bücher.test/a", "https://xn--bcher-kva.test/a"), ("https://faß.test/", "https://xn--fa-hia.test/"), ("https://fass.test/", "https://fass.test/"), + ("https://☃.net/", "https://xn--n3h.net/"), + ("https://ｂüｃher．test/", "https://xn--bcher-kva.test/"), ("about:blank", "about:blank"), ]) def test_a_url_is_normalized_as_the_browser_shows_it(url, shown):