feat(browser): browser task steps for cookies, local storage and tabs (FD-4943)

packages/agentenv-protocol/README.md

@@ -935,8 +935,9 @@ are the profile's template. The base class adds:
   `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. Also framework-only, for page
   drivers and agentenv's own steps alike:
   - `navigate {url}` and `history {direction}` (`back`, `forward` or `reload`), in the active tab;
-  - `tabs/list`, `tabs/open {url?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's own tab
-    ids. The active tab is the one a dialog is open on, else the visible one whose page has focus (a hidden
+  - `tabs/list`, `tabs/open {url?, background?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's
+    own tab ids. A tab opened with `background` opens behind the active tab, which stays active; otherwise it comes
+    to the front. The active tab is the one a dialog is open on, else the visible one whose page has focus (a hidden
     tab's page that had input still says it has focus), else the one agentenv last brought to the front.
     Headless, a tab opened at a URL opens blank and goes there once it's held to the window's size;
   - `dialog/answer {accept, text?}`, the agent's answer to the dialog open on the active tab;

packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py

@@ -120,17 +120,22 @@ async def tab_list(self) -> list[dict]:
         active = await self.active_tab()
         return [OpenTab(**tab, active=tab["id"] == active).model_dump() for tab in self.tabs.values()]
 
-    async def open_tab(self, url: Optional[str] = None) -> str:
-        """A new tab at ``url``, else blank, brought to the front; its id. Headless, it opens blank and goes to ``url``
-        once it's held to the window's size, so its page loads at that size."""
+    async def open_tab(self, url: Optional[str] = None, background: bool = False) -> str:
+        """A new tab at ``url``, else blank, brought to the front, or with ``background`` behind the active tab, which
+        stays active, brought back only if the new tab took its place; its id. Headless, it opens blank and goes to
+        ``url`` once it's held to the window's size, so its page loads at that size."""
+        active = await self.active_tab() if background else None
         held = self.viewport is not None and url
         start = "about:blank" if held else url or "about:blank"
-        target = (await self.devtools.call("Target.createTarget", {"url": start}))["targetId"]
+        target = (await self.devtools.call("Target.createTarget", {"url": start, "background": background}))["targetId"]
         if held:
             await asyncio.wait_for(self._pinned.setdefault(target, asyncio.Event()).wait(), PIN_TIMEOUT_S)
             session = next(s for s, t in self._sessions.items() if t == target)
             await self.devtools.call("Page.navigate", {"url": url}, session_id=session)
-        await self._bring_to_front(target)
+        if not background:
+            await self._bring_to_front(target)
+        elif active is not None and active in self.tabs and await self.active_tab() == target:
+            await self._bring_to_front(active)
         return target
 
     async def reopen_start_tabs(self, urls: list[str]) -> None:

packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py

@@ -376,8 +376,8 @@ async def browser_tabs_list(self) -> dict:
         return {"tabs": await self.control.tab_list()}
 
     @extension(BROWSER, op="tabs/open")
-    async def browser_tabs_open(self, url: Optional[str] = None) -> dict:
-        return {"tab": await self.control.open_tab(url)}
+    async def browser_tabs_open(self, url: Optional[str] = None, background: bool = False) -> dict:
+        return {"tab": await self.control.open_tab(url, background)}
 
     @extension(BROWSER, op="tabs/activate")
     async def browser_tabs_activate(self, tab: str) -> dict:

packages/agentenv-protocol/tests/test_website_browser.py

@@ -654,7 +654,8 @@ async def test_a_tab_it_opens_loads_once_held_to_the_window_comes_to_the_front_a
     devtools.send("Target.targetDestroyed", {"targetId": "b"})
 
     on_b = [name for name, _, session in devtools.calls if session == "s-b"]
-    assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank"}, None)]
+    assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank", "background": False},
+                                                                         None)]
     assert ({"url": "https://mail.test/b"}, "s-b") in _sent(devtools, "Page.navigate")
     assert on_b.index("Emulation.setDeviceMetricsOverride") < on_b.index("Page.navigate")
     assert [(tab["id"], tab["active"]) for tab in after_open] == [("a", False), ("b", True)]
@@ -663,6 +664,49 @@ async def test_a_tab_it_opens_loads_once_held_to_the_window_comes_to_the_front_a
     assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs] == [("b", "agentenv")]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("focus_moves", [False, True])
+async def test_a_tab_opened_in_the_background_opens_behind_the_active_one_which_stays_in_front(focus_moves):
+    focus = {"s-a": True}
+    devtools = Tabbed({"Runtime.evaluate": lambda params, session: {
+        "result": {"value": ["visible", focus.get(session, False)]}}})
+    control = await _tabs(devtools, "a")
+
+    opening = asyncio.ensure_future(control.open_tab("https://mail.test/b", background=True))
+    await asyncio.sleep(0)
+    devtools.send("Target.targetCreated", _page("b", "https://mail.test/b", "Mail b"))
+    devtools.send("Target.attachedToTarget", {"sessionId": "s-b", **_page("b")})
+    if focus_moves:
+        focus.update({"s-a": False, "s-b": True})
+    opened = await opening
+
+    assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank", "background": True},
+                                                                         None)]
+    assert ({"url": "https://mail.test/b"}, "s-b") in _sent(devtools, "Page.navigate")
+    assert ({}, "s-b") not in _sent(devtools, "Page.bringToFront")
+    assert _sent(devtools, "Target.activateTarget") == ([({"targetId": "a"}, None)] if focus_moves else [])
+    if not focus_moves:
+        assert [(tab["id"], tab["active"]) for tab in await control.tab_list()] == [("a", True), ("b", False)]
+
+
+@pytest.mark.asyncio
+async def test_a_background_open_leaves_a_tab_another_call_switched_to_meanwhile_in_front():
+    focus = {"s-a": True}
+    devtools = Tabbed({"Runtime.evaluate": lambda params, session: {
+        "result": {"value": ["visible", focus.get(session, False)]}}})
+    control = await _tabs(devtools, "a", "c")
+
+    opening = asyncio.ensure_future(control.open_tab("https://mail.test/b", background=True))
+    await asyncio.sleep(0)
+    focus.update({"s-a": False, "s-c": True})
+    devtools.send("Target.targetCreated", _page("b", "https://mail.test/b", "Mail b"))
+    devtools.send("Target.attachedToTarget", {"sessionId": "s-b", **_page("b")})
+    await opening
+
+    assert _sent(devtools, "Target.activateTarget") == []
+    assert [(tab["id"], tab["active"]) for tab in await control.tab_list()] == [("a", False), ("c", True), ("b", False)]
+
+
 @pytest.mark.asyncio
 async def test_a_hidden_tab_whose_page_still_says_it_has_focus_isnt_the_active_one():
     states = {"s-a": ["hidden", True], "s-b": ["visible", True]}

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "cryptography>=42",  # the local transfer CA
     "httpx>=0.27.0",
     "huggingface_hub>=1.0",  # agent-env run hf:// resolves and downloads a Hub bundle; checked on 1.0, 1.33 and 2.4
+    "idna>=3.0",  # a browser tab step matches a URL's host as Chromium spells it, by UTS 46
     "jsonschema>=4.20",  # checks an app's settings against its Settings' schema before a deploy
     "litellm>=1.96.0,<1.97",
     "mcp>=1.25,<2",  # same floor as env/gateway/requirements.txt and agentenv-protocol; 2.0.0 removed mcp.server.fastmcp, imported at module scope by env/gateway

src/agent_env/browsers/README.md

@@ -15,3 +15,16 @@ under `[settings]`: `start_urls`, opened at boot, the first tab active; `window`
 them (`env_params.app_settings = {chromium = {start_urls = [...]}}`). Its data is its profile, `~/.config/chromium`, which
 the image has none of, so each deploy and each reset starts with a clean one. Name it from another bundle as `chromium`,
 the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/browsers/chromium`.
+
+A task changes a browser's state with the steps in `agent_env.task_step.task_steps.browser`, each a call to one
+`browser/v1` operation on the browser in the env `env_id`, the one `browser` names when the env has several:
+`browser_set_cookies {cookies}`, in the shape `cookies/set` takes; `browser_delete_cookies {name?, domain?, path?}`,
+every cookie matching all of those given, a domain as the cookie stores it (`.shop.test` for a domain cookie), every
+cookie with none, the count deleted recorded; `browser_set_local_storage {origin, items}`, `items` a map of keys to
+values, the origin's other keys left as they are; `browser_clear_local_storage {origin?}`, every origin seen with none;
+`browser_open_tab {url, background?}`, in front, or with `background` behind the active tab, which stays active; and
+`browser_close_tab {url}` and `browser_activate_tab {url}`, on the one open tab at that URL as the browser shows it
+(scheme and host lowercase, no default port, `/` for an empty path), none or several failing the step with the tabs
+open listed. Preflight refuses a step on an env with no browser, none of the name given, or several and none named.
+`browser_connect_cdp` hands an agent a CDP address to the same browser. A whole state, Playwright's `storageState`
+included, loads through `load_artifact` or `data/add` instead.

src/agent_env/bundle/materialize.py

@@ -37,6 +37,7 @@
 from agent_env.store.base import NotFoundError
 from agent_env.store.routing import namespace_routing_enabled
 from agent_env.task.task import Task
+from agent_env.task_step.preflight import pinned
 from agent_env.task_step.registry import get_task_step_registry
 from agent_env.task_step.task_step import TaskStep
 from agent_env.utils.docker.build import DockerBuildError
@@ -450,8 +451,11 @@ def _preflight(plan: Plan, write: Write, task: Task, unwritten: set[tuple[str, s
     a step that deploys on a provider for the run's ``sandbox``, which overrides it."""
     outputs = {output.id for output in write.source.outputs}
     solver = plan.agent.steps.get(write.id, ()) if plan.agent is not None else ()
-    problems, unchecked = [], []
+    problems, unchecked, versions = [], [], {}
     for config, step in zip(write.source.config, task.steps):
+        before = dict(versions)
+        if (deployed := step.deploys_env()) is not None:
+            versions.setdefault(*deployed)
         overrides = {key: sandbox for key in _SANDBOX_FIELDS if sandbox is not None and hasattr(step, key)}
         if step.id in solver:
             overrides |= {"a2a_agent_id": plan.agent.agent_id, "a2a_agent_version": plan.agent.version}
@@ -466,7 +470,8 @@ def _preflight(plan: Plan, write: Write, task: Task, unwritten: set[tuple[str, s
             if type(step).preflight is not TaskStep.preflight:
                 unchecked.append(step)
             continue
-        problems.extend(step.preflight())
+        with pinned(before):
+            problems.extend(step.preflight())
     return problems, unchecked
 
 

src/agent_env/task/episode/rules.py

@@ -149,7 +149,9 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s
 
 UNAFFECTED = frozenset({
     # They name no agent.
-    "aggregate_verifiers", "apply_server_config", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks",
+    "aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage",
+    "browser_close_tab", "browser_delete_cookies", "browser_open_tab", "browser_set_cookies",
+    "browser_set_local_storage", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks",
     "deploy_env", "deploy_sandbox", "end_computer_recording", "start_computer_recording", "env_outcome_verifier",
     "modify_env_tool_access", "reset_env", "review",
     "run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "snapshot_computer_env", "sync_env_clock",

src/agent_env/task/task.py

@@ -23,6 +23,7 @@
 from agent_env.plugins._requirements import unmet_problems
 from agent_env.store.document_store import UnknownDocumentTypeError
 from agent_env.store.routing import run_scope
+from agent_env.task_step.preflight import pinned
 from agent_env.task.teardown import env_record_ids
 from agent_env.task_step.context import TaskStepContext, regraft_redacted_keys
 from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
@@ -405,9 +406,13 @@ def preflight(self) -> list[str]:
         Covers what needs the stores to answer; `_validate_dag` already covers ids and edges.
         """
         problems: list[str] = []
+        versions: dict[str, Optional[int]] = {}
         for step in self.steps:
             problems.extend(step.config_problems())
-            problems.extend(step.preflight())
+            with pinned(versions):
+                problems.extend(step.preflight())
+            if (deployed := step.deploys_env()) is not None:
+                versions.setdefault(*deployed)
         return problems
 
     def to_dict(self) -> dict:

src/agent_env/task_step/preflight.py

@@ -0,0 +1,33 @@
+"""What a step's preflight knows of the task it's in: the env version each deploy_env step before it pins, so a step
+reads the env its run will deploy rather than the latest one stored."""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Iterator, Mapping
+from contextvars import ContextVar
+from typing import Optional
+
+from agent_env.env.env import Env
+from agent_env.store.base import NotFoundError
+
+_PINNED: ContextVar[Mapping[str, Optional[int]]] = ContextVar("pinned_env_versions", default={})
+
+
+@contextlib.contextmanager
+def pinned(versions: Mapping[str, Optional[int]]) -> Iterator[None]:
+    """``versions``, by env id, the ones a preflight in the block reads its envs at."""
+    token = _PINNED.set(dict(versions))
+    try:
+        yield
+    finally:
+        _PINNED.reset(token)
+
+
+def stored_env(env_id: str) -> Optional[Env]:
+    """The env ``env_id`` at the version a deploy_env step before this one pins, else its latest; None when the store
+    has no such env, which the run reports."""
+    try:
+        return Env.get(env_id, _PINNED.get().get(env_id))
+    except NotFoundError:
+        return None

src/agent_env/task_step/registry.py

@@ -27,7 +27,17 @@ def get_task_step_registry() -> dict[str, type["TaskStep"]]:
 
 
 def _builtin_registry() -> dict[str, type["TaskStep"]]:
-    from agent_env.task_step.task_steps.browser_connect_cdp import BrowserConnectCdpTaskStep
+    from agent_env.task_step.task_steps.browser.connect_cdp import BrowserConnectCdpTaskStep
+    from agent_env.task_step.task_steps.browser.cookies import BrowserDeleteCookiesTaskStep, BrowserSetCookiesTaskStep
+    from agent_env.task_step.task_steps.browser.local_storage import (
+        BrowserClearLocalStorageTaskStep,
+        BrowserSetLocalStorageTaskStep,
+    )
+    from agent_env.task_step.task_steps.browser.tabs import (
+        BrowserActivateTabTaskStep,
+        BrowserCloseTabTaskStep,
+        BrowserOpenTabTaskStep,
+    )
     from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep
     from agent_env.task_step.task_steps.computer_recording import (
         EndComputerRecordingTaskStep,
@@ -89,6 +99,13 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
         _get_type(ReviewTaskStep): ReviewTaskStep,
         _get_type(BuildMcpCliTaskStep): BuildMcpCliTaskStep,
         _get_type(BrowserConnectCdpTaskStep): BrowserConnectCdpTaskStep,
+        _get_type(BrowserSetCookiesTaskStep): BrowserSetCookiesTaskStep,
+        _get_type(BrowserDeleteCookiesTaskStep): BrowserDeleteCookiesTaskStep,
+        _get_type(BrowserSetLocalStorageTaskStep): BrowserSetLocalStorageTaskStep,
+        _get_type(BrowserClearLocalStorageTaskStep): BrowserClearLocalStorageTaskStep,
+        _get_type(BrowserOpenTabTaskStep): BrowserOpenTabTaskStep,
+        _get_type(BrowserCloseTabTaskStep): BrowserCloseTabTaskStep,
+        _get_type(BrowserActivateTabTaskStep): BrowserActivateTabTaskStep,
         _get_type(CollectArtifactsTaskStep): CollectArtifactsTaskStep,
         _get_type(StartComputerRecordingTaskStep): StartComputerRecordingTaskStep,
         _get_type(EndComputerRecordingTaskStep): EndComputerRecordingTaskStep,

src/agent_env/task_step/task_step.py

@@ -122,6 +122,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext | None:
         """Execute this task step."""
         pass
 
+    def deploys_env(self) -> Optional[tuple[str, Optional[int]]]:
+        """The env id this step deploys and the version it pins, None for its latest, which the preflights of the
+        steps after it read that env at; None for a step that deploys no env."""
+        return None
+
     def preflight(self) -> list[str]:
         """Config problems detectable before a run, one message each; empty when fine.
 

src/agent_env/task_step/task_steps/browser/__init__.py

@@ -0,0 +1,2 @@
+"""Task steps on a deployed env's website browser, through browser/v1: its cookies, local storage and tabs set or changed,
+and a CDP address to it handed to an agent."""