diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 5462c89e6..a6ecae16b 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1007,6 +1007,23 @@ class MyAgentConfig(AgentConfig): provider_token: WriteOnly[str | None] = None ``` +An agent that can turn its own tools off, such as its shell, file or web tools, takes `builtin_tools`, typed +`BuiltinTools` over a `Literal` of its tools' names, so its card's schema names them. A task selects them on +`deploy_agent`: `{"disable": "*"}` turns them all off, `{"disable": [...]}` the ones named, and `{"only": [...]}` keeps +the ones named; the agent applies `request.config.builtin_tools` to its runtime. The MCP tools AgentEnv gives it are not +its own and stay. A selection for an agent whose config has +no `builtin_tools`, or that names a tool its schema doesn't, is refused before the deploy sends it anything. + +```python +from typing import Literal + +from agentenv_protocol.a2a_agent import AgentConfig, BuiltinTools + + +class MyAgentConfig(AgentConfig): + builtin_tools: BuiltinTools[Literal["shell", "read_file", "web_fetch"]] | None = None +``` + `output_format` is author-owned in v1 rather than a field on the base `AgentConfig`. An agent that supports structured output must declare the field on its config subclass, apply it to its model or runtime, and return the value diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 67a1b904c..8ccde546d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -102,6 +102,7 @@ from .staging import STAGING_ENDPOINT, STAGING_V1_URI, StagingStore, staging_routes from .tasks.v1 import ( AgentConfig, + BuiltinTools, AgentRunResult, DataPart, FilePart, @@ -154,6 +155,7 @@ "AgentIdentity", "AgentRunResult", "AttributionProbeResponse", + "BuiltinTools", "BundleSkillRequest", "ChangelogIncrement", "ContextEventsTrajectoryRequest", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 90fd4bc45..a2eb5d3b4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -17,7 +17,7 @@ from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union from a2a.types import TaskStatusUpdateEvent -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator logger = logging.getLogger(__name__) @@ -127,6 +127,26 @@ class AgentConfig(BaseModel): timeout_seconds: int = 600 +ToolName = TypeVar("ToolName", bound=str) + + +class BuiltinTools(BaseModel, Generic[ToolName]): + """Which of an agent's own tools it may use, as a task selects them: ``disable`` turns off the ones it names, or + all of them with ``"*"``; ``only`` keeps the ones it names. An agent's config takes it as + ``builtin_tools: BuiltinTools[] | None``, so its card's agent-config schema names them.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + disable: Literal["*"] | list[ToolName] | None = None + only: list[ToolName] | None = None + + @model_validator(mode="after") + def _one_of(self) -> BuiltinTools: + if (self.disable is None) == (self.only is None): + raise ValueError("builtin_tools takes exactly one of disable and only") + return self + + ConfigT = TypeVar("ConfigT", bound=AgentConfig) diff --git a/packages/agentenv-protocol/tests/test_a2a_agent.py b/packages/agentenv-protocol/tests/test_a2a_agent.py index 422b4e334..5e82d5501 100644 --- a/packages/agentenv-protocol/tests/test_a2a_agent.py +++ b/packages/agentenv-protocol/tests/test_a2a_agent.py @@ -6,7 +6,7 @@ import time from dataclasses import FrozenInstanceError from functools import wraps -from typing import Any, Union +from typing import Any, Literal, Union from unittest.mock import MagicMock import httpx @@ -44,6 +44,7 @@ AgentEnvAgent, AgentIdentity, AttributionProbeResponse, + BuiltinTools, BundleSkillRequest, ContextObjectTrajectoryRequest, ContextTrajectoryRequest, @@ -1774,6 +1775,26 @@ async def run(self, request: TaskRequest[RuntimeConfig]) -> TaskResult: assert "default" not in schema["$defs"]["NestedConfig"]["properties"]["token"] +def test_builtin_tools_names_the_agents_own_tools_on_its_card_and_takes_one_selection() -> None: + class RuntimeConfig(AgentConfig): + builtin_tools: BuiltinTools[Literal["Bash", "Read", "WebFetch"]] | None = None + + @a2a_agent(identity=AgentIdentity(name="tooled", description="test", version="1"), config=RuntimeConfig) + class Agent(AgentEnvAgent): + async def run(self, request: TaskRequest[RuntimeConfig]) -> TaskResult: + return TaskResult.text("ok") + + with TestClient(Agent().create_app()) as client: + card = client.get("/.well-known/agent-card.json").json() + statuses = [_operation(client, card, AGENT_CONFIG_V1.uri, "set", {"builtin_tools": selection}).status_code + for selection in ({"disable": "*"}, {"only": ["Read"]}, {"disable": ["Glob"]}, + {"disable": "*", "only": ["Read"]}, {})] + + request = _extension(card, AGENT_CONFIG_V1.uri)["params"]["methods"]["set"]["request"] + assert "builtin_tools" in request["supported"] and '"enum": ["Bash", "Read", "WebFetch"]' in json.dumps(request) + assert statuses == [200, 200, 400, 400, 400] + + def test_validation_errors_do_not_echo_rejected_values() -> None: secret = "sk-live-SUPERSECRET" diff --git a/src/agent_env/a2a_agent/validator.py b/src/agent_env/a2a_agent/validator.py index 42ccac326..094f1dee6 100644 --- a/src/agent_env/a2a_agent/validator.py +++ b/src/agent_env/a2a_agent/validator.py @@ -185,6 +185,7 @@ async def _run( VerifyA2AAgentCardStep, VerifyA2AAgentConfigIdentityStep, VerifyA2AAgentMCPStep, + VerifyA2ABuiltinToolsStep, VerifyA2AModalitiesStep, VerifyA2APeerAgentsStep, VerifyA2ARoleStep, @@ -279,6 +280,7 @@ async def _run( VerifyA2AAgentCardStep(id=f"{task_id}-card", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyA2AAgentConfigIdentityStep(id=f"{task_id}-agent-config-identity", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyA2ARoleStep(id=f"{task_id}-role", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), + VerifyA2ABuiltinToolsStep(id=f"{task_id}-builtin-tools", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyA2ASystemPromptStep(id=f"{task_id}-system-prompt", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), PromptAgentTaskStep(id=protocol_prompt_id, version=None, prompt="Hello, respond with OK.", prompt_id=protocol_prompt_id, timeout_seconds=120, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyCoreA2AProtocolStep(id=f"{task_id}-protocol", version=None, a2a_agent_id=agent.id, prompt_id=protocol_prompt_id, depends_on=[TaskStepDependency(task_step_id=protocol_prompt_id)], fail_task_on_error=False), diff --git a/src/agent_env/agents/README.md b/src/agent_env/agents/README.md index 3ec6562c2..b2c29aeca 100644 --- a/src/agent_env/agents/README.md +++ b/src/agent_env/agents/README.md @@ -21,7 +21,12 @@ the folder changes. A run that deploys the agent registers that image when it's pulls it by its digest and builds it from this folder only when it can't, and a provider that runs images by name can run it. The CLI's version is the Dockerfile's `CLAUDE_CODE_VERSION`. The image gives the scripts the model runs -`node`, without npm or npx, and the Python packages `model-requirements.txt` pins. +`node`, without npm or npx, and the Python packages `model-requirements.txt` pins. A `deploy_agent` `builtin_tools` +selection picks the CLI's own tools: `{"disable": "*"}` and `{"only": [...]}` become its `--tools` list, a list to +turn off becomes `--disallowedTools`, which keeps the CLI's defaults for the rest, and any selection adds +`--strict-mcp-config`, so its MCP servers are only the task env's. The names are `ClaudeCodeTool`'s in +`claude_bridge.py`: every tool the pinned CLI has, though which it offers by default depends on the model. A test fails +when the Dockerfile's `CLAUDE_CODE_VERSION` moves, until `ClaudeCodeTool` and `TOOLS_CLI_VERSION` move with it. `install_agent` installs it too, into a container a task already runs or onto a `vm` sandbox's Linux host, with `install.sh` from its folder: Claude Code's native build for the target's CPU and libc, uv, and the agent's requirements diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index 082ee207b..7dc27625e 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -73,6 +73,7 @@ ) from claude_bridge import ( CLAUDE_STATE_DIR, + ClaudeCodeTools, SESSION_JSONL_DIR, WORKSPACE_DIR, kill_running_clis, @@ -134,6 +135,8 @@ class ClaudeCodeConfig(AgentConfig): effort: str | None = None max_turns: int | None = None max_thinking_tokens: int | None = None + # Which of the CLI's own tools it may use. With any selection, its MCP servers are only the ones agent-env gives it. + builtin_tools: ClaudeCodeTools | None = None def _rewrite_session_ids(jsonl_text: str, session_id: str) -> tuple[list[str], int]: @@ -488,6 +491,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon effort=cfg.effort, max_turns=cfg.max_turns, max_thinking_tokens=cfg.max_thinking_tokens, + builtin_tools=cfg.builtin_tools, event_callback=request.trajectory.append, ) except Exception as e: diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index c7163188e..cecd8286f 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -16,6 +16,9 @@ from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, model_validator from kit import in_thread_to_its_end @@ -52,6 +55,33 @@ def project_dir(workspace: Path) -> Path: MODEL_HEADERS_ENV = "LITELLM_EXTRA_HEADERS" +# The CLI version ClaudeCodeTool names the tools of; a test holds it to the Dockerfile's CLAUDE_CODE_VERSION. +TOOLS_CLI_VERSION = "2.1.280" +# The CLI's own tools, by the names its model sees. Which of them it offers by default depends on the model, and its MCP +# resource tools come only with MCP servers. +ClaudeCodeTool = Literal["Agent", "Bash", "CronCreate", "CronDelete", "CronList", "Edit", "EnterWorktree", + "ExitWorktree", "ListAgents", "ListMcpResourcesTool", "NotebookEdit", "Read", + "ReadMcpResourceDirTool", "ReadMcpResourceTool", "ReportFindings", "ScheduleWakeup", + "SendMessage", "Skill", "TaskCreate", "TaskGet", "TaskList", "TaskStop", "TaskUpdate", + "WebFetch", "WebSearch", "Workflow", "Write"] + + +class ClaudeCodeTools(BaseModel): + """The protocol's ``BuiltinTools[ClaudeCodeTool]``, which the image's protocol floor doesn't have yet: the same + fields and checks, so the card's schema is the same.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + disable: Literal["*"] | list[ClaudeCodeTool] | None = None + only: list[ClaudeCodeTool] | None = None + + @model_validator(mode="after") + def _one_of(self) -> ClaudeCodeTools: + if (self.disable is None) == (self.only is None): + raise ValueError("builtin_tools takes exactly one of disable and only") + return self + + def session_jsonl_path(session_id: str) -> Path: return SESSION_JSONL_DIR / f"{session_id}.jsonl" @@ -383,6 +413,7 @@ async def run_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, + builtin_tools: ClaudeCodeTools | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: """Run one turn, resuming the CLI session when a transient upstream failure cuts it short; a resume spends what @@ -395,7 +426,7 @@ async def run_claude( _spawn_claude, model=model, allowed_tools=allowed_tools, mcp_servers=mcp_servers, system_prompt=system_prompt, output_format=output_format, project_id=project_id, task_id=task_id, role=role, effort=effort, max_turns=max_turns, max_thinking_tokens=max_thinking_tokens, - event_callback=event_callback, + builtin_tools=builtin_tools, event_callback=event_callback, ) attempt_prompt, attempt_resume, attempt_session = prompt, resume, session_id trajectory: list[dict] = [] @@ -572,8 +603,11 @@ def _with_1m_window(model: str) -> str: def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: list[str] | None, mcp_servers: dict[str, dict] | None, system_prompt: str | None, output_format: dict | None, - role: str | None, effort: str | None, max_turns: int | None) -> list[str]: - """The CLI's argv. The prompt goes on stdin, since one argument over 128 KiB fails exec.""" + role: str | None, effort: str | None, max_turns: int | None, + builtin_tools: ClaudeCodeTools | None = None) -> list[str]: + """The CLI's argv. The prompt goes on stdin, since one argument over 128 KiB fails exec. ``builtin_tools`` selects + the CLI's own tools: ``{"disable": "*"}`` offers none, ``{"only": [...]}`` just those, and ``{"disable": [...]}`` + the CLI's defaults but those; with any selection, its MCP servers are only ``mcp_servers``.""" cmd = ["claude", "-p", "--output-format", "stream-json", "--verbose", "--dangerously-skip-permissions", "--setting-sources", "user"] cmd += ["--resume", session_id] if resume else ["--session-id", session_id] @@ -593,6 +627,12 @@ def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: cmd += ["--json-schema", json.dumps(output_format["schema"])] if allowed_tools: cmd += ["--allowedTools", ",".join(allowed_tools)] + if builtin_tools is not None: + if builtin_tools.only is not None or builtin_tools.disable == "*": + cmd += ["--tools", ",".join(builtin_tools.only or [])] + elif builtin_tools.disable: + cmd += ["--disallowedTools", ",".join(builtin_tools.disable)] + cmd.append("--strict-mcp-config") if mcp_servers: def entry(info: dict) -> dict: server: dict = {"type": "http", "url": info["url"]} @@ -622,11 +662,12 @@ async def _spawn_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, + builtin_tools: ClaudeCodeTools | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: cmd = _command(session_id, resume, model=model, allowed_tools=allowed_tools, mcp_servers=mcp_servers, system_prompt=system_prompt, output_format=output_format, role=role, effort=effort, - max_turns=max_turns) + max_turns=max_turns, builtin_tools=builtin_tools) logger.info(f"Running claude CLI (resume={resume}, session_id={session_id}, model={model})") WORKSPACE_DIR.mkdir(parents=True, exist_ok=True) proc = await _started(asyncio.create_subprocess_exec( diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 86d84c921..d7409bb28 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -139,9 +139,10 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "peer_agents": _peer_agents, "rubrics_verifier": _rubrics_verifier, **dict.fromkeys(( - "verify_a2a_agent_card", "verify_a2a_agent_config_identity", "verify_a2a_agent_mcp", "verify_a2a_core_protocol", - "verify_a2a_install", "verify_a2a_modalities", "verify_a2a_peer_agents", "verify_a2a_role", - "verify_a2a_skill_config", "verify_a2a_snapshot", "verify_a2a_system_prompt", "verify_a2a_trajectory", + "verify_a2a_agent_card", "verify_a2a_agent_config_identity", "verify_a2a_agent_mcp", "verify_a2a_builtin_tools", + "verify_a2a_core_protocol", "verify_a2a_install", "verify_a2a_modalities", "verify_a2a_peer_agents", + "verify_a2a_role", "verify_a2a_skill_config", "verify_a2a_snapshot", "verify_a2a_system_prompt", + "verify_a2a_trajectory", ), _validates_an_agent), } diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 83821a632..d343df2c2 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -69,6 +69,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_skill_config import VerifyA2ASkillConfigStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_snapshot import VerifyA2ASnapshotStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_trajectory import VerifyA2ATrajectoryStep + from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_builtin_tools import VerifyA2ABuiltinToolsStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_role import VerifyA2ARoleStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_system_prompt import VerifyA2ASystemPromptStep from agent_env.task_step.task_steps.add_skills import AddSkillsTaskStep @@ -135,6 +136,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(VerifyA2AModalitiesStep): VerifyA2AModalitiesStep, _get_type(VerifySandboxTaskStep): VerifySandboxTaskStep, _get_type(AggregateVerifiersTaskStep): AggregateVerifiersTaskStep, + _get_type(VerifyA2ABuiltinToolsStep): VerifyA2ABuiltinToolsStep, _get_type(VerifyA2ARoleStep): VerifyA2ARoleStep, _get_type(VerifyA2ASystemPromptStep): VerifyA2ASystemPromptStep, } diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py index b879cad97..a386ca654 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py @@ -1,6 +1,7 @@ from .verify_a2a_agent_card import VerifyA2AAgentCardStep from .verify_a2a_agent_config_identity import VerifyA2AAgentConfigIdentityStep from .verify_a2a_agent_mcp import VerifyA2AAgentMCPStep +from .verify_a2a_builtin_tools import VerifyA2ABuiltinToolsStep from .verify_a2a_core_protocol import VerifyCoreA2AProtocolStep from .verify_a2a_install import VerifyA2AInstallStep from .verify_a2a_modalities import VerifyA2AModalitiesStep @@ -15,6 +16,7 @@ "VerifyA2AAgentCardStep", "VerifyA2AAgentConfigIdentityStep", "VerifyA2AAgentMCPStep", + "VerifyA2ABuiltinToolsStep", "VerifyCoreA2AProtocolStep", "VerifyA2AInstallStep", "VerifyA2AModalitiesStep", diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py new file mode 100644 index 000000000..b3aa1d457 --- /dev/null +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py @@ -0,0 +1,117 @@ +"""Verify an agent whose agent-config takes ``builtin_tools`` honours it. A second agent is deployed with +``{"disable": "*"}`` and no envs, so it has no MCP tools and any tool call it makes is one of its own: asked to run a +shell command, it must make none. On that agent, a tool its card doesn't name is refused, and agent-config reads the +selection back. The validation agent itself is never limited, since the other checks need its tools. + +An agent whose agent-config doesn't take ``builtin_tools`` is recorded as not supporting it, which isn't a failure. + +Does NOT catch: a model that wouldn't have used a tool it has, which reads as honoured. +""" +from __future__ import annotations + +import logging +from typing import ClassVar, Optional + +import httpx + +from agent_env.a2a_agent import A2AAgent +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + +logger = logging.getLogger(__name__) + +AGENT_NAME = "builtin-tools-agent" +NO_SUCH_TOOL = "agentenv-validator-no-such-tool" +NO_TOOLS = "NO-TOOLS" +PROMPT = ("Run the shell command `uname -a` with a tool you have and reply with its exact output. If you have no tool " + f"that can run a shell command, reply with exactly {NO_TOOLS} and nothing else.") + + +class VerifyA2ABuiltinToolsStep(TaskStep): + type: ClassVar[str] = "verify_a2a_builtin_tools" + entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) + + def __init__( + self, + id: str, + version: Optional[int], + a2a_agent_id: str, + a2a_agent_version: Optional[int] = None, + depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True, + ): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.a2a_agent_id = a2a_agent_id + self.a2a_agent_version = a2a_agent_version + + def to_dict(self) -> dict: + base = super().to_dict() + base["a2a_agent_id"] = self.a2a_agent_id + base["a2a_agent_version"] = self.a2a_agent_version + return base + + @classmethod + def from_dict(cls, data: dict) -> VerifyA2ABuiltinToolsStep: + return cls( + **cls._base_from_dict(data), + a2a_agent_id=data["a2a_agent_id"], + a2a_agent_version=data.get("a2a_agent_version"), + ) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + main = next((a for a in context.deployed_agents), None) + if main is None: + raise RuntimeError("No deployed agent found in context") + params = (A2AAgent.find_extension(main.a2a_card or {}, A2AAgent.EXT_AGENT_CONFIG) or {}).get("params") or {} + methods = params.get("methods") or {} + if "builtin_tools" not in (methods.get("set") or {}).get("request", {}).get("supported", []): + self._record(context, {"supported": False, "reason": "agent-config takes no builtin_tools"}) + return context + + prompt_id = f"{self.id}-prompt" + try: + await DeployAgentTaskStep(id=f"{self.id}-deploy", version=None, a2a_agent_id=self.a2a_agent_id, + a2a_agent_version=self.a2a_agent_version, agent_name=AGENT_NAME, + builtin_tools={"disable": "*"}).execute(context) + limited = next(a for a in context.deployed_agents if a.agent_name == AGENT_NAME) + url = (limited.a2a_url or limited.api_url) + params.get("endpoint", "/ext/agent-config") + async with httpx.AsyncClient(timeout=30.0) as client: + refused = (await client.post(url, json={"builtin_tools": {"only": [NO_SUCH_TOOL]}})).status_code + config = (((await client.get(url)).json() or {}).get("config") or {}) if "get" in methods else None + await PromptAgentTaskStep(id=prompt_id, version=None, prompt=PROMPT, prompt_id=prompt_id, + agent_name=AGENT_NAME, timeout_seconds=180).execute(context) + except Exception as e: + self._record(context, {"supported": True, "passed": False, "reason": f"{type(e).__name__}: {e}"[:500]}) + return context + + response = next(r for r in reversed(context.prompt_responses) if r.prompt_id == prompt_id) + count = response.tool_call_count + checks = { + "unknown_tool_refused": refused == 400, + "read_back": None if config is None else _set(config.get("builtin_tools")) == {"disable": "*"}, + "no_tool_calls": count == 0 if count is not None else response.response.strip() == NO_TOOLS, + } + passed = all(check is not False for check in checks.values()) + if not passed: + logger.warning("builtin_tools not honoured by '%s': %s", self.a2a_agent_id, checks) + self._record(context, {"supported": True, "passed": passed, **checks, "tool_call_count": count, + "response": response.response[:500]}) + return context + + def _record(self, context: TaskStepContext, validated: dict) -> None: + # As the role check: the context's record first, then the agent's metadata, whose CAS can lose to the other + # validator steps running beside this one. + context.metadata.setdefault("verifications", {})["a2a_builtin_tools"] = validated + try: + fresh = A2AAgent.get(self.a2a_agent_id, self.a2a_agent_version) + fresh.update_metadata({**fresh.metadata, "validated_builtin_tools_via_agent_config": validated}) + except Exception as e: + logger.warning("Failed to persist validated_builtin_tools_via_agent_config (CAS conflict?): %s", e) + + +def _set(selection: dict | None) -> dict: + """A selection as read back, without the fields it leaves unset, which an agent may return as null.""" + return {key: value for key, value in (selection or {}).items() if value is not None} diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index 8e763e702..cbacdc804 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -8,11 +8,15 @@ import httpx from agentenv_protocol.a2a_agent import ( + AGENT_CONFIG_V1, + BuiltinTools, NamespaceChangelogEnableResponse, ObjectChangelogApplyResponse, ObjectSnapshotLoadResponse, card_request_accepts, ) +from jsonschema import Draft202012Validator +from jsonschema.exceptions import best_match from agent_env.a2a_agent import protocol from agent_env.a2a_agent.object_transfer import ( @@ -146,6 +150,21 @@ def _skill_fields(skill: dict) -> dict: } + +def _check_builtin_tools(agent_config: dict | None, agent_name: str, selection: dict) -> None: + """Refuse a selection of the agent's own tools that its card's agent-config extension can't take: it has no + ``builtin_tools``, or its schema doesn't name a tool the selection does.""" + request = ((agent_config or {}).get("params") or {}).get("methods", {}).get("set", {}).get("request", {}) + if "builtin_tools" not in request.get("supported", []): + raise RuntimeError(f"Agent '{agent_name}' cannot have its own tools selected: its card's " + f"{AGENT_CONFIG_V1.uri} takes no 'builtin_tools'") + schema = request.get("schema") or {} + field = (schema.get("properties") or {}).get("builtin_tools") + if field is not None: + error = best_match(Draft202012Validator({**field, "$defs": schema.get("$defs", {})}).iter_errors(selection)) + if error is not None: + raise RuntimeError(f"Agent '{agent_name}' can't take builtin_tools {selection}: {error.message}") + class DeployAgentTaskStep(TaskStep): type: ClassVar[str] = "deploy_agent" entity_refs = ( @@ -186,6 +205,7 @@ def __init__( agent_snapshot_target_context_id: Optional[str] = None, network_policy: Optional[dict] = None, role: Optional[str] = None, + builtin_tools: Optional[dict] = None, enable_agent_changelog: bool = False, agent_changelog_toolcall_position_exclusive: Optional[int] = None, sandbox_name: Optional[str] = None, @@ -217,6 +237,8 @@ def __init__( self.metadata = metadata or {} self.network_policy = NetworkPolicy.from_dict(network_policy).to_dict() if network_policy else None self.role = role or None + self.builtin_tools = (None if builtin_tools is None + else BuiltinTools[str].model_validate(builtin_tools).model_dump(exclude_none=True)) self.enable_agent_changelog = enable_agent_changelog self.agent_changelog_object_url = agent_changelog_object_url self.agent_changelog_toolcall_position_exclusive = agent_changelog_toolcall_position_exclusive @@ -254,6 +276,7 @@ def to_dict(self) -> dict: base["metadata"] = self.metadata base["network_policy"] = self.network_policy base["role"] = self.role + base["builtin_tools"] = self.builtin_tools base["enable_agent_changelog"] = self.enable_agent_changelog base["agent_changelog_object_url"] = self.agent_changelog_object_url base["agent_changelog_toolcall_position_exclusive"] = self.agent_changelog_toolcall_position_exclusive @@ -286,6 +309,7 @@ def from_dict(cls, data: dict) -> DeployAgentTaskStep: metadata=dict(data.get("metadata") or {}), network_policy=data.get("network_policy"), role=data.get("role"), + builtin_tools=data.get("builtin_tools"), enable_agent_changelog=data.get("enable_agent_changelog", False), agent_changelog_object_url=data.get("agent_changelog_object_url"), agent_changelog_toolcall_position_exclusive=data.get("agent_changelog_toolcall_position_exclusive"), @@ -417,6 +441,11 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas identity_payload["description"] = self.agent_description if self.role is not None: identity_payload["role"] = self.role + if self.builtin_tools is not None: + # Before the first request to the agent, so an agent that would keep its own tools never runs the task. + _check_builtin_tools(A2AAgent.find_extension(card, A2AAgent.EXT_AGENT_CONFIG), self.agent_name, + self.builtin_tools) + identity_payload["builtin_tools"] = self.builtin_tools identity = A2AAgent.negotiate_agent_config(card, identity_payload) if self.role is not None: carried_by = [via for via, carried in ( diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 145fd5e2d..c11fb716b 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -486,6 +486,8 @@ async def _converse( "output_format": self.output_format, "timeout_seconds": self.timeout_seconds, "task_id": context.metadata.get("task_id"), "agentenv_tools": self.agentenv_tools, + # Again each turn, so an agent that restarted since deploy_agent sent it doesn't get its own tools back. + "builtin_tools": card.get("builtin_tools"), } desired_config = {k: v for k, v in desired_config.items() if v is not None} model_params = get_config().get_model_params(self.model_params) diff --git a/tst/integration/cli/run_bundle_claude_code_test.py b/tst/integration/cli/run_bundle_claude_code_test.py index 019d0728b..c4f59b81e 100644 --- a/tst/integration/cli/run_bundle_claude_code_test.py +++ b/tst/integration/cli/run_bundle_claude_code_test.py @@ -6,6 +6,7 @@ import json import logging import uuid +from typing import get_args import pytest from click.testing import CliRunner @@ -26,6 +27,7 @@ from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from tst.unit.agents.claude_code.agent import claude_bridge from tst.util.fake_anthropic import FakeAnthropic, answer, delegate_then_answer, tool_then_answer pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -156,24 +158,41 @@ def test_the_alias_claude_code_names_the_shipped_agent_until_the_store_has_an_ag assert named == [shipped, "claude-code"] -def test_the_shipped_claude_code_calls_the_tools_of_the_env_it_is_deployed_with(state, endpoint): +@pytest.mark.parametrize("builtin_tools, kept", [ + (None, lambda defaults: defaults), + ({"disable": "*"}, lambda defaults: set()), + ({"disable": ["WebFetch"]}, lambda defaults: defaults - {"WebFetch"}), + ({"only": ["Read", "TaskList"]}, lambda defaults: {"Read", "TaskList"}), +], ids=["own-tools", "none-of-its-own", "all-but-webfetch", "only-two"]) +def test_the_shipped_claude_code_calls_the_tools_of_the_env_it_is_deployed_with(state, endpoint, builtin_tools, kept): + """The CLI offers only tools claude-code's card names, and a selection offers what it leaves of them: ``only`` + the tools it names, a list to turn off the CLI's defaults but those.""" endpoint.respond = tool_then_answer("__echo", {"text": "ping-from-tool"}, prefix="the tool said: ") - root = _bundle(state / "tools", "echo", [ - {"id": "env", "type": "deploy_env", "env_id": ECHO_ENV, "sandbox_type": "local"}, - *_ask("Call the echo tool with the text ping-from-tool.", "ping-from-tool", env_ids=[ECHO_ENV]), - ]) - root.joinpath("tasks/echo.json").write_text(json.dumps( - [{**step, "depends_on": ["env"]} if step["id"] == "agent" else step - for step in json.loads(root.joinpath("tasks/echo.json").read_text())])) - - result = _run(root) - offered = sorted({tool["name"] for call in endpoint.messages() for tool in call.body.get("tools") or []}) - seen = (result.output, offered, endpoint.tool_results()) - assert result.exit_code == 0 and "tasks/echo.json v1: passed" in result.output, seen - # The fake never repeats the prompt, so the text reached the answer through the tool. - assert any(name.endswith("__echo") for name in offered), seen - assert ["ping-from-tool" in result for result in endpoint.tool_results()] == [True], seen + def run(name, selection): + root = _bundle(state / name, "echo", [ + {"id": "env", "type": "deploy_env", "env_id": ECHO_ENV, "sandbox_type": "local"}, + *_ask("Call the echo tool with the text ping-from-tool.", "ping-from-tool", env_ids=[ECHO_ENV]), + ]) + selected = {} if selection is None else {"builtin_tools": selection} + root.joinpath("tasks/echo.json").write_text(json.dumps( + [{**step, "depends_on": ["env"], **selected} if step["id"] == "agent" else step + for step in json.loads(root.joinpath("tasks/echo.json").read_text())])) + before = len(endpoint.messages()) + result = _run(root) + offered = {tool["name"] for call in endpoint.messages()[before:] for tool in call.body.get("tools") or []} + seen = (result.output, sorted(offered), endpoint.tool_results()) + assert result.exit_code == 0 and "tasks/echo.json v1: passed" in result.output, seen + # The fake never repeats the prompt, so the text reached the answer through the tool. + assert any(name.endswith("__echo") for name in offered), seen + return {name for name in offered if not name.startswith("mcp__")} + + defaults = run("tools", None) + offered = defaults if builtin_tools is None else run("selected", builtin_tools) + + assert defaults <= set(get_args(claude_bridge.ClaudeCodeTool)), sorted(defaults) + assert offered == kept(defaults), sorted(offered) + assert all("ping-from-tool" in result for result in endpoint.tool_results()) async def _send_project_id(step, context): diff --git a/tst/unit/agents/claude_code/test_card_contract.py b/tst/unit/agents/claude_code/test_card_contract.py index 510c8965b..0ace58b4f 100644 --- a/tst/unit/agents/claude_code/test_card_contract.py +++ b/tst/unit/agents/claude_code/test_card_contract.py @@ -67,7 +67,7 @@ def test_extensions_and_their_routes(app): def test_the_config_takes_every_field_prompt_agent_sends_that_the_cli_has(app): config = _extensions(app)["urn:agentenv:agent-config/v1"]["params"]["methods"] assert config["set"]["request"]["supported"] == [ - "allowed_tools", "description", "effort", "max_thinking_tokens", "max_turns", "model", "name", + "allowed_tools", "builtin_tools", "description", "effort", "max_thinking_tokens", "max_turns", "model", "name", "output_format", "project_id", "role", "system_prompt", "task_id", "timeout_seconds", ] assert set(config["set"]["request"]["schema"]["properties"]) == set(config["set"]["request"]["supported"]) diff --git a/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py new file mode 100644 index 000000000..b5d324052 --- /dev/null +++ b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py @@ -0,0 +1,61 @@ +"""A selection of the CLI's own tools: ``"*"`` and ``only`` name the tools the CLI may offer, a list to turn off leaves +the CLI's defaults but those, and with any selection its MCP servers are only agent-env's; with none, the CLI keeps +its own. A tool the CLI doesn't have is refused. The selection's type has the protocol's schema, and its tool names +are those of the CLI version the Dockerfile pins.""" + +import re +from typing import Any + +import pytest +from agentenv_protocol.a2a_agent import BuiltinTools + +from tst.unit.agents.claude_code.agent import AGENT_DIR, a2a_server, claude_bridge + + +def _command(selection: dict | None) -> list[str]: + return claude_bridge._command( + "sess-1", False, model=None, allowed_tools=None, mcp_servers=None, system_prompt=None, output_format=None, + role=None, effort=None, max_turns=None, + builtin_tools=a2a_server.ClaudeCodeConfig(builtin_tools=selection).builtin_tools) + + +def test_no_selection_leaves_the_clis_tools_and_mcp_servers_alone(): + command = _command(None) + + assert not {"--tools", "--disallowedTools", "--strict-mcp-config"} & set(command) + + +@pytest.mark.parametrize("selection, flags", [ + ({"disable": "*"}, ["--tools", ""]), + ({"only": ["Read", "Agent"]}, ["--tools", "Read,Agent"]), + ({"only": []}, ["--tools", ""]), + ({"disable": ["WebFetch", "WebSearch"]}, ["--disallowedTools", "WebFetch,WebSearch"]), +]) +def test_a_selection_is_the_clis_tool_flags_with_only_agent_envs_mcp_servers(selection, flags): + command = _command(selection) + at = command.index(flags[0]) + + assert command[at:at + 2] == flags and "--strict-mcp-config" in command + assert not ({"--tools", "--disallowedTools"} - {flags[0]}) & set(command) + + +def test_a_tool_the_cli_does_not_have_is_refused(): + with pytest.raises(ValueError, match="Glob"): + a2a_server.ClaudeCodeConfig(builtin_tools={"only": ["Glob"]}) + + +def test_the_selection_has_the_protocols_schema(): + def untitled(schema: Any) -> Any: + if isinstance(schema, dict): + return {k: untitled(v) for k, v in schema.items() if k not in ("title", "description")} + return [untitled(v) for v in schema] if isinstance(schema, list) else schema + + assert untitled(claude_bridge.ClaudeCodeTools.model_json_schema()) == untitled( + BuiltinTools[claude_bridge.ClaudeCodeTool].model_json_schema()) + + +def test_the_tool_names_are_those_of_the_cli_version_the_dockerfile_pins(): + pinned = re.search(r"^ARG CLAUDE_CODE_VERSION=(\S+)$", (AGENT_DIR / "Dockerfile").read_text(), re.M).group(1) + + assert claude_bridge.TOOLS_CLI_VERSION == pinned, ( + "the CLI version changed: record the tools the new CLI offers in ClaudeCodeTool, then TOOLS_CLI_VERSION") diff --git a/tst/unit/task_step/test_deploy_agent_role_delivery.py b/tst/unit/task_step/test_deploy_agent_role_delivery.py index d9e0707ae..e642fd0de 100644 --- a/tst/unit/task_step/test_deploy_agent_role_delivery.py +++ b/tst/unit/task_step/test_deploy_agent_role_delivery.py @@ -1,13 +1,16 @@ """deploy_agent delivers the step's role to the agent, or fails the step before any request reaches it; it never -records a role the agent did not get.""" +records a role the agent did not get. Its selection of the agent's own tools goes the same way: refused before any +request by an agent whose agent-config doesn't take it or names no such tool.""" import json import logging from types import SimpleNamespace +from typing import Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from agentenv_protocol.a2a_agent import AgentConfig, BuiltinTools from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.env.env import DeployedGatewayEnv @@ -46,6 +49,14 @@ def _card(*extensions: dict) -> dict: ROLELESS_CONFIG = _agent_config(["name", "description"]) +class _ToolConfig(AgentConfig): + builtin_tools: BuiltinTools[Literal["Bash", "Read"]] | None = None + + +TOOLS_CONFIG = _agent_config(["name", "description", "builtin_tools"]) +TOOLS_CONFIG["params"]["methods"]["set"]["request"]["schema"] = _ToolConfig.model_json_schema() + + @pytest.mark.asyncio async def test_a_card_with_both_paths_gets_the_role_on_both_and_the_agent_records_its_envs(wiring, caplog): with caplog.at_level(logging.WARNING, logger=_LOGGER): @@ -186,6 +197,36 @@ async def test_an_explicit_cli_role_env_var_is_kept(wiring): assert wiring.env_vars == {ROLE_ENV_VAR: "cli"} +@pytest.mark.asyncio +async def test_a_selection_of_the_agents_own_tools_is_sent_once_with_its_identity(wiring): + context = await wiring.deploy(_card(HEADERS_MCP, TOOLS_CONFIG), role=None, builtin_tools={"disable": "*"}) + + assert wiring.bodies("/ext/agent-config") == [{"name": "solver", "builtin_tools": {"disable": "*"}}] + assert len(context.deployed_agents) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("card, refusal", [ + (_card(HEADERS_MCP, ROLE_CONFIG), "takes no 'builtin_tools'"), + (_card(HEADERS_MCP), "takes no 'builtin_tools'"), + (_card(HEADERS_MCP, TOOLS_CONFIG), "'Glob' is not one of"), +], ids=["config-without-it", "no-agent-config", "unknown-tool"]) +async def test_a_selection_the_agent_cannot_take_fails_before_any_request(wiring, card, refusal): + with pytest.raises(RuntimeError, match=refusal): + await wiring.deploy(card, role=None, builtin_tools={"only": ["Read", "Glob"]}) + + assert wiring.requests == [] + assert wiring.agent.close.await_count == 1 + + +def test_a_selection_takes_exactly_one_of_disable_and_only(): + for selection in ({}, {"disable": "*", "only": ["Read"]}, {"enable": ["Read"]}): + with pytest.raises(ValueError): + DeployAgentTaskStep(id="ag", version=None, builtin_tools=selection) + step = DeployAgentTaskStep(id="ag", version=None, builtin_tools={"disable": ["Bash"]}) + assert DeployAgentTaskStep.from_dict({**step.to_dict(), "version": 1}).builtin_tools == {"disable": ["Bash"]} + + class _Wiring: """One deploy_agent run: the requests the agent received, the agent double, and the step's context.""" diff --git a/tst/unit/task_step/test_prompt_agent_builtin_tools.py b/tst/unit/task_step/test_prompt_agent_builtin_tools.py new file mode 100644 index 000000000..4066f94a3 --- /dev/null +++ b/tst/unit/task_step/test_prompt_agent_builtin_tools.py @@ -0,0 +1,67 @@ +"""prompt_agent sends the agent's selection of its own tools again on every turn, as deploy_agent recorded it on the +agent's card, so an agent that restarted since its deploy doesn't get its own tools back; an agent deployed without +one is sent none.""" + +from unittest.mock import MagicMock + +import pytest +from a2a.types import TaskState + +from agent_env.a2a_agent import A2AAgent +from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.task_steps import prompt_agent as pa_mod +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + +SUPPORTED = ["model", "builtin_tools"] + + +@pytest.fixture +def posted(monkeypatch): + """Mocks the turn's HTTP and store boundary, and collects each agent config posted.""" + cfg = MagicMock() + cfg.get_model_params.return_value = {} + cfg.get_model_for_role.return_value = None + monkeypatch.setattr(pa_mod, "get_config", lambda: cfg) + for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): + monkeypatch.setattr(pa_mod.conversation_store, fn, lambda *a, **kw: None) + configs = [] + + async def post_agent_config(url, fields): + configs.append(fields) + + async def send(url, parts, message_id, context_id, timeout): + return f"task-{message_id}", None + + async def poll(url, task_id, timeout, interval, **_): + return {"status": {"state": TaskState.completed, "message": {"parts": [{"kind": "text", "text": "done"}]}}} + + monkeypatch.setattr(pa_mod.protocol, "post_agent_config", post_agent_config) + monkeypatch.setattr(pa_mod.protocol, "send_a2a_message", send) + monkeypatch.setattr(pa_mod.protocol, "poll_a2a_task", poll) + return configs + + +def _context(**stamped) -> TaskStepContext: + card = {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", "methods": {"set": {"request": {"supported": SUPPORTED}}}}}]}, **stamped} + context = TaskStepContext(instance_id="ti-1") + context.deployed_agents.append(DeployedAgent(agent_name="solver", api_url="http://solver", a2a_url="http://solver", + a2a_card=card)) + return context + + +@pytest.mark.asyncio +async def test_every_turn_sends_the_selection_deploy_agent_recorded(posted): + context = _context(builtin_tools={"disable": "*"}) + + for turn in ("one", "two"): + await PromptAgentTaskStep(id=turn, version=None, prompt=turn, agent_name="solver").execute(context) + + assert [fields.get("builtin_tools") for fields in posted] == [{"disable": "*"}, {"disable": "*"}] + + +@pytest.mark.asyncio +async def test_an_agent_deployed_without_a_selection_is_sent_none(posted): + await PromptAgentTaskStep(id="one", version=None, prompt="one", agent_name="solver").execute(_context()) + + assert all("builtin_tools" not in fields for fields in posted) diff --git a/tst/unit/task_step/test_verify_a2a_builtin_tools.py b/tst/unit/task_step/test_verify_a2a_builtin_tools.py new file mode 100644 index 000000000..de3c59987 --- /dev/null +++ b/tst/unit/task_step/test_verify_a2a_builtin_tools.py @@ -0,0 +1,116 @@ +"""The validator's builtin_tools check: an agent whose agent-config doesn't take the field is recorded as not +supporting it and nothing is deployed; otherwise a second agent, deployed with {"disable": "*"} and no envs, must +refuse an unknown tool, read the selection back, and make no tool call when asked to run a shell command, or say it +has no tool when it reports no count.""" + +import json + +import httpx +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.a2a_agent_validator import verify_a2a_builtin_tools as module +from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_builtin_tools import NO_SUCH_TOOL, VerifyA2ABuiltinToolsStep + + +def _card(*supported: str) -> dict: + return {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", + "methods": {"set": {"request": {"supported": list(supported)}}, "get": {"method": "GET"}}}}]}} + + +class _Agent: + """The limited agent's agent-config: a selection it was deployed with, and the 400 an unknown tool gets.""" + + def __init__(self): + self.config, self.deploys = {}, [] + + def handle(self, request: httpx.Request) -> httpx.Response: + if request.method == "GET": + # As the protocol's readback writes a selection: with the field it leaves unset as null. + return httpx.Response(200, json={"config": {**self.config, "builtin_tools": { + "disable": None, "only": None, **self.config.get("builtin_tools", {})}}}) + body = json.loads(request.content) + if NO_SUCH_TOOL in str(body): + return httpx.Response(400, json={"detail": "not one of the agent's tools"}) + self.config.update(body) + return httpx.Response(200, json={}) + + +@pytest.fixture +def wired(monkeypatch): + def wire(tool_call_count, reply="Linux box 6.8.0"): + agent = _Agent() + real_client = httpx.AsyncClient + monkeypatch.setattr(module.httpx, "AsyncClient", + lambda **kw: real_client(transport=httpx.MockTransport(agent.handle), **kw)) + monkeypatch.setattr(A2AAgent, "get", lambda *a, **kw: type("Stored", (), { + "metadata": {}, "update_metadata": lambda self, m: None})()) + + async def deploy(step, context): + agent.deploys.append(step.builtin_tools) + agent.config["builtin_tools"] = step.builtin_tools + context.deployed_agents.append(DeployedAgent(agent_name=step.agent_name, api_url="http://limited", + a2a_url="http://limited", a2a_card={})) + return context + + async def prompt(step, context): + context.prompt_responses.append(PromptResponse(prompt_id=step.prompt_id, response=reply, + tool_call_count=tool_call_count)) + return context + + monkeypatch.setattr(module.DeployAgentTaskStep, "execute", deploy) + monkeypatch.setattr(module.PromptAgentTaskStep, "execute", prompt) + return agent + + return wire + + +def _context(card: dict) -> TaskStepContext: + context = TaskStepContext(instance_id="ti-1") + context.deployed_agents.append(DeployedAgent(agent_name="solver", api_url="http://main", a2a_url="http://main", + a2a_card=card)) + return context + + +async def _verify(card: dict) -> dict: + step = VerifyA2ABuiltinToolsStep(id="builtin-tools", version=None, a2a_agent_id="agent-1") + return (await step.execute(_context(card))).metadata["verifications"]["a2a_builtin_tools"] + + +@pytest.mark.asyncio +async def test_an_agent_whose_config_takes_no_builtin_tools_is_recorded_as_not_supporting_it(wired): + agent = wired(0) + + assert await _verify(_card("name", "role")) == {"supported": False, + "reason": "agent-config takes no builtin_tools"} + assert agent.deploys == [] + + +@pytest.mark.asyncio +async def test_an_agent_that_honours_it_passes(wired): + agent = wired(0) + + verified = await _verify(_card("name", "builtin_tools")) + + assert agent.deploys == [{"disable": "*"}] + assert {k: verified[k] for k in ("supported", "passed", "unknown_tool_refused", "read_back", "no_tool_calls")} == { + "supported": True, "passed": True, "unknown_tool_refused": True, "read_back": True, "no_tool_calls": True} + + +@pytest.mark.asyncio +async def test_an_agent_whose_limited_turn_calls_a_tool_fails(wired): + wired(2) + + verified = await _verify(_card("name", "builtin_tools")) + + assert (verified["passed"], verified["no_tool_calls"], verified["tool_call_count"]) == (False, False, 2) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reply, passed", [("NO-TOOLS", True), ("Linux box 6.8.0", False)]) +async def test_an_agent_that_reports_no_count_must_say_it_has_no_tool(wired, reply, passed): + wired(None, reply) + + assert (await _verify(_card("name", "builtin_tools")))["passed"] is passed