diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index 15de1de9..6c86b568 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -835,7 +835,7 @@ def _merged_env(self, extra: dict | None = None) -> dict: def _base_agy_command( cli: str, prompt: str, resume: bool = False, model: str = None, output_format: str = None, log_file: str = None, - timeout: str = None, + timeout: str = None, add_dir: str = None, ) -> list: """Builds the non-interactive ``agy -p`` argv shared by the eval turn path and the setup-time MCP probe. @@ -854,6 +854,11 @@ def _base_agy_command( ``timeout`` maps to ``--print-timeout`` (e.g. "20m"). If omitted, defaults to agy's internal default (5 minutes). + + ``add_dir`` maps to ``--add-dir``, registering the scenario's + ``work_dir`` as an agy workspace. Required on top of the subprocess + cwd: unregistered, agy runs its shell and write tools in + ``/scratch`` and agent writes miss ``work_dir``. """ command = [cli, "-p", prompt, "--dangerously-skip-permissions"] if model: @@ -864,6 +869,8 @@ def _base_agy_command( command += ["--log-file", log_file] if timeout: command += ["--print-timeout", timeout] + if add_dir: + command += ["--add-dir", add_dir] if resume: command.append("--continue") return command @@ -913,8 +920,11 @@ def _run_agy_cli(self, cli_cmd: CLICommand, timeout_seconds=None): command = self._base_agy_command( self.agy_bin, cli_cmd.prompt, cli_cmd.resume, self.model, output_format="stream-json", log_file=self.cli_log_path, - timeout=self.timeout, + timeout=self.timeout, add_dir=cli_cmd.cwd, ) + # The fallback is deliberately not mirrored into --add-dir: fake_home + # holds harness internals (agy binary, settings, MCP config, session + # state), which the agent has no business editing as workspace files. cwd = cli_cmd.cwd if cli_cmd.cwd else self.fake_home result = self._execute_cli_command(command, env=env, cwd=cwd, timeout_seconds=timeout_seconds) diff --git a/evalbench/test/agy_cli_test.py b/evalbench/test/agy_cli_test.py index 80b236e8..264b2a00 100644 --- a/evalbench/test/agy_cli_test.py +++ b/evalbench/test/agy_cli_test.py @@ -291,6 +291,26 @@ def test_run_command_argv_shape_with_continue(mock_run, sandbox): ] +def test_run_command_argv_passes_work_dir_as_add_dir(mock_run, sandbox): + """A scenario ``work_dir`` must reach agy as ``--add-dir``, not just as the + subprocess cwd: without a registered workspace agy runs its shell and write + tools in ``/scratch``, so agent writes miss ``work_dir``.""" + generator = AgyCliGenerator({}) + cmd = generator.create_command( + cli="agy", prompt="hello world", cwd="/tmp/workspace" + ) + assert cmd.cwd == "/tmp/workspace" + generator._run_agy_cli(cmd) + + sent_argv = mock_run.call_args[0][0] + assert sent_argv == [ + generator.agy_bin, "-p", "hello world", + "--dangerously-skip-permissions", "--output-format", "stream-json", + "--log-file", generator.cli_log_path, "--add-dir", "/tmp/workspace", + ] + assert mock_run.call_args.kwargs["cwd"] == "/tmp/workspace" + + def test_init_raises_on_non_string_timeout(sandbox): """The generator should raise TypeError if timeout is not a string.""" with pytest.raises(TypeError, match="timeout must be a string"):