From 1134e526954f184d3c96850e3df32d90ef275cf9 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 31 Aug 2026 05:11:59 +0000 Subject: [PATCH 1/3] fix(agy): register work_dir as an agy workspace via --add-dir Setting the subprocess cwd made agy resolve reads against work_dir, but its shell and write tools still ran in /scratch, so agent-created files landed outside the scenario directory. agy tracks the read workspace and the tool execution directory separately; only --add-dir sets the latter. The flag is passed only when a scenario declares work_dir, leaving the MCP startup probe and work_dir-less scenarios unchanged. --- evalbench/generators/models/agy_cli.py | 11 +++++++++-- evalbench/test/agy_cli_test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index 37eee849..d692e5e6 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -833,7 +833,7 @@ def _merged_env(self, extra: dict | None = None) -> dict: def _base_agy_command( cli: str, prompt: str, resume: bool = False, model: str = None, output_format: str = None, log_file: str = None, - timeout: str = None, + timeout: str = None, add_dir: str = None, ) -> list: """Builds the non-interactive ``agy -p`` argv shared by the eval turn path and the setup-time MCP probe. @@ -852,6 +852,11 @@ def _base_agy_command( ``timeout`` maps to ``--print-timeout`` (e.g. "20m"). If omitted, defaults to agy's internal default (5 minutes). + + ``add_dir`` maps to ``--add-dir``, registering the scenario's + ``work_dir`` as an agy workspace. Required on top of the subprocess + cwd: unregistered, agy runs its shell and write tools in + ``/scratch`` and agent writes miss ``work_dir``. """ command = [cli, "-p", prompt, "--dangerously-skip-permissions"] if model: @@ -862,6 +867,8 @@ def _base_agy_command( command += ["--log-file", log_file] if timeout: command += ["--print-timeout", timeout] + if add_dir: + command += ["--add-dir", add_dir] if resume: command.append("--continue") return command @@ -901,7 +908,7 @@ def _run_agy_cli(self, cli_cmd: CLICommand): command = self._base_agy_command( self.agy_bin, cli_cmd.prompt, cli_cmd.resume, self.model, output_format="stream-json", log_file=self.cli_log_path, - timeout=self.timeout, + timeout=self.timeout, add_dir=cli_cmd.cwd, ) cwd = cli_cmd.cwd if cli_cmd.cwd else self.fake_home result = self._execute_cli_command(command, env=env, cwd=cwd) diff --git a/evalbench/test/agy_cli_test.py b/evalbench/test/agy_cli_test.py index 80b236e8..d602adc4 100644 --- a/evalbench/test/agy_cli_test.py +++ b/evalbench/test/agy_cli_test.py @@ -291,6 +291,32 @@ def test_run_command_argv_shape_with_continue(mock_run, sandbox): ] +def test_run_command_argv_passes_work_dir_as_add_dir(mock_run, sandbox): + """A scenario ``work_dir`` must reach agy as ``--add-dir``, not just as the + subprocess cwd: without a registered workspace agy runs its shell and write + tools in ``/scratch``, so agent writes miss ``work_dir``.""" + generator = AgyCliGenerator({}) + cmd = CLICommand(cli="agy", prompt="hello world", cwd="/tmp/workspace") + generator._run_agy_cli(cmd) + + sent_argv = mock_run.call_args[0][0] + assert sent_argv == [ + generator.agy_bin, "-p", "hello world", + "--dangerously-skip-permissions", "--output-format", "stream-json", + "--log-file", generator.cli_log_path, "--add-dir", "/tmp/workspace", + ] + assert mock_run.call_args.kwargs["cwd"] == "/tmp/workspace" + + +def test_run_command_argv_omits_add_dir_without_work_dir(mock_run, sandbox): + """No ``work_dir`` means no ``--add-dir``, leaving agy's default workspace + behaviour untouched for scenarios that never asked for one.""" + generator = AgyCliGenerator({}) + generator._run_agy_cli(CLICommand(cli="agy", prompt="hello world")) + + assert "--add-dir" not in mock_run.call_args[0][0] + + def test_init_raises_on_non_string_timeout(sandbox): """The generator should raise TypeError if timeout is not a string.""" with pytest.raises(TypeError, match="timeout must be a string"): From 05e8a26af2900748da92b35733651252910115a1 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 31 Aug 2026 05:18:53 +0000 Subject: [PATCH 2/3] test(agy): drop redundant --add-dir absence test test_run_command_argv_shape already asserts exact argv equality for a command with no work_dir, so it fails if the flag is emitted unconditionally. --- evalbench/test/agy_cli_test.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/evalbench/test/agy_cli_test.py b/evalbench/test/agy_cli_test.py index d602adc4..7d9bbdae 100644 --- a/evalbench/test/agy_cli_test.py +++ b/evalbench/test/agy_cli_test.py @@ -308,15 +308,6 @@ def test_run_command_argv_passes_work_dir_as_add_dir(mock_run, sandbox): assert mock_run.call_args.kwargs["cwd"] == "/tmp/workspace" -def test_run_command_argv_omits_add_dir_without_work_dir(mock_run, sandbox): - """No ``work_dir`` means no ``--add-dir``, leaving agy's default workspace - behaviour untouched for scenarios that never asked for one.""" - generator = AgyCliGenerator({}) - generator._run_agy_cli(CLICommand(cli="agy", prompt="hello world")) - - assert "--add-dir" not in mock_run.call_args[0][0] - - def test_init_raises_on_non_string_timeout(sandbox): """The generator should raise TypeError if timeout is not a string.""" with pytest.raises(TypeError, match="timeout must be a string"): From 79700f3760466a6dec0c41bbe65bb3d2447f7931 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 31 Aug 2026 05:25:48 +0000 Subject: [PATCH 3/3] refactor(agy): cover create_command in the --add-dir test, note cwd asymmetry Route the test through create_command so the full evaluator path is pinned: work_dir -> CLICommand.cwd -> --add-dir plus subprocess cwd. Document why the fake_home cwd fallback is not mirrored into --add-dir. --- evalbench/generators/models/agy_cli.py | 3 +++ evalbench/test/agy_cli_test.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index d692e5e6..471198dc 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -910,6 +910,9 @@ def _run_agy_cli(self, cli_cmd: CLICommand): output_format="stream-json", log_file=self.cli_log_path, timeout=self.timeout, add_dir=cli_cmd.cwd, ) + # The fallback is deliberately not mirrored into --add-dir: fake_home + # holds harness internals (agy binary, settings, MCP config, session + # state), which the agent has no business editing as workspace files. cwd = cli_cmd.cwd if cli_cmd.cwd else self.fake_home result = self._execute_cli_command(command, env=env, cwd=cwd) diff --git a/evalbench/test/agy_cli_test.py b/evalbench/test/agy_cli_test.py index 7d9bbdae..264b2a00 100644 --- a/evalbench/test/agy_cli_test.py +++ b/evalbench/test/agy_cli_test.py @@ -296,7 +296,10 @@ def test_run_command_argv_passes_work_dir_as_add_dir(mock_run, sandbox): subprocess cwd: without a registered workspace agy runs its shell and write tools in ``/scratch``, so agent writes miss ``work_dir``.""" generator = AgyCliGenerator({}) - cmd = CLICommand(cli="agy", prompt="hello world", cwd="/tmp/workspace") + cmd = generator.create_command( + cli="agy", prompt="hello world", cwd="/tmp/workspace" + ) + assert cmd.cwd == "/tmp/workspace" generator._run_agy_cli(cmd) sent_argv = mock_run.call_args[0][0]