diff --git a/tests/tasks/uipath-ixp/_shared/mock_template/README.md b/tests/tasks/uipath-ixp/_shared/mock_template/README.md index 361664d464..f8e8ca9ed4 100644 --- a/tests/tasks/uipath-ixp/_shared/mock_template/README.md +++ b/tests/tasks/uipath-ixp/_shared/mock_template/README.md @@ -85,14 +85,22 @@ sinks in the same format, then delegates unchanged to the real CLI. Tasks graded with `cli_called` declare `sandbox.record_cli` instead, and the framework generates the recorders, seeds the log and PATH-prepends `cli_mocks/` itself — no `mock_path_dirs`, no `template_sources`, no `log:` on the criterion. -Prefer that for a new structured task. This template remains the answer whenever -a recorder is not enough: `file_matches_regex` over the flat `calls.log`, a read -that must return fixture data (`mock_template_taxonomy`), or per-invocation -responses, none of which `record_cli` serves. `calls.jsonl` here has no consumer -today for that reason — it is the structured sink for tasks that need this +Prefer that for a new structured task. Since coder-eval 0.12.1 a `record_cli` +entry can also carry `responses:` — per-invocation canned answers matched by the +same facets `cli_called` grades with — so serving a read before the graded write +no longer requires this template. No task uses that yet: a shape-only smoke task +grades the invocation, not the answer, so the blanket failure below is enough for +it. Reach for `responses:` when a criterion asserts a value the agent could only +have CARRIED from a read. This template remains the answer whenever a recorder is +still not enough: `file_matches_regex` over the flat `calls.log`, or a served read +that must be STATEFUL, where the answer depends on what the agent already wrote +(`mock_template_taxonomy`) — `select_rule` is stateless by design. `calls.jsonl` +here has no consumer today — it is the structured sink for tasks that need this template's other behavior too. A task whose correct path reads before it writes cannot be graded on this mock alone — the read fails, so there is nothing to carry into the graded write, and an agent that declines to invent the values correctly stops. Overlay -`mock_template_taxonomy` (listed second) to serve that read. +`mock_template_taxonomy` (listed second) to serve that read, or — for a task on +`record_cli` whose grading needs the carried value — declare the read as a +`responses:` rule on the entry itself. diff --git a/tests/tasks/uipath-ixp/smoke/list_documents.yaml b/tests/tasks/uipath-ixp/smoke/list_documents.yaml index c1c3dd6419..efe2368146 100644 --- a/tests/tasks/uipath-ixp/smoke/list_documents.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_documents.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, list all documents. @@ -24,10 +26,11 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent listed documents via `documents list `" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+documents\s+list\b(?=.*my_invoices-f1afa9ef-ixp).*)[^\r\n]*$ + - type: cli_called + description: "Agent listed documents via `documents list my_invoices-f1afa9ef-ixp`" + verb: "ixp documents list" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + value_flags: ["limit", "l", "offset", "output"] weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/list_projects.yaml b/tests/tasks/uipath-ixp/smoke/list_projects.yaml index 3b1445b068..723b3c2a36 100644 --- a/tests/tasks/uipath-ixp/smoke/list_projects.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_projects.yaml @@ -11,9 +11,11 @@ run_limits: expected_turns: 4 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | A document automation engineer wants to see every IXP project on the @@ -27,10 +29,9 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent invoked `uip ixp projects list`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+list(\s|$))[^\r\n]*$ + verb: "ixp projects list" + tool: "uip" weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index da433950a3..8beb40149f 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -11,9 +11,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, show the values the model @@ -26,10 +28,10 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent fetched predictions via `labellings get-predictions … doc-abc-123`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+get-predictions\b(?=.*doc-abc-123).*)[^\r\n]*$ + - type: cli_called + description: "Agent fetched predictions via `labellings get-predictions my_invoices-f1afa9ef-ixp doc-abc-123`" + verb: "ixp labellings get-predictions" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] weight: 3.0 pass_threshold: 1.0