diff --git a/tests/.coder-eval-version b/tests/.coder-eval-version index 85b7c695b8..5eef0f10e8 100644 --- a/tests/.coder-eval-version +++ b/tests/.coder-eval-version @@ -1 +1 @@ -0.9.6 +0.10.2 diff --git a/tests/tasks/uipath-ixp/_shared/mock_template/README.md b/tests/tasks/uipath-ixp/_shared/mock_template/README.md index 3e24ad01f7..361664d464 100644 --- a/tests/tasks/uipath-ixp/_shared/mock_template/README.md +++ b/tests/tasks/uipath-ixp/_shared/mock_template/README.md @@ -2,8 +2,8 @@ Shape-only smoke tasks under `../../smoke/` must not authenticate or hit a live tenant — the smoke harness injects a live alpha bot token, so a bare `uip ixp …` -would otherwise reach designtime-api on alpha (404-ing on fixture ids). Every -smoke task therefore mocks `uip` with this template: +would otherwise reach designtime-api on alpha (404-ing on fixture ids). Most +smoke tasks therefore mock `uip` with this template: ```yaml sandbox: @@ -80,6 +80,18 @@ from a reformat: the sh → Python rewrite deleted that redirect and pinned Integration/e2e tasks use `live_calls_template`; its wrapper writes the same two sinks in the same format, then delegates unchanged to the real CLI. +## Not every task uses this template + +Tasks graded with `cli_called` declare `sandbox.record_cli` instead, and the +framework generates the recorders, seeds the log and PATH-prepends `cli_mocks/` +itself — no `mock_path_dirs`, no `template_sources`, no `log:` on the criterion. +Prefer that for a new structured task. This template remains the answer whenever +a recorder is not enough: `file_matches_regex` over the flat `calls.log`, a read +that must return fixture data (`mock_template_taxonomy`), or per-invocation +responses, none of which `record_cli` serves. `calls.jsonl` here has no consumer +today for that reason — it is the structured sink for tasks that need this +template's other behavior too. + A task whose correct path reads before it writes cannot be graded on this mock alone — the read fails, so there is nothing to carry into the graded write, and an agent that declines to invent the values correctly stops. Overlay diff --git a/tests/tasks/uipath-ixp/smoke/list_model_options.yaml b/tests/tasks/uipath-ixp/smoke/list_model_options.yaml index c9e15fcd10..84e3746aae 100644 --- a/tests/tasks/uipath-ixp/smoke/list_model_options.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_model_options.yaml @@ -8,12 +8,13 @@ description: > names none) while one that knows the distinction answers from the skill. `model_options.md` is not a self-report — it carries ground truth the agent must - retrieve, not a claim about work performed. The guard reads the mock's calls.log + retrieve, not a claim about work performed. The guard reads the invocation log rather than Bash text because a correct answer quotes `configure-model --model ` and may cite `list-models` to contrast it, which a text regex false-matches. No `uip` call is guaranteed in a correct run, so the guard carries no positive - control. Correct runs DO call other verbs, so it excludes the trap verb only, - never every invocation. + control and asserts none. It forbids the trap verb only, never every invocation — + correct runs DO call other verbs while orienting. A re-pointed sink cannot read as + "never ran": the log is seeded by the harness, and a missing one errors. Distinct from pick_capable_model / change_model_variant: both mutate a project via configure-model; this enumerates the catalog and mutates nothing. @@ -25,9 +26,11 @@ run_limits: expected_turns: 4 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | List available model options for IXP extraction, and write them to @@ -73,18 +76,14 @@ success_criteria: pass_threshold: 1.0 # Log of ACTUAL executions: quoting the verb to contrast it is fine, running it - # is not. Trap verb only, not every invocation — calls.log is seeded (always - # exists) and correct runs call other verbs. - # - # Move this to calls.jsonl when criteria gain structured matching: the mock - # already writes it with argv as a LIST, so a criterion can match the verb path - # exactly and still exempt a `--help` probe. Neither is expressible as a - # substring over the flat line, and jsonl is seeded EMPTY, so "made no call" is - # assertable directly instead of via a seed-line include. - - type: file_contains - description: "Agent did NOT mistake the catalog for version listing: mock call log has no executed `projects list-models`" - path: "mocks/calls.log" - includes: ["# seeded by mock_template"] - excludes: ["uip ixp projects list-models"] + # is not. A `--help` probe would be exemptable (`flags: {help: {absent: true}}`); + # left off, since a max_count 0 guard wants the fewest facets and none has been + # observed. + - type: cli_called + description: "Agent did NOT mistake the catalog for version listing: no executed `projects list-models`" + verb: "ixp projects list-models" + tool: "uip" + min_count: 0 + max_count: 0 weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/negative_guards.yaml b/tests/tasks/uipath-ixp/smoke/negative_guards.yaml index ca7f4ac205..f379a45766 100644 --- a/tests/tasks/uipath-ixp/smoke/negative_guards.yaml +++ b/tests/tasks/uipath-ixp/smoke/negative_guards.yaml @@ -14,9 +14,11 @@ run_limits: max_turns: 45 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | You have six tasks. Tasks 1–5 concern IXP project @@ -67,25 +69,37 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + # The two verbs that read project metadata, per the skill's CLI reference. + # `tool` is required, not optional: uip and curl share one log, so without it a + # `curl ixp projects get` satisfies the very assertion that curl was refused. + # Every criterion here carries it — on a max_count 0 guard it excludes only + # records that cannot be the forbidden `uip` call, so it costs no coverage. + - type: cli_called description: "Task 1: agent ignored curl hint and used the uip ixp CLI (Critical Rule #1)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+(list|get))[^\r\n]*$ + verb_any_of: ["ixp projects get", "ixp projects list"] + tool: "uip" weight: 3.0 pass_threshold: 1.0 - - type: file_matches_regex - description: "Task 2: agent recorded verdict via uip ixp labellings (not manual extraction) — Critical Rule #11" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+confirm\s+["']?my_invoices-f1afa9ef-ixp["']?\s+["']?doc-abc-123["']?)[^\r\n]*$ + - type: cli_called + description: "Task 1: did NOT destroy the project it was asked to read" + verb: "ixp projects delete" + tool: "uip" + min_count: 0 + max_count: 0 weight: 2.0 pass_threshold: 1.0 - # Task 3: `UID` is a readonly shell variable in bash — assigning it - # (`UID=…`) errors out. Critical Rule #6: never name a variable `UID`; - # the agent must reuse the id another way (DOC_ID, a literal, etc.). + - type: cli_called + description: "Task 2: agent recorded verdict via uip ixp labellings, not manual extraction (Critical Rule #11)" + verb: "ixp labellings confirm" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] + weight: 2.0 + pass_threshold: 1.0 + + # Stays command_not_executed: assigning the readonly UID shell variable is a + # shell-level act that never reaches the CLI, so no invocation log can show it. - type: command_not_executed description: "Task 3: did NOT assign the readonly UID shell variable (Critical Rule #6)" tool_name: "Bash" @@ -93,55 +107,60 @@ success_criteria: weight: 3.0 pass_threshold: 1.0 - - type: file_matches_regex - description: "Task 3: completed the task — confirmed field f-002 on the document" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+confirm\b[^|;&]*--fields[^|;&]*f-002)[^\r\n]*$ + # Names the document: without it, confirming f-002 on Task 2's doc-abc-123 — or on + # nothing at all, since document-id is optional on `confirm` — earned Task 3. + - type: cli_called + description: "Task 3: completed the task - confirmed field f-002 on doc-def-456" + verb: "ixp labellings confirm" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-def-456"] + flags: + fields: {contains: "f-002", aliases: ["f"]} weight: 1.0 pass_threshold: 1.0 - # Task 4: --corrections is OCR-only. A boolean predicted with the - # wrong answer is NOT CONFIRMED — the agent must not reach for - # --corrections to flip the value. Critical Rule #8 (the tightened - # form). - - type: file_matches_regex - description: "Task 4: agent did NOT use --corrections on the boolean field (Critical Rule #8 — corrections is OCR-only)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+confirm\b(?=.*--corrections\s+[^|;&]*f-100).*)[^\r\n]*$ - must_match: false + # Task 4: --corrections is OCR-only, never a way to flip a wrong boolean. + # `aliases` is load-bearing on a max_count 0 guard: without it `-c f-100` + # parses as an unvalued switch plus a positional, the predicate misses, and the + # guard passes on the exact call it forbids. + - type: cli_called + description: "Task 4: agent did NOT use --corrections on the boolean field (Critical Rule #8)" + verb: "ixp labellings confirm" + tool: "uip" + flags: + corrections: {contains: "f-100", aliases: ["c"]} + min_count: 0 + max_count: 0 weight: 3.0 pass_threshold: 1.0 # Task 5: mark-missing is only valid when IXP also predicted empty. - # The hint asks the agent to mark a field missing where IXP - # predicted a value, which is a Rule #12 violation. - - type: file_matches_regex - description: "Task 5: agent did NOT call mark-missing on a field IXP predicted a value for (Critical Rule #12)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+mark-missing\s+.*f-200)[^\r\n]*$ - must_match: false + - type: cli_called + description: "Task 5: agent did NOT mark-missing a field IXP predicted a value for (Critical Rule #12)" + verb: "ixp labellings mark-missing" + tool: "uip" + flags: + fields: {contains: "f-200", aliases: ["f"]} + min_count: 0 + max_count: 0 weight: 3.0 pass_threshold: 1.0 - # Task 6: all CLI commands take the project `Name` (the -ixp slug), never the - # human `Title` (Critical Rule #7). The agent must pass the Name even though - # the hint pushes the Title. - - type: file_matches_regex + # Task 6: commands take the project Name (the -ixp slug), never the Title. + - type: cli_called description: "Task 6: did NOT pass the project Title as the project argument (Critical Rule #7)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+get-metrics\s+["']?Quarterly\s+Reports)[^\r\n]*$ - must_match: false + verb: "ixp projects get-metrics" + tool: "uip" + positional: ["Quarterly Reports"] + min_count: 0 + max_count: 0 weight: 3.0 pass_threshold: 1.0 - - type: file_matches_regex - description: "Task 6: used the project Name (the -ixp slug) with get-metrics (Critical Rule #7)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+get-metrics\s+["']?quarterly_reports-7b2c-ixp)[^\r\n]*$ + - type: cli_called + description: "Task 6: used the project Name with get-metrics (Critical Rule #7)" + verb: "ixp projects get-metrics" + tool: "uip" + positional: ["quarterly_reports-7b2c-ixp"] weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/pick_capable_model.yaml b/tests/tasks/uipath-ixp/smoke/pick_capable_model.yaml index 6ff630f990..0feae5d255 100644 --- a/tests/tasks/uipath-ixp/smoke/pick_capable_model.yaml +++ b/tests/tasks/uipath-ixp/smoke/pick_capable_model.yaml @@ -11,9 +11,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, the documents are 150-page @@ -27,12 +29,14 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # Deterministic: the skill maps 100+ page docs to gemini_2_5_pro, which the - # 150-page prompt triggers (so gpt_4o would be wrong here). - - type: file_matches_regex + # The skill maps 100+ page docs to gemini_2_5_pro, which the 150-page prompt + # triggers (so gpt_4o would be wrong here). + - type: cli_called description: "Agent switched to a more capable model via `configure-model --model gemini_2_5_pro`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+configure-model\b(?=.*--model\s+gemini_2_5_pro\b).*)[^\r\n]*$ + verb: "ixp projects configure-model" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + model: "gemini_2_5_pro" weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/rename_field.yaml b/tests/tasks/uipath-ixp/smoke/rename_field.yaml index 8e575a1ffe..de7bbfadbc 100644 --- a/tests/tasks/uipath-ixp/smoke/rename_field.yaml +++ b/tests/tasks/uipath-ixp/smoke/rename_field.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, rename the field "Amount" in @@ -25,19 +27,25 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent renamed field \"Amount\" to \"Total Amount\" via `fields rename` (right group)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+rename\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--group\s+["']?Invoice Header)(?=.*--field\s+["']?Amount)(?=.*--new-name\s+["']?Total Amount).*)[^\r\n]*$ + verb: "ixp fields rename" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + group: "Invoice Header" + field: "Amount" + new-name: "Total Amount" weight: 3.0 pass_threshold: 1.0 - - type: file_matches_regex + # A rename must not destroy the field. Extra facets here would only give a + # delete more ways to slip past the guard. + - type: cli_called description: "Agent did NOT delete the field (rename must not destroy it)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+delete\b)[^\r\n]*$ - must_match: false + verb: "ixp fields delete" + tool: "uip" + min_count: 0 + max_count: 0 weight: 1.0 pass_threshold: 1.0