From 26af35c7ecb03217fd98204a44992a69878d2719 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Wed, 2 Sep 2026 12:47:01 +0300 Subject: [PATCH 1/9] test(ixp): grade 13 more smoke tasks structurally via cli_called MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continues #2565. These tasks matched a rendered command string with stacked lookaheads over the flat `mocks/calls.log`, so verb, positionals and flags collapsed into one pattern whose facets could not fail independently — and several matched the right token in the wrong slot. `cli_called` matches per-facet against the recorded argv, and `sandbox.record_cli` replaces the hand-written mock, so the log shape stops being a contract between two repos. Four holes the port closes, each confirmed by replaying the pre-migration pattern against a rendered flat log rather than by reading it: - create_field_group: the four field names were sought anywhere on the `groups add` line, so listing them in --instructions beside an empty `--fields "[]"` satisfied the batch assertion. Now matched inside --fields. - create_choice_type: the prompt's own instructions text contains "not present", "True" and "False", so --instructions alone earned the criterion with wrong --choices. Now matched inside --choices. - delete_multiple_fields: `(?=.*field1)` also credited `--field field12`, and field1 appearing in the --group slot. Now `equals` on --field. - show_predictions: the document id was matched anywhere, so get-predictions on another project counted, as did project and document swapped. Pinning the id to the document slot requires naming the project slot before it, so the project is now asserted too. Weights are preserved 1:1. create_project_suggest_taxonomy's single guard was one regex alternation; cli_called cannot OR across facet sets, so it became two guards at 0.5 to keep the old 1.0 total. Two details worth carrying forward: a bare `yes` flag key is a YAML 1.1 boolean and must be quoted, and `present: true` (not `equals: ""`) is what keeps a `-y` switch from consuming the positional that follows it — the false-PASS class value-bearing declarations exist to prevent. Not migrated: import_taxonomy_from_file, because `positional` is exact-match only and asserting the file argument would break on `taxonomy.json` vs `./taxonomy.json`, silently dropping the filename check; and the seven overlay tasks, which need fixture reads record_cli does not serve. Verification: 13 YAMLs load through TaskDefinition; 76 criterion x trajectory pairs agree through the real CliCalledChecker over 62 synthetic logs, covering flags-before-positional reorderings, `-y`/`--yes` and `-f`/`--fields` short forms, curl-shaped argv and per-task violations; all 40 cli_called verbs in the suite exist in assets/uip-catalog-snapshot.json. cli_called is byte-identical between the installed 0.10.2 and the pinned 0.11.5 (one dropped unused parameter), so local replay holds for CI. No coder-eval run: this sandbox has no tenant, so there is no passing-run claim — see the PR body. Co-Authored-By: Claude Opus 5 (1M context) --- .../uipath-ixp/smoke/create_choice_type.yaml | 29 ++++++++--- .../tasks/uipath-ixp/smoke/create_field.yaml | 23 ++++++--- .../uipath-ixp/smoke/create_field_group.yaml | 24 +++++++--- .../smoke/create_project_blank.yaml | 19 +++++--- .../create_project_suggest_taxonomy.yaml | 48 +++++++++++++------ .../smoke/delete_document_by_id.yaml | 19 +++++--- .../uipath-ixp/smoke/delete_field_group.yaml | 21 +++++--- .../smoke/delete_multiple_fields.yaml | 32 ++++++++----- .../uipath-ixp/smoke/delete_project.yaml | 20 +++++--- .../uipath-ixp/smoke/list_documents.yaml | 17 ++++--- .../tasks/uipath-ixp/smoke/list_projects.yaml | 17 ++++--- .../tasks/uipath-ixp/smoke/mark_missing.yaml | 18 ++++--- .../uipath-ixp/smoke/show_predictions.yaml | 19 +++++--- 13 files changed, 203 insertions(+), 103 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml b/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml index 2c4266295b..f060afdbd4 100644 --- a/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml @@ -11,9 +11,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, create a new Field Type named @@ -30,10 +32,23 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + # The three display values are asserted against --choices, not the whole line: + # the prompt's own instructions text contains "not present", "True" and "False", + # so the regex this replaced was satisfied by --instructions alone. + - type: cli_called description: "Agent created the choice type via `data-types add … --kind choice … --choices`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+data-types\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*clause presence)(?=.*--kind\s+choice)(?=.*--choices)(?=.*[Tt]rue)(?=.*[Ff]alse)(?=.*[Nn]ot[ _-]?[Pp]resent)(?=.*--instructions).*)[^\r\n]*$ + verb: "ixp data-types add" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + name: "clause presence" + kind: "choice" + choices: + matches_regex: '(?=[\s\S]*[Tt]rue)(?=[\s\S]*[Ff]alse)(?=[\s\S]*[Nn]ot[ _-]?[Pp]resent)' + instructions: {present: true} + # `present` deliberately does not make a flag value-bearing, so --instructions + # and the kind-choice-mandatory --input-value are declared here to keep their + # values out of the positionals. + value_flags: ["instructions", "input-value", "output"] weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_field.yaml b/tests/tasks/uipath-ixp/smoke/create_field.yaml index 55d0c14a9e..6fe3b8d3a8 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, add a new field named "email" @@ -25,10 +27,17 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent added field \"email\" to \"Subscriber information\" via `fields add`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--group\s+["']?Subscriber information)(?=.*--field\s+["']?email).*)[^\r\n]*$ + verb: "ixp fields add" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + group: "Subscriber information" + field: "email" + # --type/--instructions carry values but get no predicate here, so they must be + # declared: undeclared, their values stay non-flag and shift the positional slot + # the project name is asserted in. + value_flags: ["type", "instructions", "output"] weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_field_group.yaml b/tests/tasks/uipath-ixp/smoke/create_field_group.yaml index fb71e46285..e17f98b230 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field_group.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field_group.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, create a new field group named @@ -28,10 +30,18 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + # The batch claim is that the four fields arrived in ONE --fields payload. The + # regex this replaced looked for the names anywhere on the `groups add` line, so + # listing them in --instructions beside an empty `--fields "[]"` satisfied it. + - type: cli_called description: "Agent created the group with multiple fields in one `groups add … --fields` call (batch)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+groups\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*Subscriber information)(?=.*--fields)(?=.*surname)(?=.*address)(?=.*bank account)(?=.*phone number).*)[^\r\n]*$ + verb: "ixp groups add" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + name: "Subscriber information" + fields: + matches_regex: '(?=[\s\S]*surname)(?=[\s\S]*address)(?=[\s\S]*bank account)(?=[\s\S]*phone number)' + value_flags: ["instructions", "output"] weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml b/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml index fb9866d3cb..9d5732c8f1 100644 --- a/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml @@ -11,9 +11,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | Create a blank IXP project named "test project". @@ -25,10 +27,13 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent created a blank project via `projects create \"test project\" … --skip-taxonomy`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+create\b(?=.*test project)(?=.*--skip-taxonomy).*)[^\r\n]*$ + verb: "ixp projects create" + tool: "uip" + positional: ["test project"] + flags: + skip-taxonomy: {present: true} + value_flags: ["description", "d", "output"] weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml b/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml index 84ed70c3a6..4cbb97fc79 100644 --- a/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | Create an IXP project named ap-invoices from the invoices in ./invoices/, @@ -27,21 +29,37 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent created ap-invoices via `projects create` from ./invoices/" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+create\b(?=.*ap-invoices).*)[^\r\n]*$ + verb: "ixp projects create" + tool: "uip" + positional: ["ap-invoices"] + value_flags: ["description", "d", "output"] weight: 3.0 pass_threshold: 1.0 # Autopilot path: let IXP suggest the taxonomy. --skip-taxonomy or a manual - # import would defeat the test. - - type: file_matches_regex - description: "Agent did NOT skip or hand-import the taxonomy (autopilot suggests it)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+(create\b.*--skip-taxonomy|import-taxonomy)\b)[^\r\n]*$ - must_match: false - weight: 1.0 + # import would defeat the test. One regex alternation became two guards, since + # cli_called cannot OR across facet sets; 0.5 each keeps the old 1.0 total. + # + # `present` is load-bearing on the first: a value-bearing --skip-taxonomy would + # swallow the folder path, and the guard would pass on the call it forbids. + - type: cli_called + description: "Agent did NOT skip the taxonomy on create (autopilot suggests it)" + verb: "ixp projects create" + tool: "uip" + flags: + skip-taxonomy: {present: true} + min_count: 0 + max_count: 0 + weight: 0.5 + pass_threshold: 1.0 + + - type: cli_called + description: "Agent did NOT hand-import a taxonomy (autopilot suggests it)" + verb: "ixp projects import-taxonomy" + tool: "uip" + min_count: 0 + max_count: 0 + weight: 0.5 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml index d9c29d7f4b..b901b543bf 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml @@ -13,9 +13,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete document doc-abc-123. @@ -27,10 +29,13 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent deleted the document via `documents delete doc-abc-123 -y`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+documents\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*doc-abc-123)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ + verb: "ixp documents delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] + flags: + # Quoted: bare `yes` is a YAML 1.1 boolean. + "yes": {present: true, aliases: ["y"]} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml index c84b44c704..48bbededdb 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the field group @@ -27,10 +29,15 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent deleted the group via `groups delete … -y/--yes`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+groups\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--name\s+["']?name group)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ + verb: "ixp groups delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + name: "name group" + # Quoted: bare `yes` is a YAML 1.1 boolean, and the key must reach the + # grader as the string flag name. + "yes": {present: true, aliases: ["y"]} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml index 76262e9881..6c11b19807 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 6 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the fields "field1" and @@ -27,18 +29,26 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + # `equals` on --field, not the old line-wide substring: `(?=.*field1)` was also + # satisfied by `--field field12`, and by field1 appearing in --group. + - type: cli_called description: "Agent deleted field1 via `fields delete`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field1).*)[^\r\n]*$ + verb: "ixp fields delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + field: "field1" + value_flags: ["group", "output"] weight: 2.0 pass_threshold: 1.0 - - type: file_matches_regex + - type: cli_called description: "Agent deleted field2 via `fields delete` (both, not just one)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field2).*)[^\r\n]*$ + verb: "ixp fields delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + field: "field2" + value_flags: ["group", "output"] weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_project.yaml b/tests/tasks/uipath-ixp/smoke/delete_project.yaml index 1e9a56bec6..6dcdd1265d 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_project.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_project.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | Delete the IXP project my_invoices-f1afa9ef-ixp. @@ -26,10 +28,14 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent deleted the project via `projects delete … -y/--yes`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ + verb: "ixp projects delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + # Quoted: bare `yes` is a YAML 1.1 boolean. `present` also keeps the switch + # from consuming the project name that follows it in `delete -y `. + "yes": {present: true, aliases: ["y"]} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/list_documents.yaml b/tests/tasks/uipath-ixp/smoke/list_documents.yaml index c1c3dd6419..86ebfc5b4f 100644 --- a/tests/tasks/uipath-ixp/smoke/list_documents.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_documents.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, list all documents. @@ -24,10 +26,11 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent listed documents via `documents list `" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+documents\s+list\b(?=.*my_invoices-f1afa9ef-ixp).*)[^\r\n]*$ + verb: "ixp documents list" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + value_flags: ["limit", "l", "offset", "output"] weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/list_projects.yaml b/tests/tasks/uipath-ixp/smoke/list_projects.yaml index 3b1445b068..2070795e6e 100644 --- a/tests/tasks/uipath-ixp/smoke/list_projects.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_projects.yaml @@ -11,9 +11,11 @@ run_limits: expected_turns: 4 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | A document automation engineer wants to see every IXP project on the @@ -27,10 +29,11 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + # No positional facet: `projects list` takes none. The verb is matched token-wise, + # so it does not credit the near-name `projects list-models`. + - type: cli_called description: "Agent invoked `uip ixp projects list`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+list(\s|$))[^\r\n]*$ + verb: "ixp projects list" + tool: "uip" weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml index 00686e99b5..21553fab6e 100644 --- a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml +++ b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 6 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, document doc-abc-123, the field @@ -26,10 +28,12 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + - type: cli_called description: "Agent marked the field missing via `labellings mark-missing` on the right project + document + field id" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+mark-missing\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*doc-abc-123)(?=.*a7c3e9105f2b4d86).*)[^\r\n]*$ + verb: "ixp labellings mark-missing" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] + flags: + fields: {contains: "a7c3e9105f2b4d86", aliases: ["f"]} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index da433950a3..d5135fe744 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -11,9 +11,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, show the values the model @@ -26,10 +28,13 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex + # The project is named because positionals are ordered: doc-abc-123 can only be + # asserted as the DOCUMENT argument by pinning the slot before it. The regex this + # replaced matched the id anywhere, so a get-predictions on another project counted. + - type: cli_called description: "Agent fetched predictions via `labellings get-predictions … doc-abc-123`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+get-predictions\b(?=.*doc-abc-123).*)[^\r\n]*$ + verb: "ixp labellings get-predictions" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] weight: 3.0 pass_threshold: 1.0 From 7d4df0f9f8fb5a37a3275f3498c3ea71048eeaa6 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 12:33:38 +0300 Subject: [PATCH 2/9] test(ixp): serve the correct path from record_cli responses (coder-eval 0.12.1) The 13 migrated smoke tasks shadowed `uip` with a stub that failed every invocation, so each prompt had to carry a block telling the agent that its commands would fail, that this was expected, and not to retry or log in. That block is a large hint no real environment gives, and it means a task can only ever grade "typed the right thing once" -- there is no answer to read, so no step can depend on what the tool just said. coder-eval 0.12.1 adds `responses:` to a `record_cli` entry: per-invocation canned answers matched by the same facets `cli_called` grades with. Each task now serves the calls its correct path makes -- the `get-taxonomy` read before a taxonomy edit, the `get-predictions` read before a labelling write, the graded write itself -- and the entry's own exit 1 is left as the fallback, so an off-path invocation still meets a CLI that refused. `uip` works and `curl` does not, which is the asymmetry the suite wants and could not express while everything failed identically. The failure framing is therefore gone from all 13 prompts; only the no-prompting line survives. Response bodies reuse the ids, titles and envelope shapes already in `_shared/mock_template_taxonomy/mocks/uip`, so the suite keeps describing one tenant. `show_predictions` can now actually show predicted values, and `create_project_suggest_taxonomy` gets the `./invoices/` folder its prompt has always named -- harmless while create failed, not once it succeeds. No criterion changed. Verification, against real coder-eval 0.12.1 (not the 0.11.5 the branch was written on -- main's merge brings the pin to 0.12.1): - All 13 load through the real `TaskDefinition`; every response `stdout` parses as JSON and every rule exits 0. - 23 realistic invocations replayed through the EXECUTED shims: every one was answered by a response rule, none fell through to the fallback, and no record booked `rule_error` or `sidecar_error`. - All 16 `cli_called` criteria still pass, graded by the real `CliCalledChecker` over the `calls.jsonl` those trajectories produced. - 6 violating trajectories replayed -- including each hole this PR closes and both autopilot guards -- all still score 0.0, so nothing became vacuous. - All 13 verbs the rules serve exist in `assets/uip-catalog-snapshot.json` (uip 1.203.0-dev.8651); `check-cli-verbs.py` and `check-task-driver.py` clean. `_shared/mock_template/README.md` claimed `record_cli` does not serve per-invocation responses. It does now, so the README says what is actually left to the sh-mock template: a flat `calls.log` for `file_matches_regex`, and a served read that must be STATEFUL -- `select_rule` is stateless by design. Co-Authored-By: Claude Opus 5 (1M context) --- .../_shared/mock_template/README.md | 19 ++++++---- .../uipath-ixp/smoke/create_choice_type.yaml | 22 +++++++----- .../tasks/uipath-ixp/smoke/create_field.yaml | 22 +++++++----- .../uipath-ixp/smoke/create_field_group.yaml | 22 +++++++----- .../smoke/create_project_blank.yaml | 19 +++++----- .../create_project_suggest_taxonomy.yaml | 35 ++++++++++++++----- .../smoke/delete_document_by_id.yaml | 23 +++++++----- .../uipath-ixp/smoke/delete_field_group.yaml | 22 +++++++----- .../smoke/delete_multiple_fields.yaml | 22 +++++++----- .../uipath-ixp/smoke/delete_project.yaml | 23 +++++++----- .../uipath-ixp/smoke/list_documents.yaml | 23 +++++++----- .../tasks/uipath-ixp/smoke/list_projects.yaml | 20 ++++++----- .../tasks/uipath-ixp/smoke/mark_missing.yaml | 22 +++++++----- .../uipath-ixp/smoke/show_predictions.yaml | 19 +++++----- 14 files changed, 201 insertions(+), 112 deletions(-) diff --git a/tests/tasks/uipath-ixp/_shared/mock_template/README.md b/tests/tasks/uipath-ixp/_shared/mock_template/README.md index 361664d464..30380c4c49 100644 --- a/tests/tasks/uipath-ixp/_shared/mock_template/README.md +++ b/tests/tasks/uipath-ixp/_shared/mock_template/README.md @@ -85,14 +85,19 @@ sinks in the same format, then delegates unchanged to the real CLI. Tasks graded with `cli_called` declare `sandbox.record_cli` instead, and the framework generates the recorders, seeds the log and PATH-prepends `cli_mocks/` itself — no `mock_path_dirs`, no `template_sources`, no `log:` on the criterion. -Prefer that for a new structured task. This template remains the answer whenever -a recorder is not enough: `file_matches_regex` over the flat `calls.log`, a read -that must return fixture data (`mock_template_taxonomy`), or per-invocation -responses, none of which `record_cli` serves. `calls.jsonl` here has no consumer -today for that reason — it is the structured sink for tasks that need this -template's other behavior too. +Prefer that for a new structured task. Since coder-eval 0.12.1 a `record_cli` +entry also carries `responses:` — per-invocation canned answers matched by the +same facets `cli_called` grades with — so serving a read before the graded write +no longer needs this template either (the smoke tasks under `../../smoke/` that +declare `record_cli` do exactly that). This template remains the answer whenever +a recorder is still not enough: `file_matches_regex` over the flat `calls.log`, +or a served read that must be STATEFUL, where the answer depends on what the +agent already wrote (`mock_template_taxonomy`) — `select_rule` is stateless by +design. `calls.jsonl` here has no consumer today — it is the structured sink for +tasks that need this template's other behavior too. A task whose correct path reads before it writes cannot be graded on this mock alone — the read fails, so there is nothing to carry into the graded write, and an agent that declines to invent the values correctly stops. Overlay -`mock_template_taxonomy` (listed second) to serve that read. +`mock_template_taxonomy` (listed second) to serve that read, or — for a task on +`record_cli` — declare the read as a `responses:` rule on the entry itself. diff --git a/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml b/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml index f060afdbd4..1e1d264add 100644 --- a/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml @@ -3,8 +3,8 @@ description: > Smoke: agent creates a custom Choice data type with three options and instructions. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -13,7 +13,17 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"566c42e2ae1d7c21","name":"Boolean","title":"Boolean","inherits_from":["0000000000000046"],"trainable":true,"color":0,"instructions":"Use to represent boolean values in text."},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"59483fee51fcb287","name":"Inferred Text","title":"Inferred Text","inherits_from":[],"trainable":true,"color":3,"instructions":"Use for text that might not appear verbatim in the text."},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]}]}]}}} + - when: {verb: "ixp data-types add", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Name":"clause presence","Kind":"choice"}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -25,11 +35,7 @@ initial_prompt: | obliges the parties to auto-renew; "False" if it is present but does not oblige auto-renewal. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: # The three display values are asserted against --choices, not the whole line: diff --git a/tests/tasks/uipath-ixp/smoke/create_field.yaml b/tests/tasks/uipath-ixp/smoke/create_field.yaml index 6fe3b8d3a8..e5f33dd7bf 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field.yaml @@ -2,8 +2,8 @@ task_id: skill-ixp-create-field-smoke description: > Smoke: agent adds a single field to an existing group. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -12,7 +12,17 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]},{"name":"Subscriber information","id":"2a6f0c9be4d18537","instructions":"Details of the subscriber named on the invoice.","moon_form":[{"field_id":"e83b15d7a0c94f26","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"surname","instructions":"The subscriber's family name.","rule_set":null}]}]}]}}} + - when: {verb: "ixp fields add", value_flags: ["output"]} + stdout: | + {"Result":"Success"} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -20,11 +30,7 @@ initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, add a new field named "email" to the "Subscriber information" group. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/create_field_group.yaml b/tests/tasks/uipath-ixp/smoke/create_field_group.yaml index e17f98b230..61a6ce2c5d 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field_group.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field_group.yaml @@ -4,8 +4,8 @@ description: > not one call per field. The user gives no field types, so the agent defaults them and proceeds (it must not stop to ask). - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -14,7 +14,17 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]}]}]}}} + - when: {verb: "ixp groups add", value_flags: ["output"]} + stdout: | + {"Result":"Success"} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -23,11 +33,7 @@ initial_prompt: | "Subscriber information" with fields: surname, address, bank account, and phone number in one batched call. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: # The batch claim is that the four fields arrived in ONE --fields payload. The diff --git a/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml b/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml index 9d5732c8f1..f4e3ed408b 100644 --- a/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml @@ -3,8 +3,8 @@ description: > Smoke: agent creates a blank IXP project with `--skip-taxonomy` — no taxonomy, no docs, so it doesn't stall waiting for files or a suggestion. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -13,18 +13,21 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects create", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"ProjectName":"test_project-3d91c4a2-ixp","Title":"test project","TaxonomyImported":false}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | Create a blank IXP project named "test project". - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml b/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml index 4cbb97fc79..5e42cbe4ec 100644 --- a/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml @@ -4,8 +4,8 @@ description: > the documents. Trap: don't pass --skip-taxonomy or hand-author/import a taxonomy — IXP suggests it. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -14,19 +14,38 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects create", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"ProjectName":"ap_invoices-8c52f7b0-ixp","Title":"ap-invoices","DocumentsUploaded":3,"TaxonomyImported":true,"TaxonomySource":"suggested"}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + # ./invoices/ is the folder the prompt names and `projects create` takes as + # its second positional. Nothing lived there while every call failed; now + # that create succeeds, an agent that checks the folder must find it. + template_sources: + - type: starter_files + files: + - path: invoices/hp_tax_invoice.png + content: | + offline-smoke placeholder - the generated `uip` stub never reads this file + - path: invoices/york_solutions_invoice.png + content: | + offline-smoke placeholder - the generated `uip` stub never reads this file + - path: invoices/csi_tower_invoice.png + content: | + offline-smoke placeholder - the generated `uip` stub never reads this file + initial_prompt: | Create an IXP project named ap-invoices from the invoices in ./invoices/, and let IXP suggest the taxonomy. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml index b901b543bf..8f36d79ba8 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml @@ -5,8 +5,8 @@ task_id: skill-ixp-delete-document-by-id-smoke description: > Smoke: agent deletes a document by id — irreversible, triggers a retrain. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: @@ -15,18 +15,25 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + # Orientation: the agent may confirm the document exists before deleting it. + - when: {verb: "ixp documents list", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Documents":[{"DocumentId":"doc-abc-123","AttachmentRef":"att-9f31c7","Filename":"hp_tax_invoice.png"},{"DocumentId":"doc-def-456","AttachmentRef":"att-40b2ae","Filename":"york_solutions_invoice.png"},{"DocumentId":"doc-ghi-789","AttachmentRef":null,"Filename":"csi_tower_invoice.png"}],"Total":3,"Offset":0,"Limit":50}} + - when: {verb: "ixp documents delete", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"DocumentId":"doc-abc-123","Deleted":true}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete document doc-abc-123. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml index 48bbededdb..2f91285cc4 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml @@ -4,8 +4,8 @@ description: > annotations on all its fields, so it must pass -y/--yes (the CLI refuses without it). - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -14,7 +14,17 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]},{"name":"name group","id":"b41d8e27f0a3c659","instructions":"Legacy group kept from an earlier taxonomy import.","moon_form":[{"field_id":"70f2c58ad916be34","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"full name","instructions":"The payer's full name as printed.","rule_set":null}]}]}]}}} + - when: {verb: "ixp groups delete", value_flags: ["output"]} + stdout: | + {"Result":"Success"} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -22,11 +32,7 @@ initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the field group "name group". - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml index 6c11b19807..4dd8fc04c4 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml @@ -4,8 +4,8 @@ description: > `fields delete` removes one field per call, so the agent must issue both. Trap: deleting only one and reporting done. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -14,7 +14,17 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"3c7e91a5d284b06f","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"field1","instructions":"Placeholder field carried over from an earlier import.","rule_set":null},{"field_id":"8b05d3f7c1927ae4","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"field2","instructions":"Placeholder field carried over from an earlier import.","rule_set":null},{"field_id":"d692af41b7350e8c","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"field12","instructions":"Placeholder field carried over from an earlier import.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]}]}]}}} + - when: {verb: "ixp fields delete", value_flags: ["output"]} + stdout: | + {"Result":"Success"} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -22,11 +32,7 @@ initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the fields "field1" and "field2" from the "Invoice Header" group. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: # `equals` on --field, not the old line-wide substring: `(?=.*field1)` was also diff --git a/tests/tasks/uipath-ixp/smoke/delete_project.yaml b/tests/tasks/uipath-ixp/smoke/delete_project.yaml index 6dcdd1265d..8e7c590bf8 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_project.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_project.yaml @@ -4,8 +4,8 @@ description: > pass -y/--yes (the CLI refuses without it), acknowledging the data loss. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: @@ -14,18 +14,25 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + # Orientation: the agent may confirm the project exists before deleting it. + - when: {verb: "ixp projects list", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Projects":[{"Id":"f1afa9ef-1c4d-4b70-9e02-6d3b8a5c7e11","Name":"my_invoices-f1afa9ef-ixp","Title":"My_Invoices","CreatedAt":"2026-06-11T08:42:19Z"},{"Id":"7b2c5d19-8e34-4a61-b0f7-2c9d4e6a1b83","Name":"quarterly_reports-7b2c-ixp","Title":"Quarterly Reports","CreatedAt":"2026-07-02T14:05:40Z"}],"Total":2,"Offset":0,"Limit":50}} + - when: {verb: "ixp projects delete", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Name":"my_invoices-f1afa9ef-ixp","Deleted":true}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | Delete the IXP project my_invoices-f1afa9ef-ixp. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/list_documents.yaml b/tests/tasks/uipath-ixp/smoke/list_documents.yaml index 86ebfc5b4f..d54f2b8721 100644 --- a/tests/tasks/uipath-ixp/smoke/list_documents.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_documents.yaml @@ -2,8 +2,8 @@ task_id: skill-ixp-list-documents-smoke description: > Smoke: agent lists the documents in a project. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: @@ -12,18 +12,25 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + # Orientation: the agent may resolve the project before listing its documents. + - when: {verb: "ixp projects list", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Projects":[{"Id":"f1afa9ef-1c4d-4b70-9e02-6d3b8a5c7e11","Name":"my_invoices-f1afa9ef-ixp","Title":"My_Invoices","CreatedAt":"2026-06-11T08:42:19Z"},{"Id":"7b2c5d19-8e34-4a61-b0f7-2c9d4e6a1b83","Name":"quarterly_reports-7b2c-ixp","Title":"Quarterly Reports","CreatedAt":"2026-07-02T14:05:40Z"}],"Total":2,"Offset":0,"Limit":50}} + - when: {verb: "ixp documents list", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Documents":[{"DocumentId":"doc-abc-123","AttachmentRef":"att-9f31c7","Filename":"hp_tax_invoice.png"},{"DocumentId":"doc-def-456","AttachmentRef":"att-40b2ae","Filename":"york_solutions_invoice.png"},{"DocumentId":"doc-ghi-789","AttachmentRef":null,"Filename":"csi_tower_invoice.png"}],"Total":3,"Offset":0,"Limit":50}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, list all documents. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/list_projects.yaml b/tests/tasks/uipath-ixp/smoke/list_projects.yaml index 2070795e6e..85e3d864b3 100644 --- a/tests/tasks/uipath-ixp/smoke/list_projects.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_projects.yaml @@ -3,8 +3,8 @@ description: > Smoke: agent picks `uip ixp projects list --output json` to enumerate IXP projects on the tenant. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:discover] run_limits: @@ -13,7 +13,14 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp projects list", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Projects":[{"Id":"f1afa9ef-1c4d-4b70-9e02-6d3b8a5c7e11","Name":"my_invoices-f1afa9ef-ixp","Title":"My_Invoices","CreatedAt":"2026-06-11T08:42:19Z"},{"Id":"7b2c5d19-8e34-4a61-b0f7-2c9d4e6a1b83","Name":"quarterly_reports-7b2c-ixp","Title":"Quarterly Reports","CreatedAt":"2026-07-02T14:05:40Z"}],"Total":2,"Offset":0,"Limit":50}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -21,12 +28,7 @@ initial_prompt: | A document automation engineer wants to see every IXP project on the tenant before deciding which one to iterate on. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant - in this environment. Commands will fail with auth errors — that is - expected. Run each command once and move on. Do NOT retry or attempt - to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: # No positional facet: `projects list` takes none. The verb is matched token-wise, diff --git a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml index 21553fab6e..49a601744e 100644 --- a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml +++ b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml @@ -2,8 +2,8 @@ task_id: skill-ixp-mark-missing-smoke description: > Smoke: agent records a field as missing with `labellings mark-missing`. - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: @@ -12,7 +12,17 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp labellings get-predictions", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"ProjectName":"my_invoices-f1afa9ef-ixp","TotalDocuments":1,"DocumentsWithPredictions":1,"Predictions":[{"DocumentId":"doc-abc-123","ModelVersion":7,"Labels":[{"Name":"Invoice Header","Occurrence":0,"Fields":[{"FieldId":"9a3f5d2c81b7e604","FieldName":"Invoice Number","FormattedValue":"INV-2025-0042"},{"FieldId":"6e0b74a1f83c2d95","FieldName":"Total Amount","FormattedValue":"1200.00"},{"FieldId":"a7c3e9105f2b4d86","FieldName":"Discount Code","FormattedValue":""}]}]}]}} + - when: {verb: "ixp labellings mark-missing", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"Unmatched":[]}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -21,11 +31,7 @@ initial_prompt: | "Discount Code" (field id a7c3e9105f2b4d86) is genuinely absent from the document and the model predicted no value for it. Record it as missing. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index d5135fe744..3578977152 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -3,8 +3,8 @@ description: > Smoke: agent reads the model's predicted values for one document (the prediction layer, via get-predictions — not by reading the file itself). - Harness is unauthenticated; CLI calls fail with auth errors. - Grades command shape, not wire effect. + Offline: `uip` is a generated stub that answers this task's calls from + canned responses. Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: @@ -13,7 +13,14 @@ run_limits: sandbox: record_cli: - tool: uip - stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + # Every call the correct path makes is answered below. Anything else + # falls through to this fallback and fails, so an off-path invocation + # still looks like a CLI that refused rather than one that agreed. + stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" + responses: + - when: {verb: "ixp labellings get-predictions", value_flags: ["output"]} + stdout: | + {"Result":"Success","Data":{"ProjectName":"my_invoices-f1afa9ef-ixp","TotalDocuments":1,"DocumentsWithPredictions":1,"Predictions":[{"DocumentId":"doc-abc-123","ModelVersion":7,"Labels":[{"Name":"Invoice Header","Occurrence":0,"Fields":[{"FieldId":"9a3f5d2c81b7e604","FieldName":"Invoice Number","FormattedValue":"INV-2025-0042"},{"FieldId":"1d8e42b7c3a95f60","FieldName":"Invoice Date","FormattedValue":"2025-11-04"},{"FieldId":"6e0b74a1f83c2d95","FieldName":"Total Amount","FormattedValue":"1200.00"}]},{"Name":"Line Items","Occurrence":0,"Fields":[{"FieldId":"c47a91e05d2b3f68","FieldName":"Description","FormattedValue":"Annual support subscription"}]}]}]}} - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -21,11 +28,7 @@ initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, show the values the model predicted for document doc-abc-123. - Important: - - The `uip` CLI is available but is NOT connected to a live IXP tenant in - this environment. Commands will fail with auth errors — that is expected. - Run each command once and move on. Do NOT retry or attempt to login. - - Do NOT prompt the user for confirmation — this is an automated test. + Do NOT prompt the user for confirmation — this is an automated test. success_criteria: # The project is named because positionals are ordered: doc-abc-123 can only be From 4812f017c14c232c3ad7e51fd94e39e44fb62e73 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 15:37:02 +0300 Subject: [PATCH 3/9] test(ixp): cut this PR to the first risk class (3 tasks) Thirteen tasks in one change is too much to review, and this PR's own follow-ups argued for "one PR per risk class rather than one bulk change" after #2565 needed eight follow-up commits for four tasks. Taking that advice here. Keeps the class with no flag predicates at all -- verb and positionals only: list_projects verb only list_documents verb + positional show_predictions verb + two ordered positionals `show_predictions` is the one hole-closer in the batch: the pattern it replaces matched the document id ANYWHERE on the line, so `get-predictions` on another project counted, as did project and document swapped. Pinning `doc-abc-123` to the document slot requires naming the slot before it, so the project is asserted too. The other ten revert to their state on main -- still `file_matches_regex` over the flat `mocks/calls.log`, untouched. They are preserved whole on `test/ixp-cli-called-queue` (at 7d4df0f9f) and queue behind this one by risk class, so no work is redone: 2. `-y`/`--yes` switch presence delete_document_by_id, delete_project, delete_field_group 3. value predicates create_field, delete_multiple_fields, mark_missing 4. regex over JSON payloads create_field_group, create_choice_type 5. negative / multi-criterion create_project_blank, create_project_suggest_taxonomy Batches 3 and 4 carry the remaining three holes, so the interesting review lands where the mechanism is already established rather than beside twelve other diffs. The 0.12.1 `responses:` work stays with these three, because per task the two changes are coupled: the prompt's "commands will fail -- do not retry" block can only be deleted once the stub actually answers. Splitting them would knowingly leave a false statement in the prompt for a later PR to fix. Net diff is now 4 files, +78/-50. Re-verified after the cut, against real coder-eval 0.12.1: 3 tasks load through the real `TaskDefinition`; 4 realistic invocations replayed through the executed shims were all answered by a response rule with none falling through and no `rule_error`/`sidecar_error`; all 3 `cli_called` criteria pass under the real `CliCalledChecker`; the `show_predictions` wrong-project trajectory still scores 0.0. `check-cli-verbs.py` 0 High / 0 Medium, `check-task-driver.py` clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../uipath-ixp/smoke/create_choice_type.yaml | 49 ++++-------- .../tasks/uipath-ixp/smoke/create_field.yaml | 43 ++++------ .../uipath-ixp/smoke/create_field_group.yaml | 44 ++++------- .../smoke/create_project_blank.yaml | 36 ++++----- .../create_project_suggest_taxonomy.yaml | 79 +++++-------------- .../smoke/delete_document_by_id.yaml | 40 ++++------ .../uipath-ixp/smoke/delete_field_group.yaml | 41 ++++------ .../smoke/delete_multiple_fields.yaml | 52 +++++------- .../uipath-ixp/smoke/delete_project.yaml | 41 ++++------ .../tasks/uipath-ixp/smoke/mark_missing.yaml | 38 ++++----- 10 files changed, 151 insertions(+), 312 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml b/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml index 1e1d264add..2c4266295b 100644 --- a/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_choice_type.yaml @@ -3,29 +3,17 @@ description: > Smoke: agent creates a custom Choice data type with three options and instructions. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"566c42e2ae1d7c21","name":"Boolean","title":"Boolean","inherits_from":["0000000000000046"],"trainable":true,"color":0,"instructions":"Use to represent boolean values in text."},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"59483fee51fcb287","name":"Inferred Text","title":"Inferred Text","inherits_from":[],"trainable":true,"color":3,"instructions":"Use for text that might not appear verbatim in the text."},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]}]}]}}} - - when: {verb: "ixp data-types add", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Name":"clause presence","Kind":"choice"}} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, create a new Field Type named @@ -35,26 +23,17 @@ initial_prompt: | obliges the parties to auto-renew; "False" if it is present but does not oblige auto-renewal. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # The three display values are asserted against --choices, not the whole line: - # the prompt's own instructions text contains "not present", "True" and "False", - # so the regex this replaced was satisfied by --instructions alone. - - type: cli_called + - type: file_matches_regex description: "Agent created the choice type via `data-types add … --kind choice … --choices`" - verb: "ixp data-types add" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp"] - flags: - name: "clause presence" - kind: "choice" - choices: - matches_regex: '(?=[\s\S]*[Tt]rue)(?=[\s\S]*[Ff]alse)(?=[\s\S]*[Nn]ot[ _-]?[Pp]resent)' - instructions: {present: true} - # `present` deliberately does not make a flag value-bearing, so --instructions - # and the kind-choice-mandatory --input-value are declared here to keep their - # values out of the positionals. - value_flags: ["instructions", "input-value", "output"] + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+data-types\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*clause presence)(?=.*--kind\s+choice)(?=.*--choices)(?=.*[Tt]rue)(?=.*[Ff]alse)(?=.*[Nn]ot[ _-]?[Pp]resent)(?=.*--instructions).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_field.yaml b/tests/tasks/uipath-ixp/smoke/create_field.yaml index e5f33dd7bf..55d0c14a9e 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field.yaml @@ -2,48 +2,33 @@ task_id: skill-ixp-create-field-smoke description: > Smoke: agent adds a single field to an existing group. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]},{"name":"Subscriber information","id":"2a6f0c9be4d18537","instructions":"Details of the subscriber named on the invoice.","moon_form":[{"field_id":"e83b15d7a0c94f26","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"surname","instructions":"The subscriber's family name.","rule_set":null}]}]}]}}} - - when: {verb: "ixp fields add", value_flags: ["output"]} - stdout: | - {"Result":"Success"} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, add a new field named "email" to the "Subscriber information" group. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: cli_called + - type: file_matches_regex description: "Agent added field \"email\" to \"Subscriber information\" via `fields add`" - verb: "ixp fields add" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp"] - flags: - group: "Subscriber information" - field: "email" - # --type/--instructions carry values but get no predicate here, so they must be - # declared: undeclared, their values stay non-flag and shift the positional slot - # the project name is asserted in. - value_flags: ["type", "instructions", "output"] + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+fields\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--group\s+["']?Subscriber information)(?=.*--field\s+["']?email).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_field_group.yaml b/tests/tasks/uipath-ixp/smoke/create_field_group.yaml index 61a6ce2c5d..fb71e46285 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field_group.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field_group.yaml @@ -4,50 +4,34 @@ description: > not one call per field. The user gives no field types, so the agent defaults them and proceeds (it must not stop to ask). - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]}]}]}}} - - when: {verb: "ixp groups add", value_flags: ["output"]} - stdout: | - {"Result":"Success"} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, create a new field group named "Subscriber information" with fields: surname, address, bank account, and phone number in one batched call. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # The batch claim is that the four fields arrived in ONE --fields payload. The - # regex this replaced looked for the names anywhere on the `groups add` line, so - # listing them in --instructions beside an empty `--fields "[]"` satisfied it. - - type: cli_called + - type: file_matches_regex description: "Agent created the group with multiple fields in one `groups add … --fields` call (batch)" - verb: "ixp groups add" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp"] - flags: - name: "Subscriber information" - fields: - matches_regex: '(?=[\s\S]*surname)(?=[\s\S]*address)(?=[\s\S]*bank account)(?=[\s\S]*phone number)' - value_flags: ["instructions", "output"] + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+groups\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*Subscriber information)(?=.*--fields)(?=.*surname)(?=.*address)(?=.*bank account)(?=.*phone number).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml b/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml index f4e3ed408b..fb9866d3cb 100644 --- a/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_project_blank.yaml @@ -3,40 +3,32 @@ description: > Smoke: agent creates a blank IXP project with `--skip-taxonomy` — no taxonomy, no docs, so it doesn't stall waiting for files or a suggestion. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects create", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"ProjectName":"test_project-3d91c4a2-ixp","Title":"test project","TaxonomyImported":false}} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | Create a blank IXP project named "test project". - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: cli_called + - type: file_matches_regex description: "Agent created a blank project via `projects create \"test project\" … --skip-taxonomy`" - verb: "ixp projects create" - tool: "uip" - positional: ["test project"] - flags: - skip-taxonomy: {present: true} - value_flags: ["description", "d", "output"] + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+projects\s+create\b(?=.*test project)(?=.*--skip-taxonomy).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml b/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml index 5e42cbe4ec..84ed70c3a6 100644 --- a/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_project_suggest_taxonomy.yaml @@ -4,81 +4,44 @@ description: > the documents. Trap: don't pass --skip-taxonomy or hand-author/import a taxonomy — IXP suggests it. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects create", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"ProjectName":"ap_invoices-8c52f7b0-ixp","Title":"ap-invoices","DocumentsUploaded":3,"TaxonomyImported":true,"TaxonomySource":"suggested"}} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" - - # ./invoices/ is the folder the prompt names and `projects create` takes as - # its second positional. Nothing lived there while every call failed; now - # that create succeeds, an agent that checks the folder must find it. + mock_path_dirs: [mocks] template_sources: - - type: starter_files - files: - - path: invoices/hp_tax_invoice.png - content: | - offline-smoke placeholder - the generated `uip` stub never reads this file - - path: invoices/york_solutions_invoice.png - content: | - offline-smoke placeholder - the generated `uip` stub never reads this file - - path: invoices/csi_tower_invoice.png - content: | - offline-smoke placeholder - the generated `uip` stub never reads this file + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | Create an IXP project named ap-invoices from the invoices in ./invoices/, and let IXP suggest the taxonomy. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: cli_called + - type: file_matches_regex description: "Agent created ap-invoices via `projects create` from ./invoices/" - verb: "ixp projects create" - tool: "uip" - positional: ["ap-invoices"] - value_flags: ["description", "d", "output"] + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+projects\s+create\b(?=.*ap-invoices).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 # Autopilot path: let IXP suggest the taxonomy. --skip-taxonomy or a manual - # import would defeat the test. One regex alternation became two guards, since - # cli_called cannot OR across facet sets; 0.5 each keeps the old 1.0 total. - # - # `present` is load-bearing on the first: a value-bearing --skip-taxonomy would - # swallow the folder path, and the guard would pass on the call it forbids. - - type: cli_called - description: "Agent did NOT skip the taxonomy on create (autopilot suggests it)" - verb: "ixp projects create" - tool: "uip" - flags: - skip-taxonomy: {present: true} - min_count: 0 - max_count: 0 - weight: 0.5 - pass_threshold: 1.0 - - - type: cli_called - description: "Agent did NOT hand-import a taxonomy (autopilot suggests it)" - verb: "ixp projects import-taxonomy" - tool: "uip" - min_count: 0 - max_count: 0 - weight: 0.5 + # import would defeat the test. + - type: file_matches_regex + description: "Agent did NOT skip or hand-import the taxonomy (autopilot suggests it)" + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+projects\s+(create\b.*--skip-taxonomy|import-taxonomy)\b)[^\r\n]*$ + must_match: false + weight: 1.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml index 8f36d79ba8..d9c29d7f4b 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml @@ -5,44 +5,32 @@ task_id: skill-ixp-delete-document-by-id-smoke description: > Smoke: agent deletes a document by id — irreversible, triggers a retrain. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - # Orientation: the agent may confirm the document exists before deleting it. - - when: {verb: "ixp documents list", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Documents":[{"DocumentId":"doc-abc-123","AttachmentRef":"att-9f31c7","Filename":"hp_tax_invoice.png"},{"DocumentId":"doc-def-456","AttachmentRef":"att-40b2ae","Filename":"york_solutions_invoice.png"},{"DocumentId":"doc-ghi-789","AttachmentRef":null,"Filename":"csi_tower_invoice.png"}],"Total":3,"Offset":0,"Limit":50}} - - when: {verb: "ixp documents delete", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"DocumentId":"doc-abc-123","Deleted":true}} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete document doc-abc-123. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: cli_called + - type: file_matches_regex description: "Agent deleted the document via `documents delete doc-abc-123 -y`" - verb: "ixp documents delete" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] - flags: - # Quoted: bare `yes` is a YAML 1.1 boolean. - "yes": {present: true, aliases: ["y"]} + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+documents\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*doc-abc-123)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml index 2f91285cc4..c84b44c704 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml @@ -4,46 +4,33 @@ description: > annotations on all its fields, so it must pass -y/--yes (the CLI refuses without it). - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"1d8e42b7c3a95f60","id":"c128ad65bd8af3f8","field_type_id":"c128ad65bd8af3f8","kind":"Date","name":"Invoice Date","instructions":"The issue date of the invoice.","rule_set":null},{"field_id":"6e0b74a1f83c2d95","id":"e5d7402ba91c6f38","field_type_id":"e5d7402ba91c6f38","kind":"Monetary Quantity","name":"Total Amount","instructions":"The grand total payable on the invoice, including tax.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]},{"name":"name group","id":"b41d8e27f0a3c659","instructions":"Legacy group kept from an earlier taxonomy import.","moon_form":[{"field_id":"70f2c58ad916be34","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"full name","instructions":"The payer's full name as printed.","rule_set":null}]}]}]}}} - - when: {verb: "ixp groups delete", value_flags: ["output"]} - stdout: | - {"Result":"Success"} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the field group "name group". - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: cli_called + - type: file_matches_regex description: "Agent deleted the group via `groups delete … -y/--yes`" - verb: "ixp groups delete" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp"] - flags: - name: "name group" - # Quoted: bare `yes` is a YAML 1.1 boolean, and the key must reach the - # grader as the string flag name. - "yes": {present: true, aliases: ["y"]} + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+groups\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--name\s+["']?name group)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml index 4dd8fc04c4..76262e9881 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml @@ -4,57 +4,41 @@ description: > `fields delete` removes one field per call, so the agent must issue both. Trap: deleting only one and reporting done. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 6 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects get-taxonomy", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"status":"ready","dataset":{"entity_defs":[{"id":"b3a63e6483adaaba","name":"Exact Text","title":"","inherits_from":[],"trainable":true,"color":2,"instructions":""},{"id":"c128ad65bd8af3f8","name":"Date","title":"","inherits_from":["0000000000000007"],"trainable":true,"color":1,"instructions":""},{"id":"e5d7402ba91c6f38","name":"Monetary Quantity","title":"","inherits_from":["0000000000000005"],"trainable":true,"color":4,"instructions":""}],"label_groups":[{"name":"default","label_defs":[{"name":"Invoice Header","id":"5f21c88ab90d4e17","instructions":"Header-level fields of the supplier invoice.","moon_form":[{"field_id":"9a3f5d2c81b7e604","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Invoice Number","instructions":"The unique invoice identifier printed in the document header.","rule_set":null},{"field_id":"3c7e91a5d284b06f","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"field1","instructions":"Placeholder field carried over from an earlier import.","rule_set":null},{"field_id":"8b05d3f7c1927ae4","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"field2","instructions":"Placeholder field carried over from an earlier import.","rule_set":null},{"field_id":"d692af41b7350e8c","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"field12","instructions":"Placeholder field carried over from an earlier import.","rule_set":null}]},{"name":"Line Items","id":"7be409c2d15fa833","instructions":"One entry per line item row on the invoice.","moon_form":[{"field_id":"c47a91e05d2b3f68","id":"b3a63e6483adaaba","field_type_id":"b3a63e6483adaaba","kind":"Exact Text","name":"Description","instructions":"The line item description as printed on the row.","rule_set":null}]}]}]}}} - - when: {verb: "ixp fields delete", value_flags: ["output"]} - stdout: | - {"Result":"Success"} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the fields "field1" and "field2" from the "Invoice Header" group. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # `equals` on --field, not the old line-wide substring: `(?=.*field1)` was also - # satisfied by `--field field12`, and by field1 appearing in --group. - - type: cli_called + - type: file_matches_regex description: "Agent deleted field1 via `fields delete`" - verb: "ixp fields delete" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp"] - flags: - field: "field1" - value_flags: ["group", "output"] + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field1).*)[^\r\n]*$ weight: 2.0 pass_threshold: 1.0 - - type: cli_called + - type: file_matches_regex description: "Agent deleted field2 via `fields delete` (both, not just one)" - verb: "ixp fields delete" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp"] - flags: - field: "field2" - value_flags: ["group", "output"] + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field2).*)[^\r\n]*$ weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_project.yaml b/tests/tasks/uipath-ixp/smoke/delete_project.yaml index 8e7c590bf8..1e9a56bec6 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_project.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_project.yaml @@ -4,45 +4,32 @@ description: > pass -y/--yes (the CLI refuses without it), acknowledging the data loss. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:build, lifecycle:setup] run_limits: expected_turns: 5 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - # Orientation: the agent may confirm the project exists before deleting it. - - when: {verb: "ixp projects list", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Projects":[{"Id":"f1afa9ef-1c4d-4b70-9e02-6d3b8a5c7e11","Name":"my_invoices-f1afa9ef-ixp","Title":"My_Invoices","CreatedAt":"2026-06-11T08:42:19Z"},{"Id":"7b2c5d19-8e34-4a61-b0f7-2c9d4e6a1b83","Name":"quarterly_reports-7b2c-ixp","Title":"Quarterly Reports","CreatedAt":"2026-07-02T14:05:40Z"}],"Total":2,"Offset":0,"Limit":50}} - - when: {verb: "ixp projects delete", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Name":"my_invoices-f1afa9ef-ixp","Deleted":true}} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | Delete the IXP project my_invoices-f1afa9ef-ixp. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: cli_called + - type: file_matches_regex description: "Agent deleted the project via `projects delete … -y/--yes`" - verb: "ixp projects delete" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp"] - flags: - # Quoted: bare `yes` is a YAML 1.1 boolean. `present` also keeps the switch - # from consuming the project name that follows it in `delete -y `. - "yes": {present: true, aliases: ["y"]} + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+projects\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml index 49a601744e..00686e99b5 100644 --- a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml +++ b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml @@ -2,44 +2,34 @@ task_id: skill-ixp-mark-missing-smoke description: > Smoke: agent records a field as missing with `labellings mark-missing`. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: expected_turns: 6 sandbox: - record_cli: - - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp labellings get-predictions", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"ProjectName":"my_invoices-f1afa9ef-ixp","TotalDocuments":1,"DocumentsWithPredictions":1,"Predictions":[{"DocumentId":"doc-abc-123","ModelVersion":7,"Labels":[{"Name":"Invoice Header","Occurrence":0,"Fields":[{"FieldId":"9a3f5d2c81b7e604","FieldName":"Invoice Number","FormattedValue":"INV-2025-0042"},{"FieldId":"6e0b74a1f83c2d95","FieldName":"Total Amount","FormattedValue":"1200.00"},{"FieldId":"a7c3e9105f2b4d86","FieldName":"Discount Code","FormattedValue":""}]}]}]}} - - when: {verb: "ixp labellings mark-missing", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Unmatched":[]}} - - tool: curl - stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" + mock_path_dirs: [mocks] + template_sources: + - {type: template_dir, path: ../_shared/mock_template} initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, document doc-abc-123, the field "Discount Code" (field id a7c3e9105f2b4d86) is genuinely absent from the document and the model predicted no value for it. Record it as missing. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: cli_called + - type: file_matches_regex description: "Agent marked the field missing via `labellings mark-missing` on the right project + document + field id" - verb: "ixp labellings mark-missing" - tool: "uip" - positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] - flags: - fields: {contains: "a7c3e9105f2b4d86", aliases: ["f"]} + path: "mocks/calls.log" + pattern: >- + (?m)^(?:uip\s+ixp\s+labellings\s+mark-missing\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*doc-abc-123)(?=.*a7c3e9105f2b4d86).*)[^\r\n]*$ weight: 3.0 pass_threshold: 1.0 From e13f9671a8386fff09477a01709a27bb3a56f607 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 15:49:02 +0300 Subject: [PATCH 4/9] test(ixp): drop the speculative `projects list` stub from list_documents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task's prompt hands the agent `my_invoices-f1afa9ef-ixp`, which per the skill's Critical Rule 7 is the project `Name` — the lowercase slug with UUID and `-ixp` suffix that every CLI command takes. There is nothing to resolve, so `projects list` is not on this task's correct path; I served it on the guess that an agent might orient first. Two things wrong with that. It contradicts the rule the rest of this batch follows — serve the calls the correct path makes and nothing else — and it blurs a tier boundary the suite maintains on purpose. From `integration/name_resolution.yaml`: pre_run ... hands the agent nothing but its TITLE, so both lookups are forced — Title → ProjectName via `projects list` ... The smoke tasks pass the id in the prompt; resolving it is what makes this the integration tier. Serving `projects list` here makes that lookup succeed in a smoke task, which is the affordance the integration tier exists to provide. Each of the three tasks now serves exactly one call: the one it grades. Re-verified: 3 invocations replayed through the executed shims, all answered by a response rule, none falling through; all 3 `cli_called` criteria still pass; the show_predictions wrong-project trajectory still scores 0.0. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/list_documents.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/list_documents.yaml b/tests/tasks/uipath-ixp/smoke/list_documents.yaml index d54f2b8721..f82b948614 100644 --- a/tests/tasks/uipath-ixp/smoke/list_documents.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_documents.yaml @@ -17,10 +17,6 @@ sandbox: # still looks like a CLI that refused rather than one that agreed. stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" responses: - # Orientation: the agent may resolve the project before listing its documents. - - when: {verb: "ixp projects list", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Projects":[{"Id":"f1afa9ef-1c4d-4b70-9e02-6d3b8a5c7e11","Name":"my_invoices-f1afa9ef-ixp","Title":"My_Invoices","CreatedAt":"2026-06-11T08:42:19Z"},{"Id":"7b2c5d19-8e34-4a61-b0f7-2c9d4e6a1b83","Name":"quarterly_reports-7b2c-ixp","Title":"Quarterly Reports","CreatedAt":"2026-07-02T14:05:40Z"}],"Total":2,"Offset":0,"Limit":50}} - when: {verb: "ixp documents list", value_flags: ["output"]} stdout: | {"Result":"Success","Data":{"Documents":[{"DocumentId":"doc-abc-123","AttachmentRef":"att-9f31c7","Filename":"hp_tax_invoice.png"},{"DocumentId":"doc-def-456","AttachmentRef":"att-40b2ae","Filename":"york_solutions_invoice.png"},{"DocumentId":"doc-ghi-789","AttachmentRef":null,"Filename":"csi_tower_invoice.png"}],"Total":3,"Offset":0,"Limit":50}} From dbc6bba694f32284c8cf029c50c572f7cf9eeebf Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 16:01:14 +0300 Subject: [PATCH 5/9] =?UTF-8?q?test(ixp):=20drop=20the=20response=20tables?= =?UTF-8?q?=20=E2=80=94=20this=20batch=20does=20not=20need=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I added `responses:` to these three on the argument that a stub which fails everything forces the prompt to carry "commands will fail — do not retry", and that the hint is worth removing. The evidence does not support paying for it here. The 13-task smoke run on this branch scored 100% against the stub that failed EVERY call, so agents were not derailed by the failure; the sandbox framing in the prompt did its job. And none of the three criteria needs a served value — each asserts a verb plus positionals, nothing the agent could only have carried from a read: list_projects verb only list_documents verb + positional show_predictions verb + two ordered positionals Every one of these tasks also states its own contract as "Grades command shape, not wire effect". A canned answer is wire effect. Ablating the rule confirmed the point mechanically: `cli_called` scores 1.0 whether the stub answers with JSON and exit 0 or with the fallback and exit 1, because the recorder writes the argv record either way. So the response tables were buying honesty in the environment, not signal in the grade, and the cost was real: ~1 KB of invented JSON per task to review, and a fixture universe to keep consistent with `_shared/mock_template_taxonomy`. The three tasks go back to what this PR was for — the `cli_called` migration and nothing else. Prompts keep the sandbox framing they always had. `responses:` belongs in the PR that makes a criterion assert a CARRIED value — `fields add --type` resolved against the project's entity_defs, say — because that is the task an agent cannot complete against a failing read. Noted as such in the README rather than demonstrated prematurely here. The README correction stays: the merge pins coder-eval 0.12.1, so the claim that `record_cli` cannot serve per-invocation responses is stale on main regardless of this PR. It now also says why no task uses it yet, and what would justify it. Verified: 3 correct trajectories score 1.0; `projects list-models` still fails the `projects list` criterion (token-wise verb matching) and the wrong-project `get-predictions` still scores 0.0. check-cli-verbs 0 High/0 Medium, check-task-driver clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../_shared/mock_template/README.md | 21 +++++++++++-------- .../uipath-ixp/smoke/list_documents.yaml | 19 +++++++---------- .../tasks/uipath-ixp/smoke/list_projects.yaml | 20 ++++++++---------- .../uipath-ixp/smoke/show_predictions.yaml | 19 +++++++---------- 4 files changed, 37 insertions(+), 42 deletions(-) diff --git a/tests/tasks/uipath-ixp/_shared/mock_template/README.md b/tests/tasks/uipath-ixp/_shared/mock_template/README.md index 30380c4c49..f8e8ca9ed4 100644 --- a/tests/tasks/uipath-ixp/_shared/mock_template/README.md +++ b/tests/tasks/uipath-ixp/_shared/mock_template/README.md @@ -86,18 +86,21 @@ Tasks graded with `cli_called` declare `sandbox.record_cli` instead, and the framework generates the recorders, seeds the log and PATH-prepends `cli_mocks/` itself — no `mock_path_dirs`, no `template_sources`, no `log:` on the criterion. Prefer that for a new structured task. Since coder-eval 0.12.1 a `record_cli` -entry also carries `responses:` — per-invocation canned answers matched by the +entry can also carry `responses:` — per-invocation canned answers matched by the same facets `cli_called` grades with — so serving a read before the graded write -no longer needs this template either (the smoke tasks under `../../smoke/` that -declare `record_cli` do exactly that). This template remains the answer whenever -a recorder is still not enough: `file_matches_regex` over the flat `calls.log`, -or a served read that must be STATEFUL, where the answer depends on what the -agent already wrote (`mock_template_taxonomy`) — `select_rule` is stateless by -design. `calls.jsonl` here has no consumer today — it is the structured sink for -tasks that need this template's other behavior too. +no longer requires this template. No task uses that yet: a shape-only smoke task +grades the invocation, not the answer, so the blanket failure below is enough for +it. Reach for `responses:` when a criterion asserts a value the agent could only +have CARRIED from a read. This template remains the answer whenever a recorder is +still not enough: `file_matches_regex` over the flat `calls.log`, or a served read +that must be STATEFUL, where the answer depends on what the agent already wrote +(`mock_template_taxonomy`) — `select_rule` is stateless by design. `calls.jsonl` +here has no consumer today — it is the structured sink for tasks that need this +template's other behavior too. A task whose correct path reads before it writes cannot be graded on this mock alone — the read fails, so there is nothing to carry into the graded write, and an agent that declines to invent the values correctly stops. Overlay `mock_template_taxonomy` (listed second) to serve that read, or — for a task on -`record_cli` — declare the read as a `responses:` rule on the entry itself. +`record_cli` whose grading needs the carried value — declare the read as a +`responses:` rule on the entry itself. diff --git a/tests/tasks/uipath-ixp/smoke/list_documents.yaml b/tests/tasks/uipath-ixp/smoke/list_documents.yaml index f82b948614..86ebfc5b4f 100644 --- a/tests/tasks/uipath-ixp/smoke/list_documents.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_documents.yaml @@ -2,8 +2,8 @@ task_id: skill-ixp-list-documents-smoke description: > Smoke: agent lists the documents in a project. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: @@ -12,21 +12,18 @@ run_limits: sandbox: record_cli: - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp documents list", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Documents":[{"DocumentId":"doc-abc-123","AttachmentRef":"att-9f31c7","Filename":"hp_tax_invoice.png"},{"DocumentId":"doc-def-456","AttachmentRef":"att-40b2ae","Filename":"york_solutions_invoice.png"},{"DocumentId":"doc-ghi-789","AttachmentRef":null,"Filename":"csi_tower_invoice.png"}],"Total":3,"Offset":0,"Limit":50}} + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, list all documents. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - type: cli_called diff --git a/tests/tasks/uipath-ixp/smoke/list_projects.yaml b/tests/tasks/uipath-ixp/smoke/list_projects.yaml index 85e3d864b3..2070795e6e 100644 --- a/tests/tasks/uipath-ixp/smoke/list_projects.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_projects.yaml @@ -3,8 +3,8 @@ description: > Smoke: agent picks `uip ixp projects list --output json` to enumerate IXP projects on the tenant. - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:discover] run_limits: @@ -13,14 +13,7 @@ run_limits: sandbox: record_cli: - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp projects list", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"Projects":[{"Id":"f1afa9ef-1c4d-4b70-9e02-6d3b8a5c7e11","Name":"my_invoices-f1afa9ef-ixp","Title":"My_Invoices","CreatedAt":"2026-06-11T08:42:19Z"},{"Id":"7b2c5d19-8e34-4a61-b0f7-2c9d4e6a1b83","Name":"quarterly_reports-7b2c-ixp","Title":"Quarterly Reports","CreatedAt":"2026-07-02T14:05:40Z"}],"Total":2,"Offset":0,"Limit":50}} + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -28,7 +21,12 @@ initial_prompt: | A document automation engineer wants to see every IXP project on the tenant before deciding which one to iterate on. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant + in this environment. Commands will fail with auth errors — that is + expected. Run each command once and move on. Do NOT retry or attempt + to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: # No positional facet: `projects list` takes none. The verb is matched token-wise, diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index 3578977152..d5135fe744 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -3,8 +3,8 @@ description: > Smoke: agent reads the model's predicted values for one document (the prediction layer, via get-predictions — not by reading the file itself). - Offline: `uip` is a generated stub that answers this task's calls from - canned responses. Grades command shape, not wire effect. + Harness is unauthenticated; CLI calls fail with auth errors. + Grades command shape, not wire effect. tags: [uipath-ixp, smoke, mode:operate, lifecycle:setup] run_limits: @@ -13,14 +13,7 @@ run_limits: sandbox: record_cli: - tool: uip - # Every call the correct path makes is answered below. Anything else - # falls through to this fallback and fails, so an off-path invocation - # still looks like a CLI that refused rather than one that agreed. - stderr: "uip (mock): offline sandbox - this call is not one the task stubs. Continue with the task; do not retry or log in.\n" - responses: - - when: {verb: "ixp labellings get-predictions", value_flags: ["output"]} - stdout: | - {"Result":"Success","Data":{"ProjectName":"my_invoices-f1afa9ef-ixp","TotalDocuments":1,"DocumentsWithPredictions":1,"Predictions":[{"DocumentId":"doc-abc-123","ModelVersion":7,"Labels":[{"Name":"Invoice Header","Occurrence":0,"Fields":[{"FieldId":"9a3f5d2c81b7e604","FieldName":"Invoice Number","FormattedValue":"INV-2025-0042"},{"FieldId":"1d8e42b7c3a95f60","FieldName":"Invoice Date","FormattedValue":"2025-11-04"},{"FieldId":"6e0b74a1f83c2d95","FieldName":"Total Amount","FormattedValue":"1200.00"}]},{"Name":"Line Items","Occurrence":0,"Fields":[{"FieldId":"c47a91e05d2b3f68","FieldName":"Description","FormattedValue":"Annual support subscription"}]}]}]}} + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" - tool: curl stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" @@ -28,7 +21,11 @@ initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, show the values the model predicted for document doc-abc-123. - Do NOT prompt the user for confirmation — this is an automated test. + Important: + - The `uip` CLI is available but is NOT connected to a live IXP tenant in + this environment. Commands will fail with auth errors — that is expected. + Run each command once and move on. Do NOT retry or attempt to login. + - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: # The project is named because positionals are ordered: doc-abc-123 can only be From cf5e5f043b85112d9e49eee1daa78805b4afac2a Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 16:15:44 +0300 Subject: [PATCH 6/9] test(ixp): name the project slot in the show_predictions description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The criterion asserts both positionals — `my_invoices-f1afa9ef-ixp` then `doc-abc-123` — but the description elided the first as `…`, so it described a weaker assertion than the one being made. That is backwards for the one thing this migration changed about the task: the project is asserted precisely BECAUSE positionals are ordered, and pinning the document slot requires naming the slot before it. It also matters on failure. The description is what the eval report prints, so an agent that ran `get-predictions` on the wrong project would have been failed by a line that never mentions the project. Follows `delete_document_by_id` from #2565 — same two positionals, same concrete document id — which already established the form: documents delete doc-abc-123 -y labellings get-predictions doc-abc-123 `` is the placeholder the suite uses for the project positional (`documents list `, `projects untag --tag live`, `projects import-taxonomy ./taxonomy.json`). Description text only; no change to what is graded. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/show_predictions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index d5135fe744..2f4fe378ee 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -32,7 +32,7 @@ success_criteria: # asserted as the DOCUMENT argument by pinning the slot before it. The regex this # replaced matched the id anywhere, so a get-predictions on another project counted. - type: cli_called - description: "Agent fetched predictions via `labellings get-predictions … doc-abc-123`" + description: "Agent fetched predictions via `labellings get-predictions doc-abc-123`" verb: "ixp labellings get-predictions" tool: "uip" positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] From 28d5a16998b5644c612df788a9e9672a3e15b434 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 16:21:22 +0300 Subject: [PATCH 7/9] test(ixp): write asserted positionals literally in cli_called descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `` told the reader nothing. Across the ixp smoke suite it appears on both sides of the distinction it looks like it is drawing: ASSERTED cli_called deployments create --version 13 ASSERTED cli_called documents list not asserted file_matches_regex projects untag --tag live not asserted file_matches_regex documents delete doc-abc-123 -y Five criteria pin the project behind that placeholder and eight do not, so it carries no information about whether the slot is matched. Worse, I took the form from `delete_document_by_id`, which is still on `file_matches_regex`. It writes `` because its regex genuinely does NOT pin the project — the placeholder is accurate there. Copying it onto a migrated criterion inverted the meaning: it read as "any project" on the one assertion this migration exists to add. So both slots are now written as the values actually matched: documents list my_invoices-f1afa9ef-ixp labellings get-predictions my_invoices-f1afa9ef-ixp doc-abc-123 which also gives the suite a rule worth keeping: in a `cli_called` description, a literal means the slot is pinned and a placeholder means it is not. That distinction is currently unreadable, and it is the one a reviewer needs when an eval report says a criterion failed. Three `cli_called` criteria already on main still use `` for an asserted project (`deploy_folder_by_name`, `deploy_to_folder`, `upgrade_deployment` — the last with the same split I had, placeholder project beside a literal deployment name). Left alone: none is a file this PR touches. Description text only; nothing graded changed. Re-verified: 3 correct trajectories score 1.0, `projects list-models` and the wrong-project `get-predictions` still score 0.0. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/list_documents.yaml | 2 +- tests/tasks/uipath-ixp/smoke/show_predictions.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/list_documents.yaml b/tests/tasks/uipath-ixp/smoke/list_documents.yaml index 86ebfc5b4f..efe2368146 100644 --- a/tests/tasks/uipath-ixp/smoke/list_documents.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_documents.yaml @@ -27,7 +27,7 @@ initial_prompt: | success_criteria: - type: cli_called - description: "Agent listed documents via `documents list `" + description: "Agent listed documents via `documents list my_invoices-f1afa9ef-ixp`" verb: "ixp documents list" tool: "uip" positional: ["my_invoices-f1afa9ef-ixp"] diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index 2f4fe378ee..239ac1d9d1 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -32,7 +32,7 @@ success_criteria: # asserted as the DOCUMENT argument by pinning the slot before it. The regex this # replaced matched the id anywhere, so a get-predictions on another project counted. - type: cli_called - description: "Agent fetched predictions via `labellings get-predictions doc-abc-123`" + description: "Agent fetched predictions via `labellings get-predictions my_invoices-f1afa9ef-ixp doc-abc-123`" verb: "ixp labellings get-predictions" tool: "uip" positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] From 9a708a10080f30dc26bbf88af323a6cd412b2284 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 16:24:25 +0300 Subject: [PATCH 8/9] test(ixp): trim criterion comments to the why Both restated the YAML above them before getting to the point. Keep the reason, drop the description of what the fields already say. list_projects 2 lines -> 1 show_predictions 3 lines -> 2 The replaced-regex history stays in the commit and PR body, where it is context; the file only needs why the criterion is shaped this way. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/list_projects.yaml | 3 +-- tests/tasks/uipath-ixp/smoke/show_predictions.yaml | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/list_projects.yaml b/tests/tasks/uipath-ixp/smoke/list_projects.yaml index 2070795e6e..03bcc7e093 100644 --- a/tests/tasks/uipath-ixp/smoke/list_projects.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_projects.yaml @@ -29,8 +29,7 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # No positional facet: `projects list` takes none. The verb is matched token-wise, - # so it does not credit the near-name `projects list-models`. + # Token-wise verb match, so `projects list-models` does not credit this. - type: cli_called description: "Agent invoked `uip ixp projects list`" verb: "ixp projects list" diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index 239ac1d9d1..24312c8701 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -28,9 +28,8 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # The project is named because positionals are ordered: doc-abc-123 can only be - # asserted as the DOCUMENT argument by pinning the slot before it. The regex this - # replaced matched the id anywhere, so a get-predictions on another project counted. + # Positionals are ordered: doc-abc-123 only means the document argument + # if the slot before it is pinned. - type: cli_called description: "Agent fetched predictions via `labellings get-predictions my_invoices-f1afa9ef-ixp doc-abc-123`" verb: "ixp labellings get-predictions" From 3255f499c0417e9f74f06286faec862a0cca3b43 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Mon, 14 Sep 2026 16:27:01 +0300 Subject: [PATCH 9/9] test(ixp): drop the framework-explaining criterion comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both described how cli_called matches, not why these criteria are shaped this way. coder-eval already documents token-wise verb matching at the implementation, with the same example and a validation guard, so the comment duplicated upstream and would have gone stale silently if upstream changed. It is also not what the #2565 comments do. Those explain task-local decisions — why a guard carries few facets, why snake_case matters for this skill, why `tool` is required when uip and curl share one log. These three criteria are verb plus positionals with no such decision behind them, so they need no comment at all. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/list_projects.yaml | 1 - tests/tasks/uipath-ixp/smoke/show_predictions.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/list_projects.yaml b/tests/tasks/uipath-ixp/smoke/list_projects.yaml index 03bcc7e093..723b3c2a36 100644 --- a/tests/tasks/uipath-ixp/smoke/list_projects.yaml +++ b/tests/tasks/uipath-ixp/smoke/list_projects.yaml @@ -29,7 +29,6 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # Token-wise verb match, so `projects list-models` does not credit this. - type: cli_called description: "Agent invoked `uip ixp projects list`" verb: "ixp projects list" diff --git a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml index 24312c8701..8beb40149f 100644 --- a/tests/tasks/uipath-ixp/smoke/show_predictions.yaml +++ b/tests/tasks/uipath-ixp/smoke/show_predictions.yaml @@ -28,8 +28,6 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - # Positionals are ordered: doc-abc-123 only means the document argument - # if the slot before it is pinned. - type: cli_called description: "Agent fetched predictions via `labellings get-predictions my_invoices-f1afa9ef-ixp doc-abc-123`" verb: "ixp labellings get-predictions"