From 73da21e2ac18fa3d846153466abaeb71ba752d55 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Tue, 15 Sep 2026 11:13:43 +0300 Subject: [PATCH] test(ixp): grade the three -y/--yes delete smokes via cli_called Second risk class after #2994: verb + positionals + a boolean switch. Each of the three flat-log patterns credited a command that deleted the WRONG thing irreversibly, because its lookaheads match substrings anywhere on the space-joined line rather than a slot: projects delete my_invoices-f1afa9ef-ixp-backup -y (sibling slug) documents delete doc-abc-1234 -y (sibling id) documents delete doc-abc-123 -y (slots swapped) groups delete --name "name group 2" -y (sibling group) groups delete --name name group -y (unquoted: 2 argv) All five now score 0.0; the correct trajectories still score 1.0. The last one is the lossy-join case the mock README calls out: unquoted, `--name name group` reaches the CLI as two arguments and fails, yet the flat line is byte-identical to the quoted form. Only `argv` can tell them apart. Weights preserved 1:1. Prompts unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- .../smoke/delete_document_by_id.yaml | 20 +++++++++++------- .../uipath-ixp/smoke/delete_field_group.yaml | 21 ++++++++++++------- .../uipath-ixp/smoke/delete_project.yaml | 20 +++++++++++------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml index d9c29d7f4b..000719a573 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_document_by_id.yaml @@ -13,9 +13,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete document doc-abc-123. @@ -27,10 +29,12 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent deleted the document via `documents delete doc-abc-123 -y`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+documents\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*doc-abc-123)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ + - type: cli_called + description: "Agent deleted the document via `documents delete my_invoices-f1afa9ef-ixp doc-abc-123 -y/--yes`" + verb: "ixp documents delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] + flags: + "yes": {present: true, aliases: ["y"]} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml index c84b44c704..7ebba433f1 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_field_group.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the field group @@ -27,10 +29,13 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent deleted the group via `groups delete … -y/--yes`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+groups\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--name\s+["']?name group)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ + - type: cli_called + description: "Agent deleted the group via `groups delete my_invoices-f1afa9ef-ixp --name 'name group' -y/--yes`" + verb: "ixp groups delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + name: "name group" + "yes": {present: true, aliases: ["y"]} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_project.yaml b/tests/tasks/uipath-ixp/smoke/delete_project.yaml index 1e9a56bec6..090a1c4c54 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_project.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_project.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | Delete the IXP project my_invoices-f1afa9ef-ixp. @@ -26,10 +28,12 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent deleted the project via `projects delete … -y/--yes`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+projects\s+delete\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*(--yes\b|-y\b)).*)[^\r\n]*$ + - type: cli_called + description: "Agent deleted the project via `projects delete my_invoices-f1afa9ef-ixp -y/--yes`" + verb: "ixp projects delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + "yes": {present: true, aliases: ["y"]} weight: 3.0 pass_threshold: 1.0