diff --git a/tests/tasks/uipath-ixp/smoke/create_field.yaml b/tests/tasks/uipath-ixp/smoke/create_field.yaml index 55d0c14a9e..344ff4c495 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, add a new field named "email" @@ -25,10 +27,15 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent added field \"email\" to \"Subscriber information\" via `fields add`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--group\s+["']?Subscriber information)(?=.*--field\s+["']?email).*)[^\r\n]*$ + - type: cli_called + description: "Agent added the field via `fields add my_invoices-f1afa9ef-ixp --group 'Subscriber information' --field email`, with a non-empty --type and --instructions" + verb: "ixp fields add" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + group: "Subscriber information" + field: "email" + type: {matches_regex: ".+"} + instructions: {matches_regex: ".+"} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml index 76262e9881..88409f4225 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 6 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the fields "field1" and @@ -27,18 +29,26 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent deleted field1 via `fields delete`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field1).*)[^\r\n]*$ + - type: cli_called + description: "Agent deleted field1 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field1 -y/--yes`" + verb: "ixp fields delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + group: "Invoice Header" + field: "field1" + "yes": {present: true, aliases: ["y"]} weight: 2.0 pass_threshold: 1.0 - - type: file_matches_regex - description: "Agent deleted field2 via `fields delete` (both, not just one)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field2).*)[^\r\n]*$ + - type: cli_called + description: "Agent deleted field2 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field2 -y/--yes` (both, not just one)" + verb: "ixp fields delete" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + flags: + group: "Invoice Header" + field: "field2" + "yes": {present: true, aliases: ["y"]} weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml index 00686e99b5..f056ebf54a 100644 --- a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml +++ b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 6 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, document doc-abc-123, the field @@ -26,10 +28,23 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent marked the field missing via `labellings mark-missing` on the right project + document + field id" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+mark-missing\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*doc-abc-123)(?=.*a7c3e9105f2b4d86).*)[^\r\n]*$ + - type: cli_called + description: "Agent marked the field missing via `labellings mark-missing my_invoices-f1afa9ef-ixp doc-abc-123 --fields a7c3e9105f2b4d86`" + verb: "ixp labellings mark-missing" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] + flags: + fields: {equals: "a7c3e9105f2b4d86", aliases: ["f"]} weight: 3.0 pass_threshold: 1.0 + + - type: cli_called + description: "Did NOT mark any field other than a7c3e9105f2b4d86 missing" + verb: "ixp labellings mark-missing" + tool: "uip" + flags: + fields: {matches_regex: "^(?!a7c3e9105f2b4d86$)", aliases: ["f"]} + min_count: 0 + max_count: 0 + weight: 1.5 + pass_threshold: 1.0