From 8839cb813d5b9e6eb9905aa6ff054257a45a1d47 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Tue, 15 Sep 2026 14:25:20 +0300 Subject: [PATCH 1/4] test(ixp): grade the value-predicate smokes via cli_called The flat calls.log pattern tested for a SUBSTRING anywhere on the line, so a value that merely extended the target passed: --field email2 credited "email", --group "Subscriber information 2" credited the sibling group, and the lookaheads being order-free let field1 satisfy its criterion from the --group slot or from inside the project slug. Matching against recorded argv pins each value to its own flag and compares it whole. create_field declares value_flags for --type/--instructions because an undeclared flag stays a switch and leaves its value in the positionals, which shifts the project slug out of its slot when either flag is written before it. Co-Authored-By: Claude Opus 5 (1M context) --- .../tasks/uipath-ixp/smoke/create_field.yaml | 22 +++++++++----- .../smoke/delete_multiple_fields.yaml | 30 +++++++++++-------- .../tasks/uipath-ixp/smoke/mark_missing.yaml | 20 ++++++++----- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/create_field.yaml b/tests/tasks/uipath-ixp/smoke/create_field.yaml index 55d0c14a9e..9b9037a762 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 5 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, add a new field named "email" @@ -25,10 +27,14 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent added field \"email\" to \"Subscriber information\" via `fields add`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+add\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*--group\s+["']?Subscriber information)(?=.*--field\s+["']?email).*)[^\r\n]*$ + - type: cli_called + description: "Agent added the field via `fields add my_invoices-f1afa9ef-ixp --group 'Subscriber information' --field email`" + verb: "ixp fields add" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] + value_flags: ["type", "instructions", "output"] + flags: + group: "Subscriber information" + field: "email" weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml index 76262e9881..3d1bb7a919 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml @@ -12,9 +12,11 @@ run_limits: expected_turns: 6 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, delete the fields "field1" and @@ -27,18 +29,20 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent deleted field1 via `fields delete`" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field1).*)[^\r\n]*$ + - type: cli_called + description: "Agent deleted field1 via `fields delete --field field1`" + verb: "ixp fields delete" + tool: "uip" + flags: + field: "field1" weight: 2.0 pass_threshold: 1.0 - - type: file_matches_regex - description: "Agent deleted field2 via `fields delete` (both, not just one)" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+fields\s+delete\b(?=.*field2).*)[^\r\n]*$ + - type: cli_called + description: "Agent deleted field2 via `fields delete --field field2` (both, not just one)" + verb: "ixp fields delete" + tool: "uip" + flags: + field: "field2" weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml index 00686e99b5..54225b73ea 100644 --- a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml +++ b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml @@ -10,9 +10,11 @@ run_limits: expected_turns: 6 sandbox: - mock_path_dirs: [mocks] - template_sources: - - {type: template_dir, path: ../_shared/mock_template} + record_cli: + - tool: uip + stderr: "uip (mock): not connected to a tenant in this sandbox; offline smoke mock.\n" + - tool: curl + stderr: "curl (mock): network egress is disabled in this offline smoke sandbox.\n" initial_prompt: | In the IXP project my_invoices-f1afa9ef-ixp, document doc-abc-123, the field @@ -26,10 +28,12 @@ initial_prompt: | - Do NOT prompt the user for confirmation — this is an automated test. success_criteria: - - type: file_matches_regex - description: "Agent marked the field missing via `labellings mark-missing` on the right project + document + field id" - path: "mocks/calls.log" - pattern: >- - (?m)^(?:uip\s+ixp\s+labellings\s+mark-missing\b(?=.*my_invoices-f1afa9ef-ixp)(?=.*doc-abc-123)(?=.*a7c3e9105f2b4d86).*)[^\r\n]*$ + - type: cli_called + description: "Agent marked the field missing via `labellings mark-missing my_invoices-f1afa9ef-ixp doc-abc-123 --fields a7c3e9105f2b4d86`" + verb: "ixp labellings mark-missing" + tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] + flags: + fields: {contains: "a7c3e9105f2b4d86", aliases: ["f"]} weight: 3.0 pass_threshold: 1.0 From 0e58b4089489548d4aa544ae54b656924ba19b0a Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Tue, 15 Sep 2026 15:12:05 +0300 Subject: [PATCH 2/4] test(ixp): assert project, group and exact field set on the deletes Both advisory-lint Low findings on the value-predicate batch. The delete criteria credited the right field names deleted from the WRONG project or the WRONG group, and mark-missing credited a batch that marked an extra field missing -- which the skill forbids, since mark-missing is only for fields the model predicted no value for. Asserting positional without group would false-FAIL the legal ordering `--group G ... `, because an undeclared group leaves its value in the positionals; naming it in flags makes it value-bearing, so the two move together and no value_flags declaration is needed. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml | 8 ++++++-- tests/tasks/uipath-ixp/smoke/mark_missing.yaml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml index 3d1bb7a919..07e965e9a0 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml @@ -30,19 +30,23 @@ initial_prompt: | success_criteria: - type: cli_called - description: "Agent deleted field1 via `fields delete --field field1`" + description: "Agent deleted field1 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field1`" verb: "ixp fields delete" tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] flags: + group: "Invoice Header" field: "field1" weight: 2.0 pass_threshold: 1.0 - type: cli_called - description: "Agent deleted field2 via `fields delete --field field2` (both, not just one)" + description: "Agent deleted field2 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field2` (both, not just one)" verb: "ixp fields delete" tool: "uip" + positional: ["my_invoices-f1afa9ef-ixp"] flags: + group: "Invoice Header" field: "field2" weight: 2.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml index 54225b73ea..2152fc13ef 100644 --- a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml +++ b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml @@ -34,6 +34,6 @@ success_criteria: tool: "uip" positional: ["my_invoices-f1afa9ef-ixp", "doc-abc-123"] flags: - fields: {contains: "a7c3e9105f2b4d86", aliases: ["f"]} + fields: {equals: "a7c3e9105f2b4d86", aliases: ["f"]} weight: 3.0 pass_threshold: 1.0 From 1c43910851f0fc0621a54b8d74c2bcb4f33cb79d Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Wed, 16 Sep 2026 10:26:30 +0300 Subject: [PATCH 3/4] test(ixp): require the flags these two commands cannot run without Both criteria credited a command the CLI refuses: fields add without --type/--instructions (both required, so no field is added) and fields delete without -y/--yes (required, the CLI never prompts). The delete now matches what the other destructive IXP smokes assert. The two add flags use a value predicate rather than present: true because a presence predicate is not value-bearing, so its value would fall back into the positionals and take the project slug slot. Asserting them that way also makes the explicit value_flags declaration redundant. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/create_field.yaml | 5 +++-- tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/tasks/uipath-ixp/smoke/create_field.yaml b/tests/tasks/uipath-ixp/smoke/create_field.yaml index 9b9037a762..344ff4c495 100644 --- a/tests/tasks/uipath-ixp/smoke/create_field.yaml +++ b/tests/tasks/uipath-ixp/smoke/create_field.yaml @@ -28,13 +28,14 @@ initial_prompt: | success_criteria: - type: cli_called - description: "Agent added the field via `fields add my_invoices-f1afa9ef-ixp --group 'Subscriber information' --field email`" + description: "Agent added the field via `fields add my_invoices-f1afa9ef-ixp --group 'Subscriber information' --field email`, with a non-empty --type and --instructions" verb: "ixp fields add" tool: "uip" positional: ["my_invoices-f1afa9ef-ixp"] - value_flags: ["type", "instructions", "output"] flags: group: "Subscriber information" field: "email" + type: {matches_regex: ".+"} + instructions: {matches_regex: ".+"} weight: 3.0 pass_threshold: 1.0 diff --git a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml index 07e965e9a0..88409f4225 100644 --- a/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml +++ b/tests/tasks/uipath-ixp/smoke/delete_multiple_fields.yaml @@ -30,23 +30,25 @@ initial_prompt: | success_criteria: - type: cli_called - description: "Agent deleted field1 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field1`" + description: "Agent deleted field1 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field1 -y/--yes`" verb: "ixp fields delete" tool: "uip" positional: ["my_invoices-f1afa9ef-ixp"] flags: group: "Invoice Header" field: "field1" + "yes": {present: true, aliases: ["y"]} weight: 2.0 pass_threshold: 1.0 - type: cli_called - description: "Agent deleted field2 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field2` (both, not just one)" + description: "Agent deleted field2 via `fields delete my_invoices-f1afa9ef-ixp --group 'Invoice Header' --field field2 -y/--yes` (both, not just one)" verb: "ixp fields delete" tool: "uip" positional: ["my_invoices-f1afa9ef-ixp"] flags: group: "Invoice Header" field: "field2" + "yes": {present: true, aliases: ["y"]} weight: 2.0 pass_threshold: 1.0 From eb9d0b204ecfbc3294a32d935837dd96829dab26 Mon Sep 17 00:00:00 2001 From: Alexandru Jircan Date: Wed, 16 Sep 2026 10:30:03 +0300 Subject: [PATCH 4/4] test(ixp): guard against marking any other field missing Flag predicates are existential over a repeated flag values, so `equals` rejects `--fields id1,id2` but not `--fields id1 -f id2`, and no FlagMatch predicate can require exactly one occurrence. A max_count 0 guard on any fields value that is not our id covers both spellings, and also the case where a separate later call marks a second field. mark-missing only applies to fields the model predicted no value for, so an extra id annotates a field the agent has no basis to annotate. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tasks/uipath-ixp/smoke/mark_missing.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml index 2152fc13ef..f056ebf54a 100644 --- a/tests/tasks/uipath-ixp/smoke/mark_missing.yaml +++ b/tests/tasks/uipath-ixp/smoke/mark_missing.yaml @@ -37,3 +37,14 @@ success_criteria: fields: {equals: "a7c3e9105f2b4d86", aliases: ["f"]} weight: 3.0 pass_threshold: 1.0 + + - type: cli_called + description: "Did NOT mark any field other than a7c3e9105f2b4d86 missing" + verb: "ixp labellings mark-missing" + tool: "uip" + flags: + fields: {matches_regex: "^(?!a7c3e9105f2b4d86$)", aliases: ["f"]} + min_count: 0 + max_count: 0 + weight: 1.5 + pass_threshold: 1.0