From 1ed394f24303a8f401917370612750a156e26af4 Mon Sep 17 00:00:00 2001 From: rockymadden Date: Thu, 10 Sep 2026 18:12:52 -0600 Subject: [PATCH] test(uipath-maestro-flow): pin the two billing lookups to the Data Service connector Both prompts name only a "Data Service entity". Since #3041 that denotes two node families, and the eval tenant has the native flags on, so on 2026-09-10 both agents built `core.datafabric.read` and both structural advisories reported FAIL against a connector-shaped contract. The build was defensible; the prompt was ambiguous. Pins the connector in both, matching the sibling billing_dispute_resolution prompt, which has always said "two Data Service query-entity-records nodes" and is the one task in the family that never drifted. The reason given is real, not a checker hint: the four `canvas.nodes.*-entity` flags default to off (data-fabric/planning.md:58), so the connector is the path that works on every target tenant. Guard test beside the existing per-task prompt guards. Native authoring gets its own task rather than sharing these. Co-Authored-By: Claude Opus 5 (1M context) --- .../_shared/test_same_ground_corpus.py | 22 +++++++++++++++++++ .../billing_discrepancy_detector.yaml | 4 ++++ .../billing_invoice_lookup.yaml | 4 ++++ 3 files changed, 30 insertions(+) diff --git a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py index f934edb576..0c03540220 100644 --- a/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py +++ b/tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py @@ -42,6 +42,19 @@ DEBUG_SOLUTION_ALLOWLIST = set() +# The two billing lookups name only a "Data Service entity", which since #3041 +# denotes two node families. Their prompts pin the connector so the graded +# structure is deterministic; the sibling dispute-resolution task pins it in the +# same way. Native Data Fabric authoring is covered by its own task, not these. +CONNECTOR_PINNED_LOOKUPS = { + "multi_node/billing_invoice_lookup/billing_invoice_lookup.yaml", + "multi_node/billing_discrepancy_detector/billing_discrepancy_detector.yaml", +} +CONNECTOR_PIN = ( + "on the data service `query-entity-records` activity, " + "not the native data fabric nodes" +) + # F1 freezes the #2557 task even though its prompt predates the exact phrase. DEBUG_PROJECT_LAYOUT_EXCEPTIONS = {"single_node/coded_agent/coded_agent.yaml"} @@ -192,6 +205,15 @@ def test_debug_graded_prompts_name_a_same_name_solution() -> None: assert offenders == DEBUG_SOLUTION_ALLOWLIST | DEBUG_PROJECT_LAYOUT_EXCEPTIONS +def test_billing_lookup_prompts_pin_the_data_service_connector() -> None: + offenders = { + relative + for relative in CONNECTOR_PINNED_LOOKUPS + if CONNECTOR_PIN not in " ".join(_prompt_text(_task_text(relative)).lower().split()) + } + assert offenders == set() + + def test_external_graders_use_package_qualified_shared_imports() -> None: shared_modules = { path.stem for path in (FLOW_TASKS / "_shared").glob("*.py") diff --git a/tests/tasks/uipath-maestro-flow/multi_node/billing_discrepancy_detector/billing_discrepancy_detector.yaml b/tests/tasks/uipath-maestro-flow/multi_node/billing_discrepancy_detector/billing_discrepancy_detector.yaml index be39d6ca58..ec7e44c2e7 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/billing_discrepancy_detector/billing_discrepancy_detector.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/billing_discrepancy_detector/billing_discrepancy_detector.yaml @@ -43,6 +43,10 @@ initial_prompt: | `matchedInvoiceNumber` (string, first ERP row), `accountTier` (string, from CRM). + Build both lookups on the Data Service `query-entity-records` activity, not + the native Data Fabric nodes: the tenants this ships to do not all enable + them. + Validate the final flow file — the task is not complete until `uip maestro flow validate` passes. Grading then runs the flow under debug against live tenant data, so build it to actually run, not merely validate. diff --git a/tests/tasks/uipath-maestro-flow/multi_node/billing_invoice_lookup/billing_invoice_lookup.yaml b/tests/tasks/uipath-maestro-flow/multi_node/billing_invoice_lookup/billing_invoice_lookup.yaml index d84caaf1c7..4e3f4f26f2 100644 --- a/tests/tasks/uipath-maestro-flow/multi_node/billing_invoice_lookup/billing_invoice_lookup.yaml +++ b/tests/tasks/uipath-maestro-flow/multi_node/billing_invoice_lookup/billing_invoice_lookup.yaml @@ -31,6 +31,10 @@ initial_prompt: | - Outputs: `matchedInvoiceNumber` (string, first matched row's invoiceNumber) and `lineItemCount` (number of rows returned). + Build the lookup on the Data Service `query-entity-records` activity, not + the native Data Fabric nodes: the tenants this ships to do not all enable + them. + Validate the final flow file — the task is not complete until `uip maestro flow validate` passes. Grading then runs the flow under debug against live tenant data, so build it to actually run, not merely validate.