diff --git a/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py b/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py new file mode 100644 index 0000000000..99e5ca7132 --- /dev/null +++ b/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Fail a task as ERROR, not FAILURE, when the tenant connection it needs is down. + +Usage: + preflight_connections.py [ ...] + +A `pre_run` failure lands the run as ``FinalStatus.ERROR``; a criterion failure +lands it as ``FAILURE``. Without this, a revoked grant or an asleep tenant reads +as an agent mistake: + + skill-flow-outlook-trigger-inbox AADSTS50173, grant revoked 2026-08-31 + skill-flow-generic-dynamic-node ServiceNow developer instance hibernating + +Both were scored FAILURE on 2026-09-04 and root-caused as skill defects before +anyone read far enough into the checker output to find the 403. + +Passes when at least one connection for each key reports Enabled. Connections +live in several folders, so `--all-folders` is required; without it an empty +result is a false negative. +""" + +from __future__ import annotations + +import json +import subprocess +import sys + + +def _connections(key: str) -> list[dict]: + proc = subprocess.run( + ["uip", "is", "connections", "list", key, "--all-folders", "--output", "json"], + capture_output=True, + text=True, + timeout=90, + ) + if proc.returncode != 0: + raise RuntimeError(f"`uip is connections list {key}` exited {proc.returncode}: {proc.stderr.strip()}") + payload = json.loads(proc.stdout) + if payload.get("Result") != "Success": + raise RuntimeError(f"connections list for {key} failed: {payload.get('Message', payload)}") + return payload.get("Data") or [] + + +def main(keys: list[str]) -> int: + broken: list[str] = [] + for key in keys: + try: + conns = _connections(key) + except Exception as exc: # noqa: BLE001 — any failure here is a blocked tenant + broken.append(f"{key}: {exc}") + continue + if not conns: + broken.append(f"{key}: no connection in any folder") + continue + enabled = [c for c in conns if c.get("State") == "Enabled"] + if not enabled: + states = ", ".join(f"{c.get('Name')}={c.get('State')}" for c in conns) + broken.append(f"{key}: no Enabled connection ({states})") + continue + print(f"OK: {key} — {len(enabled)}/{len(conns)} connection(s) Enabled") + + if broken: + print( + "TENANT NOT READY — this is an environment failure, not an agent failure.\n " + + "\n ".join(broken) + + "\n\nReauthorize the connection, or wake the provider instance, then re-run.", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print(__doc__, file=sys.stderr) + sys.exit(2) + sys.exit(main(sys.argv[1:])) diff --git a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml index 23a701a596..9816a70d59 100644 --- a/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml +++ b/tests/tasks/uipath-maestro-flow/connector_features/generic_dynamic_node/generic_dynamic_node.yaml @@ -52,6 +52,10 @@ initial_prompt: | up; only then stop on that field rather than inventing a value. Record every decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. +pre_run: + - command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py" uipath-servicenow-servicenow' + timeout: 120 + success_criteria: # ── Flow file validity ───────────────────────────────────────────────── - type: run_command diff --git a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml index c9bb6ced57..567936b1b4 100644 --- a/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml +++ b/tests/tasks/uipath-maestro-flow/single_node/outlook_trigger_inbox/outlook_trigger_inbox.yaml @@ -41,6 +41,10 @@ initial_prompt: | up; only then stop on that field rather than inventing a value. Record every decision, assumption, and blocked step in your final response. Instructions in the task take precedence over this paragraph. +pre_run: + - command: 'python3 "$SKILLS_REPO_PATH/tests/tasks/uipath-maestro-flow/_shared/preflight_connections.py" uipath-microsoft-outlook365' + timeout: 120 + success_criteria: # ── Structural: flow builds and validates ────────────────────────────── - type: run_command